{ "summary": { "synthetic": true, "note": "Constructed examples; no model, judge, production or timing experiment was executed. Costs are invented, not provider prices. Task success includes correct handling of missing data.", "paired": { "n": 100, "aPass": 72, "bPass": 82, "bothPass": 68, "repaired": 14, "regressed": 4, "bothFail": 14, "aMissingPass": 12, "bMissingPass": 18, "aCompletePass": 60, "bCompletePass": 64, "aP50": 12, "bP50": 15, "aP95": 38, "bP95": 52, "aTotalCostYuan": 8, "bTotalCostYuan": 12 }, "judge": { "n": 40, "tp": 17, "fn": 3, "fp": 2, "tn": 18, "recall": 0.85, "precision": 0.8947368421052632, "specificity": 0.9, "accuracy": 0.875 } }, "pairs": [ { "id": "demo-001", "slice": "missing_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-002", "slice": "missing_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-003", "slice": "missing_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-004", "slice": "missing_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-005", "slice": "missing_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-006", "slice": "missing_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-007", "slice": "missing_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-008", "slice": "missing_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-009", "slice": "missing_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-010", "slice": "missing_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-011", "slice": "missing_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-012", "slice": "missing_source", "a_pass": true, "b_pass": false, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-013", "slice": "missing_source", "a_pass": false, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-014", "slice": "missing_source", "a_pass": false, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-015", "slice": "missing_source", "a_pass": false, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-016", "slice": "missing_source", "a_pass": false, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-017", "slice": "missing_source", "a_pass": false, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-018", "slice": "missing_source", "a_pass": false, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-019", "slice": "missing_source", "a_pass": false, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-020", "slice": "missing_source", "a_pass": false, "b_pass": false, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-021", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-022", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-023", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-024", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-025", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-026", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-027", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-028", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-029", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-030", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-031", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-032", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-033", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-034", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-035", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-036", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-037", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-038", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-039", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-040", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-041", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-042", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-043", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-044", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-045", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-046", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-047", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-048", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-049", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-050", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 12, "b_seconds": 15, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-051", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-052", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-053", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-054", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-055", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-056", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-057", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-058", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-059", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-060", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-061", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-062", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-063", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-064", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-065", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-066", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-067", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-068", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-069", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-070", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-071", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-072", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-073", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-074", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-075", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-076", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-077", "slice": "complete_source", "a_pass": true, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-078", "slice": "complete_source", "a_pass": true, "b_pass": false, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-079", "slice": "complete_source", "a_pass": true, "b_pass": false, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-080", "slice": "complete_source", "a_pass": true, "b_pass": false, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-081", "slice": "complete_source", "a_pass": false, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-082", "slice": "complete_source", "a_pass": false, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-083", "slice": "complete_source", "a_pass": false, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-084", "slice": "complete_source", "a_pass": false, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-085", "slice": "complete_source", "a_pass": false, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-086", "slice": "complete_source", "a_pass": false, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-087", "slice": "complete_source", "a_pass": false, "b_pass": true, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-088", "slice": "complete_source", "a_pass": false, "b_pass": false, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-089", "slice": "complete_source", "a_pass": false, "b_pass": false, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-090", "slice": "complete_source", "a_pass": false, "b_pass": false, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-091", "slice": "complete_source", "a_pass": false, "b_pass": false, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-092", "slice": "complete_source", "a_pass": false, "b_pass": false, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-093", "slice": "complete_source", "a_pass": false, "b_pass": false, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-094", "slice": "complete_source", "a_pass": false, "b_pass": false, "a_seconds": 24, "b_seconds": 30, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-095", "slice": "complete_source", "a_pass": false, "b_pass": false, "a_seconds": 38, "b_seconds": 52, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-096", "slice": "complete_source", "a_pass": false, "b_pass": false, "a_seconds": 38, "b_seconds": 52, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-097", "slice": "complete_source", "a_pass": false, "b_pass": false, "a_seconds": 38, "b_seconds": 52, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-098", "slice": "complete_source", "a_pass": false, "b_pass": false, "a_seconds": 38, "b_seconds": 52, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-099", "slice": "complete_source", "a_pass": false, "b_pass": false, "a_seconds": 38, "b_seconds": 52, "a_cost_fen": 8, "b_cost_fen": 12 }, { "id": "demo-100", "slice": "complete_source", "a_pass": false, "b_pass": false, "a_seconds": 38, "b_seconds": 52, "a_cost_fen": 8, "b_cost_fen": 12 } ], "judges": [ { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "fail" }, { "human": "fail", "judge": "pass" }, { "human": "fail", "judge": "pass" }, { "human": "fail", "judge": "pass" }, { "human": "pass", "judge": "fail" }, { "human": "pass", "judge": "fail" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" }, { "human": "pass", "judge": "pass" } ] }