{
  "dataset": "dataset/items.jsonl",
  "dataset_sha256": "8eeb9bae57d81c014516df62eef48fc9a11471214148cf4fdecd4e252a714d31",
  "run_at": "2026-05-26T13:47:24.646312+00:00",
  "prompt_protocol": "structured-json-v1",
  "results": [
    {
      "model": "gpt-5.1",
      "total_items": 14,
      "correct": 10,
      "partial": 0,
      "incorrect": 4,
      "accuracy": 0.7143,
      "partial_credit_score": 0.7143,
      "latency_mean_ms": 1959.4,
      "latency_p50_ms": 1825.5,
      "items": [
        {
          "item_id": "SB-001",
          "response": "{\"answer\":\"one\",\"confidence\":\"certain\",\"notes\":\"The person has a single hat on their head and is holding it up with one hand.\"}",
          "parsed_answer": "one",
          "parse_mode": "json",
          "score": 0.0,
          "score_label": "incorrect",
          "latency_ms": 2485
        },
        {
          "item_id": "SB-002",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"The mirror image shows two stacked hats while the real figure has only one hat.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1924
        },
        {
          "item_id": "SB-003",
          "response": "{\"answer\": \"no\", \"confidence\": \"certain\", \"notes\": \"There are three buoys above the water but only two in the reflection.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1795
        },
        {
          "item_id": "SB-004",
          "response": "{\"answer\": \"7\", \"confidence\": \"certain\", \"notes\": \"Six extended digits plus one thumb are visible.\"}",
          "parsed_answer": "7",
          "parse_mode": "json",
          "score": 0.0,
          "score_label": "incorrect",
          "latency_ms": 1467
        },
        {
          "item_id": "SB-005",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"The sign reads \u201cP1Z7A?\u201d, which is not standard readable English text.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1856
        },
        {
          "item_id": "SB-006",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"One glasses arm is floating and not attached to the frame on one side.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2020
        },
        {
          "item_id": "SB-007",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"The two posts cast shadows in opposite directions.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1372
        },
        {
          "item_id": "SB-008",
          "response": "{\"answer\": 5, \"confidence\": \"certain\"}",
          "parsed_answer": "5",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1701
        },
        {
          "item_id": "SB-009",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"The handle is floating beside the mug instead of being attached to it.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1658
        },
        {
          "item_id": "SB-010",
          "response": "{\"answer\": 3, \"confidence\": \"certain\"}",
          "parsed_answer": "3",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1124
        },
        {
          "item_id": "SB-011",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"The text reads \u201cL1BRXRY? WAYF1ND\u201d, which is not standard readable English spelling for a street sign.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2073
        },
        {
          "item_id": "SB-012",
          "response": "{\"answer\":\"1\",\"confidence\":\"certain\",\"notes\":\"The man is wearing a single wide-brimmed hat; the image is edited with an extra face but no additional hats.\"}",
          "parsed_answer": "1",
          "parse_mode": "json",
          "score": 0.0,
          "score_label": "incorrect",
          "latency_ms": 2501
        },
        {
          "item_id": "SB-013",
          "response": "{\"answer\": 4, \"confidence\": \"certain\"}",
          "parsed_answer": "4",
          "parse_mode": "json",
          "score": 0.0,
          "score_label": "incorrect",
          "latency_ms": 1588
        },
        {
          "item_id": "SB-014",
          "response": "{\"answer\": \"3\", \"confidence\": \"certain\"}",
          "parsed_answer": "3",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 3867
        }
      ],
      "model_ref": "openai/gpt-5.1"
    },
    {
      "model": "gpt-5.2",
      "total_items": 14,
      "correct": 13,
      "partial": 1,
      "incorrect": 0,
      "accuracy": 0.9286,
      "partial_credit_score": 0.9643,
      "latency_mean_ms": 1814.3,
      "latency_p50_ms": 1497.5,
      "items": [
        {
          "item_id": "SB-001",
          "response": "{\"answer\":2,\"confidence\":\"certain\",\"notes\":\"One cowboy hat on the head and another being held up in the hand.\"}",
          "parsed_answer": "2",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2848
        },
        {
          "item_id": "SB-002",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"Mirror shows two stacked hats, while the real figure has only one.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1305
        },
        {
          "item_id": "SB-003",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"Three buoys are above the waterline, but only two reflections are shown below.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1444
        },
        {
          "item_id": "SB-004",
          "response": "{\"answer\":6,\"confidence\":\"certain\"}",
          "parsed_answer": "6",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1660
        },
        {
          "item_id": "SB-005",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"The sign text reads \\\"P1Z7A?\\\", which is not standard readable English (looks like garbled \\\"PIZZA?\\\").\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1443
        },
        {
          "item_id": "SB-006",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"One glasses arm is detached/floating above the face instead of attaching.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1366
        },
        {
          "item_id": "SB-007",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1001
        },
        {
          "item_id": "SB-008",
          "response": "{\"answer\":5,\"confidence\":\"certain\"}",
          "parsed_answer": "5",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1207
        },
        {
          "item_id": "SB-009",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"The handle is detached and floating apart from the mug body.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1551
        },
        {
          "item_id": "SB-010",
          "response": "{\"answer\":3,\"confidence\":\"certain\"}",
          "parsed_answer": "3",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1114
        },
        {
          "item_id": "SB-011",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"Visible text is leetspeak (e.g., 'L1BRXRY? WAYF1ND'), not readable English street sign text.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1753
        },
        {
          "item_id": "SB-012",
          "response": "{\"answer\":2,\"confidence\":\"certain\",\"notes\":\"Two overlapping hats are visible on the man's head.\"}",
          "parsed_answer": "2",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2766
        },
        {
          "item_id": "SB-013",
          "response": "{\"answer\":5,\"confidence\":\"uncertain\",\"notes\":\"I can see two chairs on the left side, one at the far/top side, and two on the right side; some are partially visible.\"}",
          "parsed_answer": "5",
          "parse_mode": "json",
          "score": 0.5,
          "score_label": "partial",
          "latency_ms": 3692
        },
        {
          "item_id": "SB-014",
          "response": "{\"answer\":3,\"confidence\":\"certain\",\"notes\":\"The image shows a bicycle with two main wheels and an additional smaller wheel attached near the front.\"}",
          "parsed_answer": "3",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2250
        }
      ],
      "model_ref": "openai/gpt-5.2"
    },
    {
      "model": "gpt-5.4",
      "total_items": 14,
      "correct": 14,
      "partial": 0,
      "incorrect": 0,
      "accuracy": 1.0,
      "partial_credit_score": 1.0,
      "latency_mean_ms": 1717.6,
      "latency_p50_ms": 1490.5,
      "items": [
        {
          "item_id": "SB-001",
          "response": "{\"answer\":\"2\",\"confidence\":\"certain\"}",
          "parsed_answer": "2",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2748
        },
        {
          "item_id": "SB-002",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"The reflected figure does not horizontally match the real figure; the hat/arm orientation is inconsistent with a true mirror image.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1646
        },
        {
          "item_id": "SB-003",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"There are three buoys in the real scene above the water, but only two reflected below.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2848
        },
        {
          "item_id": "SB-004",
          "response": "{\"answer\":\"6\",\"confidence\":\"certain\"}",
          "parsed_answer": "6",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1040
        },
        {
          "item_id": "SB-005",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"The sign text appears garbled ('P1Z7A?') and is not readable English.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1435
        },
        {
          "item_id": "SB-006",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"One glasses arm is detached/floating above the face instead of attaching correctly on that side.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1546
        },
        {
          "item_id": "SB-007",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"The two posts cast shadows in opposite directions.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1126
        },
        {
          "item_id": "SB-008",
          "response": "{\"answer\":\"5\",\"confidence\":\"certain\"}",
          "parsed_answer": "5",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1142
        },
        {
          "item_id": "SB-009",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"The handle is detached and not attached to the mug.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1356
        },
        {
          "item_id": "SB-010",
          "response": "{\"answer\":\"3\",\"confidence\":\"certain\"}",
          "parsed_answer": "3",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1182
        },
        {
          "item_id": "SB-011",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"The visible text appears stylized/gibberish and is not readable English.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2091
        },
        {
          "item_id": "SB-012",
          "response": "{\"answer\":\"2\",\"confidence\":\"certain\",\"notes\":\"The image shows one main seated man wearing a hat, plus an additional overlaid face also wearing a hat, making two visible hats on the man.\"}",
          "parsed_answer": "2",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2757
        },
        {
          "item_id": "SB-013",
          "response": "{\"answer\":\"5\",\"confidence\":\"certain\"}",
          "parsed_answer": "5",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1275
        },
        {
          "item_id": "SB-014",
          "response": "{\"answer\":\"3\",\"confidence\":\"certain\",\"notes\":\"The bicycle has two standard wheels plus an additional smaller wheel attached in the middle/front area.\"}",
          "parsed_answer": "3",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1854
        }
      ],
      "model_ref": "openai/gpt-5.4"
    },
    {
      "model": "gpt-5.5",
      "total_items": 14,
      "correct": 11,
      "partial": 0,
      "incorrect": 3,
      "accuracy": 0.7857,
      "partial_credit_score": 0.7857,
      "latency_mean_ms": 3590.9,
      "latency_p50_ms": 2780.5,
      "items": [
        {
          "item_id": "SB-001",
          "response": "{\"answer\":\"1\",\"confidence\":\"certain\"}",
          "parsed_answer": "1",
          "parse_mode": "json",
          "score": 0.0,
          "score_label": "incorrect",
          "latency_ms": 3928
        },
        {
          "item_id": "SB-002",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"The mirror image includes an extra hat/shape above the head that is not present in the real figure.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 3776
        },
        {
          "item_id": "SB-003",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2117
        },
        {
          "item_id": "SB-004",
          "response": "[ERROR: empty output in response]",
          "parsed_answer": "",
          "parse_mode": "error",
          "score": 0.0,
          "score_label": "incorrect",
          "latency_ms": 5259
        },
        {
          "item_id": "SB-005",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2176
        },
        {
          "item_id": "SB-006",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\",\"notes\":\"One glasses arm is detached/floating instead of attached.\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2673
        },
        {
          "item_id": "SB-007",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2266
        },
        {
          "item_id": "SB-008",
          "response": "{\"answer\":\"5\",\"confidence\":\"certain\"}",
          "parsed_answer": "5",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 1895
        },
        {
          "item_id": "SB-009",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2218
        },
        {
          "item_id": "SB-010",
          "response": "{\"answer\":3,\"confidence\":\"certain\"}",
          "parsed_answer": "3",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2724
        },
        {
          "item_id": "SB-011",
          "response": "{\"answer\":\"no\",\"confidence\":\"certain\"}",
          "parsed_answer": "no",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 3597
        },
        {
          "item_id": "SB-012",
          "response": "{\"answer\":\"2\",\"confidence\":\"certain\"}",
          "parsed_answer": "2",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 7230
        },
        {
          "item_id": "SB-013",
          "response": "[ERROR: empty output in response]",
          "parsed_answer": "",
          "parse_mode": "error",
          "score": 0.0,
          "score_label": "incorrect",
          "latency_ms": 7577
        },
        {
          "item_id": "SB-014",
          "response": "{\"answer\":\"3\",\"confidence\":\"certain\"}",
          "parsed_answer": "3",
          "parse_mode": "json",
          "score": 1.0,
          "score_label": "correct",
          "latency_ms": 2837
        }
      ],
      "model_ref": "openai/gpt-5.5"
    }
  ]
}