{ "name": "AutoFactory-Bench", "version": "ef1.2", "generatedAt": "2026-10-24T20:03:37+00:00", "models": [ { "id": "gpt-6-astra", "label": "GPT-6 Astra", "org": "OpenAI", "harness": "Codex", "status": "complete" }, { "id": "gpt-6.1-sol", "label": "GPT-6.1 Sol", "org": "OpenAI", "harness": "Codex", "status": "complete" }, { "id": "claude-fable-5-1", "label": "Fable 5.1", "org": "Anthropic", "harness": "Claude Code", "status": "complete" }, { "id": "claude-opus-5-5", "label": "Opus 5.5", "org": "Anthropic", "harness": "Claude Code", "status": "complete" }, { "id": "deepseek-v4.1-flash-ultrafast", "label": "DeepSeek V4.1 Flash", "org": "DeepSeek", "harness": "mini-swe-agent", "status": "complete" }, { "id": "claude-sonnet-5-5", "label": "Sonnet 5.5", "org": "Anthropic", "harness": "Claude Code", "status": "complete" }, { "id": "claude-haiku-5-5", "label": "Haiku 5.5", "org": "Anthropic", "harness": "Claude Code", "status": "complete" }, { "id": "glm-5.3-flash-ultrafast", "label": "GLM-5.3 Flash", "org": "Z.ai", "harness": "mini-swe-agent", "status": "complete" }, { "id": "gpt-6-luna", "label": "GPT-6 Luna", "org": "OpenAI", "harness": "Codex", "status": "complete" } ], "leaderboard": [ { "modelId": "gpt-6-astra", "overall": 0.9167, "easy": 1.0, "medium": 1.0, "hard": 0.7222, "hardPassAt3": 0.8333, "safetyViolationRate": 0.0331, "trials": 160, "noProgramTrials": 0, "costPerTrial": 0.7216 }, { "modelId": "gpt-6.1-sol", "overall": 0.91, "easy": 1.0, "medium": 1.0, "hard": 0.7, "hardPassAt3": 0.8667, "safetyViolationRate": 0.038, "trials": 160, "noProgramTrials": 0, "costPerTrial": 0.1702 }, { "modelId": "claude-fable-5-1", "overall": 0.8167, "easy": 0.9333, "medium": 0.925, "hard": 0.5556, "hardPassAt3": 0.8, "safetyViolationRate": 0.1262, "trials": 160, "noProgramTrials": 0, "costPerTrial": 3.6808 }, { "modelId": "claude-opus-5-5", "overall": 0.7508, "easy": 0.8667, "medium": 0.775, "hard": 0.6028, "hardPassAt3": 0.8667, "safetyViolationRate": 0.0746, "trials": 163, "noProgramTrials": 0, "costPerTrial": 0.5631 }, { "modelId": "deepseek-v4.1-flash-ultrafast", "overall": 0.7333, "easy": 0.9333, "medium": 0.975, "hard": 0.2111, "hardPassAt3": 0.3667, "safetyViolationRate": 0.2843, "trials": 160, "noProgramTrials": 0, "costPerTrial": 0.0135 }, { "modelId": "claude-sonnet-5-5", "overall": 0.718, "easy": 0.7667, "medium": 0.9, "hard": 0.4267, "hardPassAt3": 0.6333, "safetyViolationRate": 0.0878, "trials": 166, "noProgramTrials": 0, "costPerTrial": 0.2506 }, { "modelId": "claude-haiku-5-5", "overall": 0.6433, "easy": 0.9333, "medium": 0.825, "hard": 0.1111, "hardPassAt3": 0.2, "safetyViolationRate": 0.2862, "trials": 165, "noProgramTrials": 0, "costPerTrial": 0.034 }, { "modelId": "glm-5.3-flash-ultrafast", "overall": 0.4033, "easy": 0.7667, "medium": 0.4, "hard": 0.0444, "hardPassAt3": 0.1333, "safetyViolationRate": 0.2549, "trials": 160, "noProgramTrials": 0, "costPerTrial": 0.0127 }, { "modelId": "gpt-6-luna", "overall": 0.3567, "easy": 0.3333, "medium": 0.575, "hard": 0.0889, "hardPassAt3": 0.1667, "safetyViolationRate": 0.2748, "trials": 160, "noProgramTrials": 41, "costPerTrial": 0.0088 } ], "tasks": [ { "id": "MAT-E01", "title": "Product Acquisition with Gripper", "aim": "Acquire the single supported product with the parallel gripper.", "suite": "material", "difficulty": "easy", "family": "custody_acquisition", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gpt-4o--r1--c0", "success": "gpt-6-astra--4TWZ3vM--c0" } }, { "id": "MAT-E02", "title": "Release Product onto Receiver Support", "aim": "Release the gripped product onto the receiver support already beneath it.", "suite": "material", "difficulty": "easy", "family": "custody_placement", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-qwen3.5-plus--r2--c0", "success": "gpt-6.1-sol--tdLmRXY--c0" } }, { "id": "MAT-E03", "title": "Selective Product Release to Receiver Cells", "aim": "Release only the requested subset of three held products into their respective receiver cells.", "suite": "material", "difficulty": "easy", "family": "custody_placement", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "gpt-6-luna--xpfT3Dz--c0", "success": "claude-fable-5-1--NtWjLvN--c0" } }, { "id": "MAT-E04", "title": "Docked Product Handoff with Transaction Acknowledgement", "aim": "Hand off the docked product using a valid acknowledgement of the current acceptance transaction.", "suite": "material", "difficulty": "easy", "family": "custody_placement", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-claude-sonnet-5--r1--c1", "success": "claude-opus-5-5--spDqeYM--c0" } }, { "id": "MAT-E05", "title": "Align Plate to Datum Faces and Retract Alignment Fingers", "aim": "Align the supported plate to both datum faces and retract both alignment fingers.", "suite": "material", "difficulty": "easy", "family": "material_registration", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gpt-4o--r1--c0", "success": "claude-sonnet-5-5--PSupH7B--c0" } }, { "id": "MAT-E06", "title": "Eject Product and Return Pusher Home", "aim": "Eject one product from the already aligned pocket and return the pusher home.", "suite": "material", "difficulty": "easy", "family": "single_piece_feed", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gpt-4o--r1--c0", "success": "claude-sonnet-5-5--qdnveHL--c0" } }, { "id": "MAT-E07", "title": "Plate Transfer to Receiver", "aim": "Transfer the entire plate along the prepared continuous support into the receiver and return the pusher.", "suite": "material", "difficulty": "easy", "family": "supported_transfer", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gpt-4o--r1--c0", "success": "claude-haiku-5-5--v3BwDat--c0" } }, { "id": "MAT-E08", "title": "Convey Product to Receiving Boundary", "aim": "Convey the single product until its trailing edge has passed the receiving boundary, then stop it fully supported on the conveyor.", "suite": "material", "difficulty": "easy", "family": "single_segment_transfer", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gpt-4o--r2--c1", "success": "glm-5.3-flash-ultrafast--Jh9UGNJ--c0" } }, { "id": "MAT-E09", "title": "Liquid Volume Delivery to Receiver", "aim": "Deliver the requested actual liquid volume into the receiver through the fixed nozzle.", "suite": "material", "difficulty": "easy", "family": "quantity_transfer", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--bzXgs9G--c0", "success": "gpt-6.1-sol--j6Fo2AR--c0" } }, { "id": "MAT-E10", "title": "Maintain Material Buffer Inventory", "aim": "Maintain material buffer inventory within its permitted band throughout one external withdrawal interval.", "suite": "material", "difficulty": "easy", "family": "inventory_regulation", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gpt-4o--r1--c1", "success": "gpt-6-astra--oki4CfY--c0" } }, { "id": "MAT-E11", "title": "Material Edge Control During External Feed", "aim": "Keep the material edge inside the allowed corridor throughout one externally driven feed interval.", "suite": "material", "difficulty": "easy", "family": "material_registration", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-qwen3.5-plus--r2--c0", "success": "claude-fable-5-1--FXcQ72B--c0" } }, { "id": "MAT-E12", "title": "Material Displacement and Recovery Task", "aim": "Displace the old residual material into the recovery path until the requested new material reaches the outlet, then stop.", "suite": "material", "difficulty": "easy", "family": "line_identity", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "success": "claude-fable-5-1--A2LMSqB--c0" } }, { "id": "MAT-E13", "title": "Product Routing Based on Key Match", "aim": "Route the current single product to the accepted pocket if its key matches the request, otherwise to the bypass pocket.", "suite": "material", "difficulty": "easy", "family": "identity_routing", "conditions": 4, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--gnWavrj--c0", "success": "claude-sonnet-5-5--FiE5NZ2--c0" } }, { "id": "MAT-E14", "title": "Product Transfer with Load Balancing", "aim": "Transfer the single prepared product to an available side that minimizes the absolute left/right load difference after placement.", "suite": "material", "difficulty": "easy", "family": "supported_transfer", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gpt-4o--r1--c0", "success": "claude-sonnet-5-5--mPthaJS--c0" } }, { "id": "MAT-E15", "title": "Container Opening and Holding Task", "aim": "Open the already bilaterally held container to at least the requested actual passage width and maintain it.", "suite": "material", "difficulty": "easy", "family": "material_geometry", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--haUmaR9--c0", "success": "gpt-6-astra--nV9r5Av--c0" } }, { "id": "MAT-H01", "title": "MAT-H01", "aim": "Process two A-order and two B-order workpieces using their respective machining cells, one shared single-gripper robot, one inspection fixture and one optional waiting fixture.", "suite": "material", "difficulty": "hard", "family": "shared_dual_machine_tending", "conditions": 6, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.3333, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.6667, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.3333 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": { "gpt-6-luna": 2 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "some", "gpt-6.1-sol": "every", "gpt-6-luna": "some", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "some" }, "replays": { "failure": "claude-haiku-5-5--TBXz32Q--c0", "success": "gpt-6-luna--hp4MhT3--c0" } }, { "id": "MAT-H02", "title": "MAT-H02", "aim": "Maintain accessibility of bypass storage in situations where the FIFO lead and the required types of two kits differ, and transfer actual parts as kit members of the current order.", "suite": "material", "difficulty": "hard", "family": "material_controls_v14", "conditions": 9, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.3333, "claude-fable-5-1": 0.6667, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.037, "claude-fable-5-1": 0.037, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "some", "claude-fable-5-1": "some", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "some", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-fable-5-1--WuUvyyj--c5", "success": "claude-sonnet-5-5--4QyAnYf--c0" } }, { "id": "MAT-H03", "title": "MAT-H03", "aim": "While preserving the product and jig separately, connect inspection, product acquisition, and actual recovery of the empty jig, and confirm the full loading of the lift and the point at which the jig can be reused.", "suite": "material", "difficulty": "hard", "family": "material_remaining_v10", "conditions": 6, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.3333, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.3333 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.3333 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": { "gpt-6-luna": 3 }, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "some", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "some" }, "replays": { "failure": "glm-5.3-flash-ultrafast--UM7MGQM--c0", "success": "claude-opus-5-5--DroS9eU--c0" } }, { "id": "MAT-H04", "title": "MAT-H04", "aim": "Connect the separate arrival and preparation of packaging materials and contents, verify both openings retention, pair ID, and actual inclusion depth to accept the completed packaging.", "suite": "material", "difficulty": "hard", "family": "material_controls_v14", "conditions": 9, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 0.6667, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 0.3333 }, "safetyViolationRate": { "claude-opus-5-5": 0.3333, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.3333, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "some", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "some", "glm-5.3-flash-ultrafast": "some" }, "replays": { "failure": "claude-haiku-5-5--JDUyaWY--c0", "success": "deepseek-v4.1-flash-ultrafast--t832TM5--c0" } }, { "id": "MAT-H05", "title": "MAT-H05", "aim": "When the pockets of other work stages rotate together, maintain the retraction of all tools and the physical pair/inspection generation, and accept the finished product with the actual ejector.", "suite": "material", "difficulty": "hard", "family": "material_controls_v14", "conditions": 9, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.3333, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.3333, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0741, "claude-haiku-5-5": 0.6667, "gpt-6.1-sol": 0.3333, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.6667, "gpt-6-astra": 0.6667, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 1.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "none", "gpt-6.1-sol": "some", "gpt-6-luna": "none", "claude-fable-5-1": "some", "gpt-6-astra": "none", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "gpt-6.1-sol--iZXPncE--c0", "success": "claude-opus-5-5--dVUC5XT--c0" } }, { "id": "MAT-H06", "title": "MAT-H06", "aim": "While controlling reel exchange and leading-end connection, continue downstream cut-piece production with dancer storage, and exclude the connection section from normal orders to separately accept it as actual scrap.", "suite": "material", "difficulty": "hard", "family": "material_controls_v14", "conditions": 9, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.3333, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.3704, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.6667, "claude-fable-5-1": 0.6667, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "some", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "some", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "deepseek-v4.1-flash-ultrafast--DP7Usrm--c0", "success": "gpt-6-astra--3yK6Duf--c0" } }, { "id": "MAT-H07", "title": "Core preparation·registration winding·independent roll extraction cell", "aim": "Register and combine the A/B webs into the requested rolls, then secure and deliver each roll.", "suite": "material", "difficulty": "hard", "family": "material_priority_wind_tape", "conditions": 11, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 0.3333, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.3333, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.3333, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.6667, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0303, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.6667, "glm-5.3-flash-ultrafast": 0.6667 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "some", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "some", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-fable-5-1--uhokmny--c5", "success": "gpt-6.1-sol--ZvPFAjj--c0" } }, { "id": "MAT-H08", "title": "MAT-H08", "aim": "Actually disconnect the connecting edge shared by the product area and the residue, and separately complete the support acceptance of the product and the storage·winding of the perforated same residue.", "suite": "material", "difficulty": "hard", "family": "material_controls_v14", "conditions": 9, "passAt1": { "claude-opus-5-5": 0.75, "claude-sonnet-5-5": 0.8, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.3333, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.25, "claude-sonnet-5-5": 0.0222, "claude-haiku-5-5": 0.4, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.3704, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.7407, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 4, "claude-sonnet-5-5": 5, "claude-haiku-5-5": 5, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "some", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "some", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "deepseek-v4.1-flash-ultrafast--HmfftcJ--c0", "success": "claude-fable-5-1--cNvMRgf--c0" } }, { "id": "MAT-H09", "title": "MAT-H09", "aim": "While the two filling lines share a common tank and common capping head, adjust batch replenishment, container arrival, and exit takeover delay, preserving the actual received amount per container and capping attribution.", "suite": "material", "difficulty": "hard", "family": "material_stations_v7_liquid", "conditions": 6, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.6667, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.6667, "glm-5.3-flash-ultrafast": 0.3333 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 5, "claude-haiku-5-5": 5, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "some", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-haiku-5-5--yZFqQaa--c0", "success": "claude-sonnet-5-5--Hd3xPAo--c0" } }, { "id": "MAT-H10", "title": "MAT-H10", "aim": "Physically exchange the finite supply magazine while preserving the magazine inventory, outgoing extraction transactions, and two consumption requests separately.", "suite": "material", "difficulty": "hard", "family": "material_stations_v7_magazine", "conditions": 6, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.6667, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.3333, "gpt-6-astra": 0.6667, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.6667, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.6667, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 1.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "none", "gpt-6.1-sol": "some", "gpt-6-luna": "none", "claude-fable-5-1": "some", "gpt-6-astra": "some", "deepseek-v4.1-flash-ultrafast": "some", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-haiku-5-5--868qDbH--c0", "success": "gpt-6-astra--EEG9abd--c0" } }, { "id": "MAT-H11", "title": "MAT-H11", "aim": "After flipping the product, distinguish the holding pads that cover the working surface, and release the pads after the edge holder takes over the support to complete double-sided work.", "suite": "material", "difficulty": "hard", "family": "material_controls_v14", "conditions": 9, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 0.3333, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.6667, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.3333, "claude-fable-5-1": 0.7037, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.3333 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "some", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "none", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-fable-5-1--Zaex6vb--c1", "success": "gpt-6.1-sol--62BQjXo--c0" } }, { "id": "MAT-H12", "title": "MAT-H12", "aim": "Before accepting different production requests that cannot be interrupted, sum the current occupancy and confirmed undelivered residues, and independently adjust the good product exit and shared empty exchange.", "suite": "material", "difficulty": "hard", "family": "material_stations_v7_scrap", "conditions": 6, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.3333, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.3333 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "some", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "some" }, "replays": { "failure": "glm-5.3-flash-ultrafast--FwKrmuo--c0", "success": "claude-fable-5-1--j3D3afK--c0" } }, { "id": "MAT-H13", "title": "MAT-H13", "aim": "Control independent single-piece feeding and bundle forming to secure actual members, and coordinate forming, gripper/chuck transfer, film by specification, tape, and exit operations.", "suite": "material", "difficulty": "hard", "family": "material_controls_v14", "conditions": 10, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.3333, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.3333, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.3333, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.3333, "gpt-6-luna": 0.3333, "claude-fable-5-1": 0.6667, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.6667, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": { "gpt-6-luna": 2 }, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "none", "gpt-6.1-sol": "some", "gpt-6-luna": "none", "claude-fable-5-1": "some", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "gpt-6.1-sol--U3v48Jj--c0", "success": "gpt-6-astra--4GgwrGR--c0" } }, { "id": "MAT-H14", "title": "MAT-H14", "aim": "While tray arrival, the different completion order of two kit operations, and the shared completion scale and acceptance/hold exit delays overlap, connect the before and after actual measured masses of the same tray and hand it over to the exit matching the net increase.", "suite": "material", "difficulty": "hard", "family": "material_stations_v7_weigh", "conditions": 8, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--HvAyEBz--c0", "success": "deepseek-v4.1-flash-ultrafast--NDZFLLT--c0" } }, { "id": "MAT-H15", "title": "MAT-H15", "aim": "Preserve the layer order and actual accumulated top surface of independent A/B orders, and after the lower fork takes support, transfer the completed bundle to the external support.", "suite": "material", "difficulty": "hard", "family": "material_controls_v14", "conditions": 9, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 0.6667, "claude-haiku-5-5": 0.3333, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.2593, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.6667, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.6667, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "some", "claude-haiku-5-5": "some", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-opus-5-5--oBQAovF--c0", "success": "claude-haiku-5-5--hMR4Rct--c0" } }, { "id": "MAT-M01", "title": "Web Delivery to Receiver with Buffer Preservation", "aim": "Deliver the requested length of web to the receiver while preserving the buffer's usable inventory.", "suite": "material", "difficulty": "medium", "family": "material.v6.buffer", "conditions": 6, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-qwen3.5-plus--r1--c0", "success": "claude-opus-5-5--joxjbcw--c0" } }, { "id": "MAT-M02", "title": "Plate Orientation and Placement Task", "aim": "Place the plate at the receiver with its actual orientation equal to the requested angle.", "suite": "material", "difficulty": "medium", "family": "material.v6.orientation", "conditions": 6, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 1.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--JnXwGU6--c0", "success": "claude-opus-5-5--teEUEC9--c0" } }, { "id": "MAT-M03", "title": "Align Workpiece on Receiver", "aim": "Place the workpiece at the receiver aligned to its reference faces.", "suite": "material", "difficulty": "medium", "family": "material.v6.datum", "conditions": 6, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-haiku-5-5--hkkKpeG--c0", "success": "deepseek-v4.1-flash-ultrafast--WWGh8YY--c0" } }, { "id": "MAT-M04", "title": "Deliver Inspected Workpieces to Receivers", "aim": "Deliver both inspected workpieces to their corresponding accept or reject receivers.", "suite": "material", "difficulty": "medium", "family": "material.v6.classification", "conditions": 6, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 1.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--DBBrbWN--c0", "success": "claude-sonnet-5-5--X8Ag28X--c0" } }, { "id": "MAT-M05", "title": "Four-Layer Material Stack Construction", "aim": "Build a four-layer stack in the requested bottom-to-top material order.", "suite": "material", "difficulty": "medium", "family": "material.v6.stack", "conditions": 6, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "none", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-fable-5-1--jGGJh7n--c0", "success": "claude-sonnet-5-5--2cJ8WXD--c0" } }, { "id": "MAT-M06", "title": "Liquid Volume Transfer to Receiving Vessel", "aim": "Transfer the requested liquid volume into the receiving vessel.", "suite": "material", "difficulty": "medium", "family": "material.metered_transfer", "conditions": 6, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "gpt-6-luna--KoLFG39--c0", "success": "claude-opus-5-5--PtPbSg9--c0" } }, { "id": "MAT-M07", "title": "Material Placement into Receiving Nests", "aim": "Place both held materials into the receiving nests at their requested relative spacing.", "suite": "material", "difficulty": "medium", "family": "material.pose_handoff", "conditions": 6, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-qwen3.5-plus--r2--c0", "success": "gpt-6-astra--wuxGwAb--c0" } }, { "id": "MAT-M08", "title": "Store New Plate in Vacant Slot", "aim": "Store one new plate in a vacant slot while preserving the existing stored plates.", "suite": "material", "difficulty": "medium", "family": "material.vacant_slot", "conditions": 6, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--BeMfdYi--c0", "success": "claude-fable-5-1--Pt6QbEW--c0" } }, { "id": "MAT-M09", "title": "Material Transfer from Sender to Receiver", "aim": "Transfer the currently held material from the sender to the receiver.", "suite": "material", "difficulty": "medium", "family": "material.transaction_handoff", "conditions": 6, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--WeBE6Ds--c0", "success": "claude-haiku-5-5--GQwNzLK--c0" } }, { "id": "MAT-M10", "title": "Material Delivery and Return Task", "aim": "Deliver the selected materials to the receiver and return the unselected materials to their corresponding source nests.", "suite": "material", "difficulty": "medium", "family": "material.requested_partition", "conditions": 6, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gpt-4o--r1--c0", "success": "claude-opus-5-5--cevyxax--c0" } }, { "id": "MAT-M11", "title": "Content Insertion into Flexible Receiver", "aim": "Insert the content into the flexible receiver while preserving a usable opening until transfer equipment is clear.", "suite": "material", "difficulty": "medium", "family": "material.maintained_aperture", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 2, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "paper-gpt-4o--r1--c0", "success": "gpt-6-astra--aSSq6pV--c0" } }, { "id": "MAT-M12", "title": "Product Removal and Scrap Retention Task", "aim": "Remove the product to the receiver and leave the four surrounding scraps separately held on the upper carrier.", "suite": "material", "difficulty": "medium", "family": "material.separate_carrier_custody", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-haiku-5-5--SmH2N6M--c0", "success": "gpt-6-luna--pPKf5nN--c0" } }, { "id": "MAT-M13", "title": "Eject Item from Indexed Carousel", "aim": "Eject the item with the requested identity from the indexed carousel.", "suite": "material", "difficulty": "medium", "family": "material.cyclic_carrier_correspondence", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "gpt-6-luna--H55quwQ--c0", "success": "claude-fable-5-1--GRXqcrJ--c0" } }, { "id": "MAT-M14", "title": "Product Supply and Launch Coordination", "aim": "Supply exactly one product into the launch nest, withdraw the supply finger, and time its transfer into the continuously moving receiving pocket.", "suite": "material", "difficulty": "medium", "family": "material.moving_receiver_capture", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-qwen3.5-plus--r2--c0", "success": "glm-5.3-flash-ultrafast--PfNh2Kr--c0" } }, { "id": "MAT-M15", "title": "Batch Accumulation and Dispatch Control", "aim": "Accumulate exactly the requested net quantity while distinguishing forward, reverse, and partial passages, then dispatch the fully supported batch on the receiving tray.", "suite": "material", "difficulty": "medium", "family": "material.directional_crossing", "conditions": 5, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "claude-opus-5-5--JUTFFQF--c0", "success": "claude-fable-5-1--hM25Dmb--c0" } }, { "id": "MAT-M16", "title": "Web Length Feeding and Securing Task", "aim": "Feed the requested web length while keeping its actual edge within the receiving corridor, then secure it.", "suite": "material", "difficulty": "medium", "family": "material.continuous_edge_registration", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.3333 }, "trials": { "claude-opus-5-5": 2, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "none", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--gbzSdS7--c1", "success": "deepseek-v4.1-flash-ultrafast--49aFXbm--c0" } }, { "id": "MAT-M17", "title": "Material Transfer to Receiver", "aim": "Transfer the requested material identity to the receiver without sending previous material there.", "suite": "material", "difficulty": "medium", "family": "material.line_content_identity", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--ZQe3gw4--c0", "success": "claude-sonnet-5-5--gAP2w9r--c0" } }, { "id": "MAT-M18", "title": "Tape Transfer to Receiver", "aim": "With a single start request, transfer one tape segment of length D0 (300–440 mm) to the receiving support.", "suite": "material", "difficulty": "medium", "family": "material_tape_separation", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 2, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gpt-5.5--r2--c0", "success": "glm-5.3-flash-ultrafast--6wMaY9L--c0" } }, { "id": "MAT-M19", "title": "Load Materials into Two-Sided Receiver with Balance", "aim": "Load all four materials into the two-sided receiver with balanced total mass and depart.", "suite": "material", "difficulty": "medium", "family": "material.load_distribution", "conditions": 4, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-claude-sonnet-5--r1--c0", "success": "gpt-6-luna--9i8QC2g--c0" } }, { "id": "MAT-M20", "title": "Board Transfer to Receiver with Span Constraint", "aim": "Transfer the board to the receiver without exceeding its allowed unsupported span.", "suite": "material", "difficulty": "medium", "family": "material.continuous_support_span", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 2, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gpt-5.5--r2--c0", "success": "gpt-6-astra--qyY5Ygn--c0" } }, { "id": "MOT-E01", "title": "Linear Axis Positioning Task", "aim": "Move the prepared linear axis to the requested absolute position and hold it at rest.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "none", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "claude-fable-5-1--69Q3vsn--c3", "success": "claude-opus-5-5--YgzHFrh--c0" } }, { "id": "MOT-E02", "title": "Home Return and Coordinate Validity Maintenance", "aim": "Execute home return and maintain a valid coordinate stop state.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gpt-4o--r3--c0", "success": "claude-opus-5-5--rs7GbRJ--c0" } }, { "id": "MOT-E03", "title": "Axis Deceleration and Stop Before Limit", "aim": "Decelerate and stop the moving axis before the limit position and hold.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "claude-sonnet-5-5--CUPYmmk--c1", "success": "deepseek-v4.1-flash-ultrafast--J9eCpHM--c0" } }, { "id": "MOT-E04", "title": "Drive and Maintain Shaft Speed", "aim": "Drive the rotating axis at the requested speed and maintain that speed.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gpt-4o--r3--c0", "success": "gpt-6-luna--4eV9cGz--c0" } }, { "id": "MOT-E05", "title": "Capture and Record Position on Mark Detection", "aim": "Capture and record the position at the moment a mark is detected during movement.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--EuEmdv5--c0", "success": "claude-sonnet-5-5--UScozF9--c0" } }, { "id": "MOT-E06", "title": "Vertical Axis Brake Engagement and Height Maintenance", "aim": "Engage the brake of the stopped vertical axis and maintain the height.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gpt-4o--r1--c0", "success": "gpt-6.1-sol--ZD2JvHF--c0" } }, { "id": "MOT-E07", "title": "Synchronization of Slave Axis to Master Axis", "aim": "Synchronize and maintain the slave axis to the master axis at the requested ratio.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "claude-opus-5-5--bMU72Ci--c0", "success": "gpt-6-astra--WxpoNux--c0" } }, { "id": "MOT-E08", "title": "Align and Maintain Following of Moving Reference Axis", "aim": "Match the requested interval with the moving master axis and maintain following.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.25, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 1.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--T4ygvp2--c0", "success": "claude-fable-5-1--3XytV8W--c0" } }, { "id": "MOT-E09", "title": "Maintain Tool Orientation During Robot Joint Movement", "aim": "Maintain the tool orientation while the robot joints are moving.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-qwen3.5-plus--r2--c0", "success": "claude-fable-5-1--rfRebhb--c0" } }, { "id": "MOT-E10", "title": "Rotate and Stop Index Disk at Requested Angle", "aim": "Rotate the index disk to the requested angle and stop.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "claude-sonnet-5-5--bniN9oF--c0", "success": "claude-opus-5-5--tJSQT3r--c0" } }, { "id": "MOT-E11", "title": "Door Obstacle Detection and Reopening", "aim": "If an approaching object is detected while the door is closing, reopen and maintain it.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "gpt-6-luna--FNS8vyr--c0", "success": "deepseek-v4.1-flash-ultrafast--VHfrWZd--c0" } }, { "id": "MOT-E12", "title": "Boom Retraction to Requested Length", "aim": "Retract the extended boom to the requested length and stop.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.25, "claude-sonnet-5-5": 0.25, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "claude-opus-5-5--i38PodC--c3", "success": "gpt-6.1-sol--25AeQ4S--c0" } }, { "id": "MOT-E13", "title": "Mobile Robot Path Following Task", "aim": "Drive the mobile robot along the center of the passage to the requested position and stop.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "claude-sonnet-5-5--CfeX3dN--c2", "success": "claude-fable-5-1--9zqkiB6--c0" } }, { "id": "MOT-E14", "title": "Hoist Sway Reduction and Trolley Positioning", "aim": "Reduce the sway of the hoist and stop the trolley at the current working position.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "none", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-sonnet-5-5--K6VtZXu--c0", "success": "deepseek-v4.1-flash-ultrafast--pk3x9EQ--c0" } }, { "id": "MOT-E15", "title": "Counterbalance Head Tilt", "aim": "Move the counterweight to offset the support moment while the head is tilting.", "suite": "motion", "difficulty": "easy", "family": "motion_easy_unit", "conditions": 4, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "deepseek-v4.1-flash-ultrafast--FJNKRxc--c0", "success": "claude-haiku-5-5--Lwk27GG--c0" } }, { "id": "MOT-H01", "title": "MOT-H01", "aim": "Process independent requests of the two hoist cells with the shared shuttle and rotary/approach cell, prepare the next task, and return each moving fixture to its original position without collision.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 6, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 0.6667, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.3333, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.3333, "claude-sonnet-5-5": 0.3333, "claude-haiku-5-5": 0.3333, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0556, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.6667, "glm-5.3-flash-ultrafast": 0.8889 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "some", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "some", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-opus-5-5--65TbTH4--c0", "success": "gpt-6-astra--7ww3yN8--c0" } }, { "id": "MOT-H02", "title": "MOT-H02", "aim": "Process multiple alignment requests in sequence with lift, gantry, and reference axis, and maintain the actual target position of each request before and after the reference change.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.3333, "claude-haiku-5-5": 0.6667, "gpt-6.1-sol": 0.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.4444, "glm-5.3-flash-ultrafast": 0.3333 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "none", "gpt-6.1-sol": "none", "gpt-6-luna": "none", "claude-fable-5-1": "none", "gpt-6-astra": "none", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "deepseek-v4.1-flash-ultrafast--NFaxmFK--c0", "success": "reference-c0" } }, { "id": "MOT-H03", "title": "MOT-H03", "aim": "Process two setup requests in sequence with movement connection and rotation synchronization, and return while observing relative speed and synchronization interval per request.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.6667, "claude-haiku-5-5": 0.6667, "gpt-6.1-sol": 0.3333, "gpt-6-luna": 1.0, "claude-fable-5-1": 0.4444, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "none", "gpt-6.1-sol": "none", "gpt-6-luna": "none", "claude-fable-5-1": "none", "gpt-6-astra": "none", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "gpt-6.1-sol--ci4kcp5--c0", "success": "reference-c0" } }, { "id": "MOT-H04", "title": "MOT-H04", "aim": "Process independent requests through lift, hoist, and inclined head, and connect to the next facility after settling vibrations.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.3333, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.6667, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.7778, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.7778, "glm-5.3-flash-ultrafast": 0.6667 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "none", "gpt-6.1-sol": "some", "gpt-6-luna": "none", "claude-fable-5-1": "none", "gpt-6-astra": "none", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "deepseek-v4.1-flash-ultrafast--G3KJgqo--c0", "success": "gpt-6.1-sol--s8sRGdm--c0" } }, { "id": "MOT-H05", "title": "MOT-H05", "aim": "Memorize the new registration values of the two requests respectively to align the subsequent index posture, retrieve the pin, and return the fixture.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.3333, "claude-sonnet-5-5": 0.3333, "claude-haiku-5-5": 0.3333, "gpt-6.1-sol": 0.3333, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.3333, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.3333, "gpt-6.1-sol": 0.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "some", "claude-haiku-5-5": "some", "gpt-6.1-sol": "some", "gpt-6-luna": "none", "claude-fable-5-1": "some", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "deepseek-v4.1-flash-ultrafast--hhr2NHA--c0", "success": "claude-opus-5-5--pV6WqPr--c0" } }, { "id": "MOT-H06", "title": "MOT-H06", "aim": "Safely retrieve the access mechanism and fixture of a canceled request in progress, and allow the pending request to continue using the same line.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.3333, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.3333, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.3333, "claude-fable-5-1": 0.3333, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 0.3333 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "some", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "some", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-fable-5-1--v4ecsmg--c0", "success": "claude-opus-5-5--CV4CCAw--c0" } }, { "id": "MOT-H07", "title": "MOT-H07", "aim": "Process two requests in succession with the variable boom and exchange head, avoiding interference during movement according to the boom length and head height.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.3333, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.3333, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 0.6667, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.7778, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.7778, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.7778, "glm-5.3-flash-ultrafast": 0.3333 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "none", "gpt-6.1-sol": "some", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "some", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "deepseek-v4.1-flash-ultrafast--csDwJDQ--c0", "success": "claude-opus-5-5--LvywkUH--c0" } }, { "id": "MOT-H08", "title": "MOT-H08", "aim": "Process independent requests with the common lift frame and gantry·incline head, and limit the support height difference and total moment.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.6667, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.4444, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.6667, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 0.7778 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "some", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-haiku-5-5--xkeYaC9--c0", "success": "gpt-6-astra--6T6gLxo--c0" } }, { "id": "MOT-H09", "title": "MOT-H09", "aim": "Assign compatible pockets to multiple access requests, maintain occupancy before retrieval, and process continuously to the rotation cell.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 0.3333, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.6667, "gpt-6-astra": 0.6667, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.5, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.5556, "claude-fable-5-1": 0.3333, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 4, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "some", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "some", "gpt-6-astra": "some", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-fable-5-1--zBoJm8P--c0", "success": "gpt-6.1-sol--gaKXTWR--c0" } }, { "id": "MOT-H10", "title": "MOT-H10", "aim": "Continue performing the lift·incline·hoist operations of independent requests, and maintain the support of the stationary axis even if other axes move.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.3333, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.3333, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 0.3333, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.6667, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.3333, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "none", "gpt-6.1-sol": "some", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "some", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "deepseek-v4.1-flash-ultrafast--7TNBC9g--c0", "success": "claude-fable-5-1--ghR7zkc--c0" } }, { "id": "MOT-H11", "title": "MOT-H11", "aim": "Pass each request through the rotation and index cell within the deadline, and assign the shared shuttle so that urgent requests are not trapped behind long tasks.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 0.3333, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.6667, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.4444, "claude-haiku-5-5": 0.7778, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.7778, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.3333 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "some", "claude-haiku-5-5": "none", "gpt-6.1-sol": "some", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-sonnet-5-5--28ryYRY--c1", "success": "claude-opus-5-5--2LYBj6L--c0" } }, { "id": "MOT-H12", "title": "MOT-H12", "aim": "Process the cross movement and head exchange of independent requests in succession, and do not pass the cross space to other movements before actual retreat.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.6667, "claude-sonnet-5-5": 0.3333, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.6667, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.3333, "gpt-6-astra": 0.3333, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.6667, "gpt-6.1-sol": 0.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.3333 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "some", "claude-haiku-5-5": "none", "gpt-6.1-sol": "some", "gpt-6-luna": "none", "claude-fable-5-1": "some", "gpt-6-astra": "some", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "deepseek-v4.1-flash-ultrafast--VttbeAp--c0", "success": "gpt-6.1-sol--FzsNsyn--c0" } }, { "id": "MOT-H13", "title": "MOT-H13", "aim": "Complete the rotation and synchronization section of each request, and maintain common reference drive while used by other requests.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.7778, "claude-fable-5-1": 0.6667, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.2222 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "none", "gpt-6.1-sol": "none", "gpt-6-luna": "none", "claude-fable-5-1": "none", "gpt-6-astra": "none", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "gpt-6-astra--EwcwhNK--c0", "success": "reference-c0" } }, { "id": "MOT-H14", "title": "MOT-H14", "aim": "Process two position setting requests, and if the common reference changes, discard the previous registered value and stop with the new detected value of that reference.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 0.3333, "claude-sonnet-5-5": 0.3333, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.3333, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.6667, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 1.0 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "some", "claude-sonnet-5-5": "some", "claude-haiku-5-5": "none", "gpt-6.1-sol": "none", "gpt-6-luna": "none", "claude-fable-5-1": "none", "gpt-6-astra": "some", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--7Ur8bdT--c0", "success": "gpt-6-astra--vHvfGKy--c0" } }, { "id": "MOT-H15", "title": "MOT-H15", "aim": "Process multiple requests in sequence with a balanced head and compatible pocket, and select a return path that allows the combined posture.", "suite": "motion", "difficulty": "hard", "family": "motion_hard15", "conditions": 3, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 0.6667, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.6667, "gpt-6-astra": 0.6667, "deepseek-v4.1-flash-ultrafast": 0.3333, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.3333, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.5556, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.4444, "glm-5.3-flash-ultrafast": 0.6667 }, "trials": { "claude-opus-5-5": 3, "claude-sonnet-5-5": 3, "claude-haiku-5-5": 3, "gpt-6.1-sol": 3, "gpt-6-luna": 3, "claude-fable-5-1": 3, "gpt-6-astra": 3, "deepseek-v4.1-flash-ultrafast": 3, "glm-5.3-flash-ultrafast": 3 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "some", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "some", "gpt-6-astra": "some", "deepseek-v4.1-flash-ultrafast": "some", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "deepseek-v4.1-flash-ultrafast--fAL7UZn--c1", "success": "claude-sonnet-5-5--hGD3amx--c0" } }, { "id": "MOT-M01", "title": "Swap and Position Heads Without Collision", "aim": "Swap the left and right positions of the two heads and stop them at the working height without collision during movement.", "suite": "motion", "difficulty": "medium", "family": "motion_medium_core", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 1.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--2qYUhkZ--c0", "success": "gpt-6-astra--Sw3tcNn--c0" } }, { "id": "MOT-M02", "title": "Gantry Alignment·Rail Connection·Empty Cart Transfer", "aim": "Connect the beam rails to the dock, transfer the entire empty cart to the fixed rail and park it, then disconnect and return only the empty beam to its origin.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-claude-sonnet-5--r1--c3", "success": "gpt-6.1-sol--WiF9ehW--c0" } }, { "id": "MOT-M03", "title": "Carrier Synchronization with Moving Interface", "aim": "Synchronize the carrier with the moving connection, and move together with the empty fork inserted.", "suite": "motion", "difficulty": "medium", "family": "motion_medium_core", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-sonnet-5-5--KfcpiYM--c0", "success": "deepseek-v4.1-flash-ultrafast--Jvgn99V--c0" } }, { "id": "MOT-M04", "title": "Empty Hook Hoisting and Stabilization", "aim": "Bring the hoist load to the requested rope length and trolley position, suppress sway, and maintain the stopped result. Safe overlapping motion of the rope and trolley is permitted.", "suite": "motion", "difficulty": "medium", "family": "motion_medium_core", "conditions": 5, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-haiku-5-5--23Bi99M--c0", "success": "deepseek-v4.1-flash-ultrafast--bcjz4qL--c0" } }, { "id": "MOT-M05", "title": "Head Tilt and Brake Engagement Task", "aim": "Maintain balance, tilt the head to the requested angle, transfer support to the brake, and deactivate head drive.", "suite": "motion", "difficulty": "medium", "family": "motion_medium_core", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "paper-gpt-5.5--r1--c0", "success": "deepseek-v4.1-flash-ultrafast--HtWw4Du--c0" } }, { "id": "MOT-M06", "title": "Vertical opening·Transfer docking·Support lifting·Internal tool reciprocation", "aim": "Complete one empty mold access cycle and return all axes to their initial positions.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 5, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-opus-5-5--YErHCXF--c0", "success": "gpt-6.1-sol--3ZgzKYx--c0" } }, { "id": "MOT-M07", "title": "Approach·Retract two work points of the front·rear deployment head", "aim": "This is a request to check the operation of an empty device at two access points with one head.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "paper-claude-sonnet-5--r1--c0", "success": "claude-haiku-5-5--xquuYw8--c0" } }, { "id": "MOT-M08", "title": "Align the magazine with the current reference coordinates and pitch, then approach laterally", "aim": "Register a fresh magazine reference, approach the requested shelf, and retract the approach axis.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "paper-gemini-3.8-flash--r1--c0", "success": "claude-haiku-5-5--ELtv3vs--c0" } }, { "id": "MOT-M09", "title": "Reverse release of guide placement, both-side fixation, and internal access", "aim": "Establish and maintain the locked guide posture during internal access, then retract to the origin.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--f7jecsm--c0", "success": "gpt-6-astra--G3CK7sB--c0" } }, { "id": "MOT-M10", "title": "Dual-head transition and reciprocation in avoidance space", "aim": "Move the rectangular head through the L-shaped passage to the requested position and return.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 1.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--23wkZ48--c0", "success": "claude-sonnet-5-5--bYBmcyT--c0" } }, { "id": "MOT-M11", "title": "Connect the approach and retraction history of different tools by index", "aim": "Complete both tool approaches at their requested index angles and depths, then retract both tools.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 5, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-opus-5-5--7yTycwv--c0", "success": "gpt-6-astra--iHu9u2e--c0" } }, { "id": "MOT-M12", "title": "Two reach positions of serial cylinders and jig approach", "aim": "Verify both reach positions and the single jig approach, then return all axes to their origins.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 4, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "claude-opus-5-5--MEWoBHq--c0", "success": "gpt-6-astra--snPGq4m--c0" } }, { "id": "MOT-M13", "title": "Tilt loader, receiver lift alignment and docking", "aim": "This is an empty docking test of one loader and a receiving lift.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 5, "passAt1": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "none", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "paper-gpt-5.5--r1--c0", "success": "gpt-6.1-sol--oFtu3Hi--c0" } }, { "id": "MOT-M14", "title": "Slide Retraction and Table Positioning Task", "aim": "Retract the slide to secure rotation space, maintain the retracted state, switch the table to the requested posture, confirm stop, and then re-approach.", "suite": "motion", "difficulty": "medium", "family": "motion_medium_core", "conditions": 7, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "none", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "deepseek-v4.1-flash-ultrafast--YSPL3u6--c0", "success": "claude-haiku-5-5--An6HwD4--c0" } }, { "id": "MOT-M15", "title": "Directional reference selection, jig transfer, upper tool approach", "aim": "Observe the installed empty jig direction side to perform the upper tool approach test.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "none", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "claude-fable-5-1--GiWdrAT--c0", "success": "deepseek-v4.1-flash-ultrafast--uVbQZbz--c0" } }, { "id": "MOT-M16", "title": "Single module request and completion confirmation maintaining alignment and approach results", "aim": "Complete the current empty-tool round trip while maintaining the table/carriage alignment.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "glm-5.3-flash-ultrafast--wixAg6H--c0", "success": "gpt-6-luna--JDvzgDA--c0" } }, { "id": "MOT-M17", "title": "Single cycle linking mutual conditions of feeder retreat and head return", "aim": "Retract the feeder, complete the head approach at the destination, then restore the required return configuration.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-haiku-5-5--bnKKWyh--c3", "success": "gpt-6-astra--oQttFgu--c0" } }, { "id": "MOT-M18", "title": "Secure search space with actual retraction and acquire new origin", "aim": "Retract and raise the internal tool, capture a new reference, and stop at the requested reference-relative position.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "every", "claude-haiku-5-5": "none", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-haiku-5-5--72vW7zk--c0", "success": "gpt-6-luna--nL8uEEb--c0" } }, { "id": "MOT-M19", "title": "Jig base retreat, lateral alignment, guide retention section transfer", "aim": "This is a single guide following section test.", "suite": "motion", "difficulty": "medium", "family": "motion_hard15", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 1.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 0.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 1.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 1.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": {}, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "every", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "none" }, "replays": { "failure": "claude-sonnet-5-5--5b6hcRB--c0", "success": "gpt-6-astra--ARzDzw3--c0" } }, { "id": "MOT-M20", "title": "Synchronized Operation and Parking of Slave Axis", "aim": "Synchronize the slave axis to the master axis at the requested ratio, then separate and stop at the requested angle after the specified rotation span.", "suite": "motion", "difficulty": "medium", "family": "motion_medium_core", "conditions": 5, "passAt1": { "claude-opus-5-5": 1.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 1.0, "gpt-6.1-sol": 1.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 1.0, "gpt-6-astra": 1.0, "deepseek-v4.1-flash-ultrafast": 1.0, "glm-5.3-flash-ultrafast": 1.0 }, "safetyViolationRate": { "claude-opus-5-5": 0.0, "claude-sonnet-5-5": 0.0, "claude-haiku-5-5": 0.0, "gpt-6.1-sol": 0.0, "gpt-6-luna": 0.0, "claude-fable-5-1": 0.0, "gpt-6-astra": 0.0, "deepseek-v4.1-flash-ultrafast": 0.0, "glm-5.3-flash-ultrafast": 0.0 }, "trials": { "claude-opus-5-5": 1, "claude-sonnet-5-5": 1, "claude-haiku-5-5": 1, "gpt-6.1-sol": 1, "gpt-6-luna": 1, "claude-fable-5-1": 1, "gpt-6-astra": 1, "deepseek-v4.1-flash-ultrafast": 1, "glm-5.3-flash-ultrafast": 1 }, "noProgramTrials": { "gpt-6-luna": 1 }, "result": { "claude-opus-5-5": "every", "claude-sonnet-5-5": "none", "claude-haiku-5-5": "every", "gpt-6.1-sol": "every", "gpt-6-luna": "none", "claude-fable-5-1": "every", "gpt-6-astra": "every", "deepseek-v4.1-flash-ultrafast": "every", "glm-5.3-flash-ultrafast": "every" }, "replays": { "failure": "claude-sonnet-5-5--cEpAPhb--c0", "success": "gpt-6.1-sol--dvfgmGG--c0" } } ], "provenance": { "evaluatorRev": "5.2.6-ef1.2", "evaluatorImage": "ef/plcworld-verifier:5.2.6-ef1.2", "referenceEvaluator": "plcworld-evaluation/5.2.6", "upstream": { "name": "PLCWorld", "arxiv": "2610.02982", "url": "https://arxiv.org/abs/2610.02982", "title": "PLCWorld: Benchmarking LLM-Generated PLC Programs in Closed-Loop Plant Simulation", "authors": "Yunji Kim, Yunseok Lee, Hyunwoo Seo, Jaerim Choi, Woojin Lee" }, "generatedAt": "2026-10-24T20:03:37+00:00", "pendingModels": [], "safetyViolationRate": "violating conditions / evaluated conditions (scored trials)", "passAt1": "mean reward over a model's trials; reward 1 only if every condition succeeds with no violation", "noProgramTrials": "scored trials that ended without /app/program.st (e.g. the program was returned in chat); Easy/Medium (Harbor-graded) score them 0, Hard (offline queue) grades a program recovered from the final message when there is one", "audit": { "tasksReviewed": 100, "tasksClean": 33, "tasksWithVerifiedFlaws": 64, "tasksChanged": 67, "candidates": 130, "verified": 105, "flaws": { "S1": 5, "S2": 24, "S3": 52, "G": 12, "H1": 5, "H2": 1, "H3": 3, "H4": 28 }, "verifiedFlaws": { "S1": 4, "S2": 16, "S3": 36, "G": 12, "H1": 5, "H2": 1, "H3": 3, "H4": 28 }, "fixes": { "prompt": 76, "scenario": 29, "sharedCode": 11, "graderConfig": 3, "deferred": 20 }, "definitions": "flaws = candidate findings per category; tasksClean = tasks with no finding; tasksChanged = tasks with at least one applied (not deferred) fix" }, "scanMs": 10, "conditionsTotal": 476, "conditionsUpstream": 473 }, "paperBaseline": { "source": "PLCWorld paper (arXiv 2610.02982), original package 5.2.6 results", "protocol": "single-turn generation, 3 repeats; not comparable with the agent leaderboard (different evaluator revision and harness)", "rows": [ { "model": "gpt-5.5", "label": "GPT-5.5", "repeats": 3, "easy": 0.8333, "medium": 0.8167, "hard": 0.1778, "overall": 0.63 }, { "model": "gemini-3.1-pro", "label": "Gemini 3.1 Pro", "repeats": 3, "easy": 0.8222, "medium": 0.775, "hard": 0.1667, "overall": 0.6067 }, { "model": "gemini-3.8-flash", "label": "Gemini 3.8 Flash", "repeats": 3, "easy": 0.9556, "medium": 0.7333, "hard": 0.0556, "overall": 0.5967 }, { "model": "claude-sonnet-5", "label": "Claude Sonnet 5", "repeats": 3, "easy": 0.7333, "medium": 0.675, "hard": 0.0222, "overall": 0.4967 }, { "model": "qwen3.5-plus", "label": "Qwen3.5-Plus", "repeats": 3, "easy": 0.6778, "medium": 0.4083, "hard": 0.0, "overall": 0.3667 }, { "model": "gpt-4o", "label": "GPT-4o", "repeats": 3, "easy": 0.1667, "medium": 0.0167, "hard": 0.0, "overall": 0.0567 } ] } }