{ "schemaVersion": 9, "generatedAt": "2026-09-28T16:49:14.280Z", "publicationScope": "completed-results", "activeExperiment": "test-05", "scopeNote": "Five distinct test scopes, including HTML-to-managed-app fidelity. Original, repeat, follow-up and HTML-reference cohorts remain separate. Compare settings against the same scope and demonstrated capabilities, not pooled scores or averages.", "app": "Device Procurement and Availability; Device Lifecycle Hub; Device Allocation and Policy Hub; Annual Leave Manager", "unitNote": "Observed Cowork task credits, not currency or a tenant invoice. Final cumulative readings are not added to initial readings.", "timingNote": "Observed build time per app sums the authoring turns: build, planned updates, clarifications and corrections. It excludes external review and screenshot-taking gaps between prompts, /cost commands and native self-review. Within-turn waits and observation uncertainty remain included; this is not pure model compute time. Missing authoring coverage is not estimated from elapsed test time.", "evidenceNote": "Exact prompts, visible transcripts, approval records, backend snapshots and raw cost evidence are retained privately. Test 5 separately publishes its authorized HTML reference and selected final app-only screenshots. Public data remains an explicit allowlisted projection, not a raw transcript export.", "experiments": [ { "id": "test-03", "route": "test-03", "title": "Test 3: complex multi-prompt virtual-data apps", "design": "Five model/effort configurations once each, four identical ordered authoring prompts in each fresh app: build, connect the existing app to Office 365 Users, extend workflows, then change requirements. Independent external review.", "runs": [ { "id": "run-01", "model": "Sonnet 5", "effort": "Medium", "replicate": 1, "block": null, "status": "reviewed", "comparisonEligible": true, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "CX14" ] }, "firstCost": { "status": "observed", "value": 176, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 1326, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 1150, "unit": "credits" }, "timing": { "build": { "seconds": 2371.299, "lowerBoundSeconds": 2358.672, "observation": "interval-bounded", "authoringPrompts": 5, "observedPrompts": 5 }, "firstBuildResponseSeconds": 533.775, "firstBuildResponseLowerBoundSeconds": 531.625, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 7606.6, "timeToAcceptanceSeconds": 7606.6, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.15, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "stages": [ { "id": "stage-01", "title": "Build the relational virtual-data app first", "kind": "build", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 176, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": null, "increaseSincePrecedingReading": null }, { "id": "stage-02", "title": "Connect the existing managed app to Office 365 Users", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 367, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 251, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 116, "unit": "credits" } }, { "id": "stage-03", "title": "Virtual approvals, asset lifecycle, themes and preferences", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 770, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 367, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 403, "unit": "credits" } }, { "id": "stage-04", "title": "Approval policy change, revisions, cost centres and responsive refinement", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 1326, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 770, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 556, "unit": "credits" } } ], "firstFullTargetAssessmentStage": "stage-04", "plannedChanges": 3, "profileProof": { "stageId": "stage-02", "verifiedInApp": true, "reverifiedAfterFinalReopen": true, "errorRetryCoverage": "blocked" }, "corrections": 1, "clarifications": 0, "approvals": 0, "screenshotFiles": 379, "archived": true, "promptSha256": "3f6463aff82d0b6a5bcfd20f0aae18e9cbe385b90c5f25f2b62c418055232ced", "checks": [ { "id": "CX01", "capability": "Same private app opens with all seven navigable screens", "critical": true, "final": "pass" }, { "id": "CX02", "capability": "Exact six-device/four-request relational seed and derived initial metrics", "critical": true, "final": "pass" }, { "id": "CX03", "capability": "Catalogue search, combined category/stock filters and zero-stock protection", "critical": false, "final": "pass" }, { "id": "CX04", "capability": "Multi-line draft creation, line edits/removal and accurate GBP totals", "critical": true, "final": "pass" }, { "id": "CX05", "capability": "Required fields, dates, whole positive quantities, duplicates and stock validation", "critical": true, "final": "pass" }, { "id": "CX06", "capability": "Draft/persona isolation, edits and exact-once submission with stable IDs", "critical": true, "final": "pass" }, { "id": "CX07", "capability": "Controller-created draft, IDs, lines and timeline survive all three planned changes and reopen", "critical": true, "final": "pass" }, { "id": "CX08", "capability": "Virtual manager/finance role gates and approval transitions at the final threshold", "critical": true, "final": "pass" }, { "id": "CX09", "capability": "Rejection comments, append-only actor/timestamp history and terminal action guards", "critical": false, "final": "pass" }, { "id": "CX10", "capability": "Request changes, requester edit/resubmit and cancellation preserve identity/history", "critical": false, "final": "pass" }, { "id": "CX11", "capability": "Atomic fulfilment, stock revalidation, per-unit assets and duplicate-fulfilment protection", "critical": true, "final": "pass" }, { "id": "CX12", "capability": "Good/damaged asset returns and duplicate-return protection", "critical": false, "final": "pass" }, { "id": "CX13", "capability": "Current-user Office 365 Users profile actually reads through supported app binding", "critical": true, "final": "pass" }, { "id": "CX14", "capability": "Profile loading/error/retry and genuinely missing optional fields are honest", "critical": false, "final": "blocked" }, { "id": "CX15", "capability": "Explicit light/dark mode applies across all screens and persists on reopen", "critical": true, "final": "pass" }, { "id": "CX16", "capability": "Local preference Save/Cancel, density and default department work without directory writes", "critical": false, "final": "pass" }, { "id": "CX17", "capability": "Cost-centre migration, persistence and department mismatch validation", "critical": false, "final": "fail" }, { "id": "CX18", "capability": "Derived status/asset/stock metrics, low-stock navigation and combined request filters", "critical": false, "final": "pass" }, { "id": "CX19", "capability": "All screens and key forms are usable at actual wide desktop and 390px mobile widths", "critical": true, "final": "pass" }, { "id": "CX20", "capability": "Labels, visible keyboard focus, readable contrast and reachable form errors", "critical": false, "final": "pass" } ] }, { "id": "run-02", "model": "Opus 5", "effort": "High", "replicate": 1, "block": null, "status": "reviewed", "comparisonEligible": true, "final": { "points": 95, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "CX14" ] }, "firstCost": { "status": "observed", "value": 561, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 3073, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 2512, "unit": "credits" }, "timing": { "build": { "seconds": 2763.457, "lowerBoundSeconds": 2752.088, "observation": "interval-bounded", "authoringPrompts": 5, "observedPrompts": 5 }, "firstBuildResponseSeconds": 881.455, "firstBuildResponseLowerBoundSeconds": 879.256, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 4851.881, "timeToAcceptanceSeconds": 4851.881, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.199, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "stages": [ { "id": "stage-01", "title": "Build the relational virtual-data app first", "kind": "build", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 561, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": null, "increaseSincePrecedingReading": null }, { "id": "stage-02", "title": "Connect the existing managed app to Office 365 Users", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 916, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 561, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 355, "unit": "credits" } }, { "id": "stage-03", "title": "Virtual approvals, asset lifecycle, themes and preferences", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 1679, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 916, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 763, "unit": "credits" } }, { "id": "stage-04", "title": "Approval policy change, revisions, cost centres and responsive refinement", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 2505, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 1679, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 826, "unit": "credits" } } ], "firstFullTargetAssessmentStage": "stage-04", "plannedChanges": 3, "profileProof": { "stageId": "stage-02", "verifiedInApp": true, "reverifiedAfterFinalReopen": true, "errorRetryCoverage": "blocked" }, "corrections": 1, "clarifications": 0, "approvals": 0, "screenshotFiles": 340, "archived": true, "promptSha256": "3f6463aff82d0b6a5bcfd20f0aae18e9cbe385b90c5f25f2b62c418055232ced", "checks": [ { "id": "CX01", "capability": "Same private app opens with all seven navigable screens", "critical": true, "final": "pass" }, { "id": "CX02", "capability": "Exact six-device/four-request relational seed and derived initial metrics", "critical": true, "final": "pass" }, { "id": "CX03", "capability": "Catalogue search, combined category/stock filters and zero-stock protection", "critical": false, "final": "pass" }, { "id": "CX04", "capability": "Multi-line draft creation, line edits/removal and accurate GBP totals", "critical": true, "final": "pass" }, { "id": "CX05", "capability": "Required fields, dates, whole positive quantities, duplicates and stock validation", "critical": true, "final": "pass" }, { "id": "CX06", "capability": "Draft/persona isolation, edits and exact-once submission with stable IDs", "critical": true, "final": "pass" }, { "id": "CX07", "capability": "Controller-created draft, IDs, lines and timeline survive all three planned changes and reopen", "critical": true, "final": "pass" }, { "id": "CX08", "capability": "Virtual manager/finance role gates and approval transitions at the final threshold", "critical": true, "final": "pass" }, { "id": "CX09", "capability": "Rejection comments, append-only actor/timestamp history and terminal action guards", "critical": false, "final": "pass" }, { "id": "CX10", "capability": "Request changes, requester edit/resubmit and cancellation preserve identity/history", "critical": false, "final": "pass" }, { "id": "CX11", "capability": "Atomic fulfilment, stock revalidation, per-unit assets and duplicate-fulfilment protection", "critical": true, "final": "pass" }, { "id": "CX12", "capability": "Good/damaged asset returns and duplicate-return protection", "critical": false, "final": "pass" }, { "id": "CX13", "capability": "Current-user Office 365 Users profile actually reads through supported app binding", "critical": true, "final": "pass" }, { "id": "CX14", "capability": "Profile loading/error/retry and genuinely missing optional fields are honest", "critical": false, "final": "blocked" }, { "id": "CX15", "capability": "Explicit light/dark mode applies across all screens and persists on reopen", "critical": true, "final": "pass" }, { "id": "CX16", "capability": "Local preference Save/Cancel, density and default department work without directory writes", "critical": false, "final": "pass" }, { "id": "CX17", "capability": "Cost-centre migration, persistence and department mismatch validation", "critical": false, "final": "pass" }, { "id": "CX18", "capability": "Derived status/asset/stock metrics, low-stock navigation and combined request filters", "critical": false, "final": "pass" }, { "id": "CX19", "capability": "All screens and key forms are usable at actual wide desktop and 390px mobile widths", "critical": true, "final": "pass" }, { "id": "CX20", "capability": "Labels, visible keyboard focus, readable contrast and reachable form errors", "critical": false, "final": "pass" } ] }, { "id": "run-03", "model": "Opus 5", "effort": "Medium", "replicate": 1, "block": null, "status": "reviewed", "comparisonEligible": true, "final": { "points": 95, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "CX14" ] }, "firstCost": { "status": "observed", "value": 505, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 2358, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 1853, "unit": "credits" }, "timing": { "build": { "seconds": 2111.675, "lowerBoundSeconds": 2097.083, "observation": "interval-bounded", "authoringPrompts": 6, "observedPrompts": 6 }, "firstBuildResponseSeconds": 644.642, "firstBuildResponseLowerBoundSeconds": 642.371, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 5105.311, "timeToAcceptanceSeconds": 5105.311, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.271, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "stages": [ { "id": "stage-01", "title": "Build the relational virtual-data app first", "kind": "build", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 505, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": null, "increaseSincePrecedingReading": null }, { "id": "stage-02", "title": "Connect the existing managed app to Office 365 Users", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 1124, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 726, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 398, "unit": "credits" } }, { "id": "stage-03", "title": "Virtual approvals, asset lifecycle, themes and preferences", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 1720, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 1124, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 596, "unit": "credits" } }, { "id": "stage-04", "title": "Approval policy change, revisions, cost centres and responsive refinement", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 2011, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 1720, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 291, "unit": "credits" } } ], "firstFullTargetAssessmentStage": "stage-04", "plannedChanges": 3, "profileProof": { "stageId": "stage-02", "verifiedInApp": true, "reverifiedAfterFinalReopen": true, "errorRetryCoverage": "blocked" }, "corrections": 2, "clarifications": 0, "approvals": 0, "screenshotFiles": 441, "archived": true, "promptSha256": "3f6463aff82d0b6a5bcfd20f0aae18e9cbe385b90c5f25f2b62c418055232ced", "checks": [ { "id": "CX01", "capability": "Same private app opens with all seven navigable screens", "critical": true, "final": "pass" }, { "id": "CX02", "capability": "Exact six-device/four-request relational seed and derived initial metrics", "critical": true, "final": "pass" }, { "id": "CX03", "capability": "Catalogue search, combined category/stock filters and zero-stock protection", "critical": false, "final": "pass" }, { "id": "CX04", "capability": "Multi-line draft creation, line edits/removal and accurate GBP totals", "critical": true, "final": "pass" }, { "id": "CX05", "capability": "Required fields, dates, whole positive quantities, duplicates and stock validation", "critical": true, "final": "pass" }, { "id": "CX06", "capability": "Draft/persona isolation, edits and exact-once submission with stable IDs", "critical": true, "final": "pass" }, { "id": "CX07", "capability": "Controller-created draft, IDs, lines and timeline survive all three planned changes and reopen", "critical": true, "final": "pass" }, { "id": "CX08", "capability": "Virtual manager/finance role gates and approval transitions at the final threshold", "critical": true, "final": "pass" }, { "id": "CX09", "capability": "Rejection comments, append-only actor/timestamp history and terminal action guards", "critical": false, "final": "pass" }, { "id": "CX10", "capability": "Request changes, requester edit/resubmit and cancellation preserve identity/history", "critical": false, "final": "pass" }, { "id": "CX11", "capability": "Atomic fulfilment, stock revalidation, per-unit assets and duplicate-fulfilment protection", "critical": true, "final": "pass" }, { "id": "CX12", "capability": "Good/damaged asset returns and duplicate-return protection", "critical": false, "final": "pass" }, { "id": "CX13", "capability": "Current-user Office 365 Users profile actually reads through supported app binding", "critical": true, "final": "pass" }, { "id": "CX14", "capability": "Profile loading/error/retry and genuinely missing optional fields are honest", "critical": false, "final": "blocked" }, { "id": "CX15", "capability": "Explicit light/dark mode applies across all screens and persists on reopen", "critical": true, "final": "pass" }, { "id": "CX16", "capability": "Local preference Save/Cancel, density and default department work without directory writes", "critical": false, "final": "pass" }, { "id": "CX17", "capability": "Cost-centre migration, persistence and department mismatch validation", "critical": false, "final": "pass" }, { "id": "CX18", "capability": "Derived status/asset/stock metrics, low-stock navigation and combined request filters", "critical": false, "final": "pass" }, { "id": "CX19", "capability": "All screens and key forms are usable at actual wide desktop and 390px mobile widths", "critical": true, "final": "pass" }, { "id": "CX20", "capability": "Labels, visible keyboard focus, readable contrast and reachable form errors", "critical": false, "final": "pass" } ] }, { "id": "run-04", "model": "Sonnet 5", "effort": "High", "replicate": 1, "block": null, "status": "reviewed", "comparisonEligible": true, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "CX14", "CX16" ] }, "firstCost": { "status": "observed", "value": 292, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 2289, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 1997, "unit": "credits" }, "timing": { "build": { "seconds": 3089.817, "lowerBoundSeconds": 3081.297, "observation": "interval-bounded", "authoringPrompts": 4, "observedPrompts": 4 }, "firstBuildResponseSeconds": 666.448, "firstBuildResponseLowerBoundSeconds": 664.215, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 5098.76, "timeToAcceptanceSeconds": 5098.76, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.233, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "stages": [ { "id": "stage-01", "title": "Build the relational virtual-data app first", "kind": "build", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 292, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": null, "increaseSincePrecedingReading": null }, { "id": "stage-02", "title": "Connect the existing managed app to Office 365 Users", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 550, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 292, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 258, "unit": "credits" } }, { "id": "stage-03", "title": "Virtual approvals, asset lifecycle, themes and preferences", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 1507, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 550, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 957, "unit": "credits" } }, { "id": "stage-04", "title": "Approval policy change, revisions, cost centres and responsive refinement", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 2289, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 1507, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 782, "unit": "credits" } } ], "firstFullTargetAssessmentStage": "stage-04", "plannedChanges": 3, "profileProof": { "stageId": "stage-02", "verifiedInApp": true, "reverifiedAfterFinalReopen": true, "errorRetryCoverage": "blocked" }, "corrections": 0, "clarifications": 0, "approvals": 0, "screenshotFiles": 356, "archived": true, "promptSha256": "3f6463aff82d0b6a5bcfd20f0aae18e9cbe385b90c5f25f2b62c418055232ced", "checks": [ { "id": "CX01", "capability": "Same private app opens with all seven navigable screens", "critical": true, "final": "pass" }, { "id": "CX02", "capability": "Exact six-device/four-request relational seed and derived initial metrics", "critical": true, "final": "pass" }, { "id": "CX03", "capability": "Catalogue search, combined category/stock filters and zero-stock protection", "critical": false, "final": "pass" }, { "id": "CX04", "capability": "Multi-line draft creation, line edits/removal and accurate GBP totals", "critical": true, "final": "pass" }, { "id": "CX05", "capability": "Required fields, dates, whole positive quantities, duplicates and stock validation", "critical": true, "final": "pass" }, { "id": "CX06", "capability": "Draft/persona isolation, edits and exact-once submission with stable IDs", "critical": true, "final": "pass" }, { "id": "CX07", "capability": "Controller-created draft, IDs, lines and timeline survive all three planned changes and reopen", "critical": true, "final": "pass" }, { "id": "CX08", "capability": "Virtual manager/finance role gates and approval transitions at the final threshold", "critical": true, "final": "pass" }, { "id": "CX09", "capability": "Rejection comments, append-only actor/timestamp history and terminal action guards", "critical": false, "final": "pass" }, { "id": "CX10", "capability": "Request changes, requester edit/resubmit and cancellation preserve identity/history", "critical": false, "final": "pass" }, { "id": "CX11", "capability": "Atomic fulfilment, stock revalidation, per-unit assets and duplicate-fulfilment protection", "critical": true, "final": "pass" }, { "id": "CX12", "capability": "Good/damaged asset returns and duplicate-return protection", "critical": false, "final": "pass" }, { "id": "CX13", "capability": "Current-user Office 365 Users profile actually reads through supported app binding", "critical": true, "final": "pass" }, { "id": "CX14", "capability": "Profile loading/error/retry and genuinely missing optional fields are honest", "critical": false, "final": "blocked" }, { "id": "CX15", "capability": "Explicit light/dark mode applies across all screens and persists on reopen", "critical": true, "final": "pass" }, { "id": "CX16", "capability": "Local preference Save/Cancel, density and default department work without directory writes", "critical": false, "final": "blocked" }, { "id": "CX17", "capability": "Cost-centre migration, persistence and department mismatch validation", "critical": false, "final": "pass" }, { "id": "CX18", "capability": "Derived status/asset/stock metrics, low-stock navigation and combined request filters", "critical": false, "final": "pass" }, { "id": "CX19", "capability": "All screens and key forms are usable at actual wide desktop and 390px mobile widths", "critical": true, "final": "pass" }, { "id": "CX20", "capability": "Labels, visible keyboard focus, readable contrast and reachable form errors", "critical": false, "final": "pass" } ] }, { "id": "run-05", "model": "GPT 5.5", "effort": "Medium", "replicate": 1, "block": null, "status": "reviewed", "comparisonEligible": true, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "CX14", "CX16" ] }, "firstCost": { "status": "observed", "value": 381, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 944, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 563, "unit": "credits" }, "timing": { "build": { "seconds": 1303.39, "lowerBoundSeconds": 1291.207, "observation": "interval-bounded", "authoringPrompts": 5, "observedPrompts": 5 }, "firstBuildResponseSeconds": 161.688, "firstBuildResponseLowerBoundSeconds": 159.638, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 3486.634, "timeToAcceptanceSeconds": 3486.634, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.05, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "stages": [ { "id": "stage-01", "title": "Build the relational virtual-data app first", "kind": "build", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 381, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": null, "increaseSincePrecedingReading": null }, { "id": "stage-02", "title": "Connect the existing managed app to Office 365 Users", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 501, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 381, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 120, "unit": "credits" } }, { "id": "stage-03", "title": "Virtual approvals, asset lifecycle, themes and preferences", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 736, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 501, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 235, "unit": "credits" } }, { "id": "stage-04", "title": "Approval policy change, revisions, cost centres and responsive refinement", "kind": "planned-change", "submitted": true, "responseComplete": true, "cost": { "status": "observed", "value": 944, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 736, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 208, "unit": "credits" } } ], "firstFullTargetAssessmentStage": "stage-04", "plannedChanges": 3, "profileProof": { "stageId": "stage-02", "verifiedInApp": true, "reverifiedAfterFinalReopen": true, "errorRetryCoverage": "blocked" }, "corrections": 0, "clarifications": 1, "approvals": 0, "screenshotFiles": 306, "archived": true, "promptSha256": "3f6463aff82d0b6a5bcfd20f0aae18e9cbe385b90c5f25f2b62c418055232ced", "checks": [ { "id": "CX01", "capability": "Same private app opens with all seven navigable screens", "critical": true, "final": "pass" }, { "id": "CX02", "capability": "Exact six-device/four-request relational seed and derived initial metrics", "critical": true, "final": "pass" }, { "id": "CX03", "capability": "Catalogue search, combined category/stock filters and zero-stock protection", "critical": false, "final": "pass" }, { "id": "CX04", "capability": "Multi-line draft creation, line edits/removal and accurate GBP totals", "critical": true, "final": "pass" }, { "id": "CX05", "capability": "Required fields, dates, whole positive quantities, duplicates and stock validation", "critical": true, "final": "pass" }, { "id": "CX06", "capability": "Draft/persona isolation, edits and exact-once submission with stable IDs", "critical": true, "final": "pass" }, { "id": "CX07", "capability": "Controller-created draft, IDs, lines and timeline survive all three planned changes and reopen", "critical": true, "final": "pass" }, { "id": "CX08", "capability": "Virtual manager/finance role gates and approval transitions at the final threshold", "critical": true, "final": "pass" }, { "id": "CX09", "capability": "Rejection comments, append-only actor/timestamp history and terminal action guards", "critical": false, "final": "pass" }, { "id": "CX10", "capability": "Request changes, requester edit/resubmit and cancellation preserve identity/history", "critical": false, "final": "pass" }, { "id": "CX11", "capability": "Atomic fulfilment, stock revalidation, per-unit assets and duplicate-fulfilment protection", "critical": true, "final": "pass" }, { "id": "CX12", "capability": "Good/damaged asset returns and duplicate-return protection", "critical": false, "final": "pass" }, { "id": "CX13", "capability": "Current-user Office 365 Users profile actually reads through supported app binding", "critical": true, "final": "pass" }, { "id": "CX14", "capability": "Profile loading/error/retry and genuinely missing optional fields are honest", "critical": false, "final": "blocked" }, { "id": "CX15", "capability": "Explicit light/dark mode applies across all screens and persists on reopen", "critical": true, "final": "pass" }, { "id": "CX16", "capability": "Local preference Save/Cancel, density and default department work without directory writes", "critical": false, "final": "blocked" }, { "id": "CX17", "capability": "Cost-centre migration, persistence and department mismatch validation", "critical": false, "final": "pass" }, { "id": "CX18", "capability": "Derived status/asset/stock metrics, low-stock navigation and combined request filters", "critical": false, "final": "pass" }, { "id": "CX19", "capability": "All screens and key forms are usable at actual wide desktop and 390px mobile widths", "critical": true, "final": "pass" }, { "id": "CX20", "capability": "Labels, visible keyboard focus, readable contrast and reachable form errors", "critical": false, "final": "pass" } ] } ], "comparisonEligible": true, "reviewMode": "external-only", "acceptanceCheckCount": 20, "stages": [ { "id": "stage-01", "title": "Build the relational virtual-data app first", "kind": "build" }, { "id": "stage-02", "title": "Connect the existing managed app to Office 365 Users", "kind": "planned-change" }, { "id": "stage-03", "title": "Virtual approvals, asset lifecycle, themes and preferences", "kind": "planned-change" }, { "id": "stage-04", "title": "Approval policy change, revisions, cost centres and responsive refinement", "kind": "planned-change" } ], "stageNote": "Build first; connect Office 365 Users through the existing app next. Planned changes are not repairs. First credits are the initial virtual build; the first full-capability assessment follows stage 4. Each stage card is cumulative. Increases use the last observed task reading before that prompt, including a preceding repair reading where present; they are not isolated connector or token prices.", "protocolSha256": "df5feaf15af92906b23619d11aafbfe6a0415eb9e57c75c819007b48471e61cf" }, { "id": "test-02-sharepoint-clean", "route": "test-02", "title": "Test 2: 10 SharePoint-connected builds", "design": "Five model/effort configurations, each repeated twice in reverse-order blocks; ten fresh tasks and apps; controller-only review.", "runs": [ { "id": "run-01", "model": "Sonnet 5", "effort": "Medium", "replicate": 1, "block": 1, "status": "reviewed", "comparisonEligible": true, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "firstCost": { "status": "observed", "value": 224, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 224, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 0, "unit": "credits" }, "timing": { "build": { "seconds": 600.06, "lowerBoundSeconds": 597.865, "observation": "interval-bounded", "authoringPrompts": 1, "observedPrompts": 1 }, "firstBuildResponseSeconds": 600.06, "firstBuildResponseLowerBoundSeconds": 597.865, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1409.916, "timeToAcceptanceSeconds": 1409.916, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.195, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "corrections": 0, "clarifications": 0, "approvals": 0, "screenshotFiles": 87, "archived": true, "promptSha256": "71281644f6f8465bac8bcde4029fec509e09ea7c18cf2cd6dd6011b6586dad53", "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] }, { "id": "run-02", "model": "Opus 5", "effort": "High", "replicate": 1, "block": 1, "status": "reviewed", "comparisonEligible": true, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "firstCost": { "status": "observed", "value": 968, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 968, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 0, "unit": "credits" }, "timing": { "build": { "seconds": 1165.748, "lowerBoundSeconds": 1163.623, "observation": "interval-bounded", "authoringPrompts": 1, "observedPrompts": 1 }, "firstBuildResponseSeconds": 1165.748, "firstBuildResponseLowerBoundSeconds": 1163.623, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1869.782, "timeToAcceptanceSeconds": 1869.782, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.125, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "corrections": 0, "clarifications": 0, "approvals": 0, "screenshotFiles": 112, "archived": true, "promptSha256": "71281644f6f8465bac8bcde4029fec509e09ea7c18cf2cd6dd6011b6586dad53", "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] }, { "id": "run-03", "model": "Opus 5", "effort": "Medium", "replicate": 1, "block": 1, "status": "reviewed", "comparisonEligible": true, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "firstCost": { "status": "observed", "value": 417, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 417, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 0, "unit": "credits" }, "timing": { "build": { "seconds": 528.115, "lowerBoundSeconds": 526.019, "observation": "interval-bounded", "authoringPrompts": 1, "observedPrompts": 1 }, "firstBuildResponseSeconds": 528.115, "firstBuildResponseLowerBoundSeconds": 526.019, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1027.525, "timeToAcceptanceSeconds": 1027.525, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.096, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "corrections": 0, "clarifications": 0, "approvals": 0, "screenshotFiles": 83, "archived": true, "promptSha256": "71281644f6f8465bac8bcde4029fec509e09ea7c18cf2cd6dd6011b6586dad53", "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] }, { "id": "run-04", "model": "Sonnet 5", "effort": "High", "replicate": 1, "block": 1, "status": "reviewed", "comparisonEligible": true, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "firstCost": { "status": "observed", "value": 620, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 620, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 0, "unit": "credits" }, "timing": { "build": { "seconds": 1018.305, "lowerBoundSeconds": 1016.281, "observation": "interval-bounded", "authoringPrompts": 1, "observedPrompts": 1 }, "firstBuildResponseSeconds": 1018.305, "firstBuildResponseLowerBoundSeconds": 1016.281, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1897.37, "timeToAcceptanceSeconds": 1897.37, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.024, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "corrections": 0, "clarifications": 0, "approvals": 0, "screenshotFiles": 111, "archived": true, "promptSha256": "71281644f6f8465bac8bcde4029fec509e09ea7c18cf2cd6dd6011b6586dad53", "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] }, { "id": "run-05", "model": "GPT 5.5", "effort": "Medium", "replicate": 1, "block": 1, "status": "reviewed", "comparisonEligible": true, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "firstCost": { "status": "observed", "value": 247, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 345, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 98, "unit": "credits" }, "timing": { "build": { "seconds": 631.581, "lowerBoundSeconds": 626.594, "observation": "interval-bounded", "authoringPrompts": 2, "observedPrompts": 2 }, "firstBuildResponseSeconds": 489.542, "firstBuildResponseLowerBoundSeconds": 486.655, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1253.959, "timeToAcceptanceSeconds": 1253.959, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.887, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "corrections": 1, "clarifications": 0, "approvals": 0, "screenshotFiles": 121, "archived": true, "promptSha256": "71281644f6f8465bac8bcde4029fec509e09ea7c18cf2cd6dd6011b6586dad53", "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] }, { "id": "run-06", "model": "GPT 5.5", "effort": "Medium", "replicate": 2, "block": 2, "status": "reviewed", "comparisonEligible": true, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "firstCost": { "status": "observed", "value": 252, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 252, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 0, "unit": "credits" }, "timing": { "build": { "seconds": 471.478, "lowerBoundSeconds": 469.131, "observation": "interval-bounded", "authoringPrompts": 1, "observedPrompts": 1 }, "firstBuildResponseSeconds": 471.478, "firstBuildResponseLowerBoundSeconds": 469.131, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1091.294, "timeToAcceptanceSeconds": 1091.294, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.347, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "corrections": 0, "clarifications": 0, "approvals": 0, "screenshotFiles": 79, "archived": true, "promptSha256": "71281644f6f8465bac8bcde4029fec509e09ea7c18cf2cd6dd6011b6586dad53", "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] }, { "id": "run-07", "model": "Sonnet 5", "effort": "High", "replicate": 2, "block": 2, "status": "reviewed", "comparisonEligible": true, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "firstCost": { "status": "observed", "value": 313, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 313, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 0, "unit": "credits" }, "timing": { "build": { "seconds": 686.485, "lowerBoundSeconds": 684.417, "observation": "interval-bounded", "authoringPrompts": 1, "observedPrompts": 1 }, "firstBuildResponseSeconds": 686.485, "firstBuildResponseLowerBoundSeconds": 684.417, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1067.872, "timeToAcceptanceSeconds": 1067.872, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.068, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "corrections": 0, "clarifications": 0, "approvals": 0, "screenshotFiles": 95, "archived": true, "promptSha256": "71281644f6f8465bac8bcde4029fec509e09ea7c18cf2cd6dd6011b6586dad53", "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] }, { "id": "run-08", "model": "Opus 5", "effort": "Medium", "replicate": 2, "block": 2, "status": "reviewed", "comparisonEligible": true, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "firstCost": { "status": "observed", "value": 425, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 425, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 0, "unit": "credits" }, "timing": { "build": { "seconds": 624.126, "lowerBoundSeconds": 621.597, "observation": "interval-bounded", "authoringPrompts": 1, "observedPrompts": 1 }, "firstBuildResponseSeconds": 624.126, "firstBuildResponseLowerBoundSeconds": 621.597, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 910.55, "timeToAcceptanceSeconds": 910.55, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.529, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "corrections": 0, "clarifications": 0, "approvals": 0, "screenshotFiles": 88, "archived": true, "promptSha256": "71281644f6f8465bac8bcde4029fec509e09ea7c18cf2cd6dd6011b6586dad53", "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] }, { "id": "run-09", "model": "Opus 5", "effort": "High", "replicate": 2, "block": 2, "status": "reviewed", "comparisonEligible": true, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "firstCost": { "status": "observed", "value": 478, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 786, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 308, "unit": "credits" }, "timing": { "build": { "seconds": 928.287, "lowerBoundSeconds": 923.679, "observation": "interval-bounded", "authoringPrompts": 2, "observedPrompts": 2 }, "firstBuildResponseSeconds": 687.571, "firstBuildResponseLowerBoundSeconds": 685.371, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1929.991, "timeToAcceptanceSeconds": 1929.991, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.2, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "corrections": 1, "clarifications": 0, "approvals": 0, "screenshotFiles": 142, "archived": true, "promptSha256": "71281644f6f8465bac8bcde4029fec509e09ea7c18cf2cd6dd6011b6586dad53", "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] }, { "id": "run-10", "model": "Sonnet 5", "effort": "Medium", "replicate": 2, "block": 2, "status": "reviewed", "comparisonEligible": true, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "firstCost": { "status": "observed", "value": 230, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 230, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 0, "unit": "credits" }, "timing": { "build": { "seconds": 526.239, "lowerBoundSeconds": 524.184, "observation": "interval-bounded", "authoringPrompts": 1, "observedPrompts": 1 }, "firstBuildResponseSeconds": 526.239, "firstBuildResponseLowerBoundSeconds": 524.184, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 857.488, "timeToAcceptanceSeconds": 857.488, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.055, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 0, "corrections": 0, "clarifications": 0, "approvals": 0, "screenshotFiles": 88, "archived": true, "promptSha256": "71281644f6f8465bac8bcde4029fec509e09ea7c18cf2cd6dd6011b6586dad53", "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] } ], "comparisonEligible": true, "reviewMode": "external-only", "acceptanceCheckCount": 10, "protocolSha256": "838a5c94f5e306fd30860a7e1b3fa4482a07fc31cecbc73a43269e4723e98acd" }, { "id": "test-01", "route": "test-01", "title": "Test 1: virtual-data apps", "design": "Five virtual-data builds: two models at two reasoning levels, with a baseline repeat. Costs include build and native review.", "runs": [ { "id": "run-01", "model": "Sonnet 5", "effort": "Medium", "replicate": 1, "block": null, "status": "reviewed", "comparisonEligible": false, "final": { "points": 95, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [] }, "firstCost": { "status": "observed", "value": 276, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 317, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 41, "unit": "credits" }, "timing": { "build": { "seconds": null, "lowerBoundSeconds": null, "observation": "not-captured", "authoringPrompts": 1, "observedPrompts": 0 }, "firstBuildResponseSeconds": null, "firstBuildResponseLowerBoundSeconds": null, "firstCompletionObservation": "not-captured", "finalReviewedResultSeconds": null, "timeToAcceptanceSeconds": null, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": null, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 1, "corrections": 0, "clarifications": 0, "approvals": 1, "screenshotFiles": 126, "archived": true, "promptSha256": "d0466119660c3b0fd7d25117d94fdd3c7db60302635650e7c00ff51dac9187e9", "checks": [ { "id": "AC01", "capability": "Exact catalogue", "critical": false, "final": "pass" }, { "id": "AC02", "capability": "Search and empty state", "critical": false, "final": "pass" }, { "id": "AC03", "capability": "Combined filtering", "critical": false, "final": "pass" }, { "id": "AC04", "capability": "Unavailable stock", "critical": true, "final": "pass" }, { "id": "AC05", "capability": "Dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "AC06", "capability": "Valid request", "critical": true, "final": "pass" }, { "id": "AC07", "capability": "Required fields", "critical": true, "final": "pass" }, { "id": "AC08", "capability": "Quantity validation", "critical": true, "final": "pass" }, { "id": "AC09", "capability": "Date validation", "critical": false, "final": "pass" }, { "id": "AC10", "capability": "Approval", "critical": true, "final": "pass" }, { "id": "AC11", "capability": "Rejection", "critical": true, "final": "pass" }, { "id": "AC12", "capability": "Oversell prevention", "critical": true, "final": "pass" }, { "id": "AC13", "capability": "Single state transition", "critical": true, "final": "pass" }, { "id": "AC14", "capability": "Managed persistence", "critical": false, "final": "not-required" }, { "id": "AC15", "capability": "Requester scoping", "critical": false, "final": "pass" }, { "id": "AC16", "capability": "Safe repeatable reset", "critical": false, "final": "pass" }, { "id": "AC17", "capability": "Keyboard and labels", "critical": false, "final": "pass" }, { "id": "AC18", "capability": "Responsive layout", "critical": false, "final": "pass" }, { "id": "AC19", "capability": "Honest demo boundary", "critical": false, "final": "pass" }, { "id": "AC20", "capability": "Actual private managed app", "critical": true, "final": "pass" } ] }, { "id": "run-02", "model": "Opus 5", "effort": "High", "replicate": 1, "block": null, "status": "reviewed", "comparisonEligible": false, "final": { "points": 95, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [] }, "firstCost": { "status": "observed", "value": 621, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 892, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 271, "unit": "credits" }, "timing": { "build": { "seconds": null, "lowerBoundSeconds": null, "observation": "not-captured", "authoringPrompts": 2, "observedPrompts": 0 }, "firstBuildResponseSeconds": null, "firstBuildResponseLowerBoundSeconds": null, "firstCompletionObservation": "not-captured", "finalReviewedResultSeconds": null, "timeToAcceptanceSeconds": null, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": null, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 1, "corrections": 1, "clarifications": 0, "approvals": 0, "screenshotFiles": 151, "archived": true, "promptSha256": "d0466119660c3b0fd7d25117d94fdd3c7db60302635650e7c00ff51dac9187e9", "checks": [ { "id": "AC01", "capability": "Exact catalogue", "critical": false, "final": "pass" }, { "id": "AC02", "capability": "Search and empty state", "critical": false, "final": "pass" }, { "id": "AC03", "capability": "Combined filtering", "critical": false, "final": "pass" }, { "id": "AC04", "capability": "Unavailable stock", "critical": true, "final": "pass" }, { "id": "AC05", "capability": "Dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "AC06", "capability": "Valid request", "critical": true, "final": "pass" }, { "id": "AC07", "capability": "Required fields", "critical": true, "final": "pass" }, { "id": "AC08", "capability": "Quantity validation", "critical": true, "final": "pass" }, { "id": "AC09", "capability": "Date validation", "critical": false, "final": "pass" }, { "id": "AC10", "capability": "Approval", "critical": true, "final": "pass" }, { "id": "AC11", "capability": "Rejection", "critical": true, "final": "pass" }, { "id": "AC12", "capability": "Oversell prevention", "critical": true, "final": "pass" }, { "id": "AC13", "capability": "Single state transition", "critical": true, "final": "pass" }, { "id": "AC14", "capability": "Managed persistence", "critical": false, "final": "not-required" }, { "id": "AC15", "capability": "Requester scoping", "critical": false, "final": "pass" }, { "id": "AC16", "capability": "Safe repeatable reset", "critical": false, "final": "pass" }, { "id": "AC17", "capability": "Keyboard and labels", "critical": false, "final": "pass" }, { "id": "AC18", "capability": "Responsive layout", "critical": false, "final": "pass" }, { "id": "AC19", "capability": "Honest demo boundary", "critical": false, "final": "pass" }, { "id": "AC20", "capability": "Actual private managed app", "critical": true, "final": "pass" } ] }, { "id": "run-03", "model": "Opus 5", "effort": "Medium", "replicate": 1, "block": null, "status": "reviewed", "comparisonEligible": false, "final": { "points": 95, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [] }, "firstCost": { "status": "observed", "value": 342, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 1083, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 741, "unit": "credits" }, "timing": { "build": { "seconds": null, "lowerBoundSeconds": null, "observation": "not-captured", "authoringPrompts": 3, "observedPrompts": 0 }, "firstBuildResponseSeconds": null, "firstBuildResponseLowerBoundSeconds": null, "firstCompletionObservation": "not-captured", "finalReviewedResultSeconds": null, "timeToAcceptanceSeconds": null, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": null, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 1, "corrections": 2, "clarifications": 0, "approvals": 1, "screenshotFiles": 98, "archived": true, "promptSha256": "d0466119660c3b0fd7d25117d94fdd3c7db60302635650e7c00ff51dac9187e9", "checks": [ { "id": "AC01", "capability": "Exact catalogue", "critical": false, "final": "pass" }, { "id": "AC02", "capability": "Search and empty state", "critical": false, "final": "pass" }, { "id": "AC03", "capability": "Combined filtering", "critical": false, "final": "pass" }, { "id": "AC04", "capability": "Unavailable stock", "critical": true, "final": "pass" }, { "id": "AC05", "capability": "Dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "AC06", "capability": "Valid request", "critical": true, "final": "pass" }, { "id": "AC07", "capability": "Required fields", "critical": true, "final": "pass" }, { "id": "AC08", "capability": "Quantity validation", "critical": true, "final": "pass" }, { "id": "AC09", "capability": "Date validation", "critical": false, "final": "pass" }, { "id": "AC10", "capability": "Approval", "critical": true, "final": "pass" }, { "id": "AC11", "capability": "Rejection", "critical": true, "final": "pass" }, { "id": "AC12", "capability": "Oversell prevention", "critical": true, "final": "pass" }, { "id": "AC13", "capability": "Single state transition", "critical": true, "final": "pass" }, { "id": "AC14", "capability": "Managed persistence", "critical": false, "final": "not-required" }, { "id": "AC15", "capability": "Requester scoping", "critical": false, "final": "pass" }, { "id": "AC16", "capability": "Safe repeatable reset", "critical": false, "final": "pass" }, { "id": "AC17", "capability": "Keyboard and labels", "critical": false, "final": "pass" }, { "id": "AC18", "capability": "Responsive layout", "critical": false, "final": "pass" }, { "id": "AC19", "capability": "Honest demo boundary", "critical": false, "final": "pass" }, { "id": "AC20", "capability": "Actual private managed app", "critical": true, "final": "pass" } ] }, { "id": "run-04", "model": "Sonnet 5", "effort": "High", "replicate": 1, "block": null, "status": "reviewed", "comparisonEligible": false, "final": { "points": 95, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [] }, "firstCost": { "status": "observed", "value": 351, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 530, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 179, "unit": "credits" }, "timing": { "build": { "seconds": null, "lowerBoundSeconds": null, "observation": "not-captured", "authoringPrompts": 2, "observedPrompts": 0 }, "firstBuildResponseSeconds": null, "firstBuildResponseLowerBoundSeconds": null, "firstCompletionObservation": "not-captured", "finalReviewedResultSeconds": null, "timeToAcceptanceSeconds": null, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": null, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 1, "corrections": 1, "clarifications": 0, "approvals": 0, "screenshotFiles": 132, "archived": true, "promptSha256": "d0466119660c3b0fd7d25117d94fdd3c7db60302635650e7c00ff51dac9187e9", "checks": [ { "id": "AC01", "capability": "Exact catalogue", "critical": false, "final": "pass" }, { "id": "AC02", "capability": "Search and empty state", "critical": false, "final": "pass" }, { "id": "AC03", "capability": "Combined filtering", "critical": false, "final": "pass" }, { "id": "AC04", "capability": "Unavailable stock", "critical": true, "final": "pass" }, { "id": "AC05", "capability": "Dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "AC06", "capability": "Valid request", "critical": true, "final": "pass" }, { "id": "AC07", "capability": "Required fields", "critical": true, "final": "pass" }, { "id": "AC08", "capability": "Quantity validation", "critical": true, "final": "pass" }, { "id": "AC09", "capability": "Date validation", "critical": false, "final": "pass" }, { "id": "AC10", "capability": "Approval", "critical": true, "final": "pass" }, { "id": "AC11", "capability": "Rejection", "critical": true, "final": "pass" }, { "id": "AC12", "capability": "Oversell prevention", "critical": true, "final": "pass" }, { "id": "AC13", "capability": "Single state transition", "critical": true, "final": "pass" }, { "id": "AC14", "capability": "Managed persistence", "critical": false, "final": "not-required" }, { "id": "AC15", "capability": "Requester scoping", "critical": false, "final": "pass" }, { "id": "AC16", "capability": "Safe repeatable reset", "critical": false, "final": "pass" }, { "id": "AC17", "capability": "Keyboard and labels", "critical": false, "final": "pass" }, { "id": "AC18", "capability": "Responsive layout", "critical": false, "final": "pass" }, { "id": "AC19", "capability": "Honest demo boundary", "critical": false, "final": "pass" }, { "id": "AC20", "capability": "Actual private managed app", "critical": true, "final": "pass" } ] }, { "id": "run-05", "model": "Sonnet 5", "effort": "Medium", "replicate": 2, "block": null, "status": "reviewed", "comparisonEligible": false, "final": { "points": 95, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [] }, "firstCost": { "status": "observed", "value": 174, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 397, "unit": "credits", "scope": "Cowork task" }, "increment": { "value": 223, "unit": "credits" }, "timing": { "build": { "seconds": null, "lowerBoundSeconds": null, "observation": "not-captured", "authoringPrompts": 2, "observedPrompts": 0 }, "firstBuildResponseSeconds": null, "firstBuildResponseLowerBoundSeconds": null, "firstCompletionObservation": "not-captured", "finalReviewedResultSeconds": null, "timeToAcceptanceSeconds": null, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": null, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "reviews": 1, "corrections": 1, "clarifications": 0, "approvals": 0, "screenshotFiles": 147, "archived": true, "promptSha256": "d0466119660c3b0fd7d25117d94fdd3c7db60302635650e7c00ff51dac9187e9", "checks": [ { "id": "AC01", "capability": "Exact catalogue", "critical": false, "final": "pass" }, { "id": "AC02", "capability": "Search and empty state", "critical": false, "final": "pass" }, { "id": "AC03", "capability": "Combined filtering", "critical": false, "final": "pass" }, { "id": "AC04", "capability": "Unavailable stock", "critical": true, "final": "pass" }, { "id": "AC05", "capability": "Dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "AC06", "capability": "Valid request", "critical": true, "final": "pass" }, { "id": "AC07", "capability": "Required fields", "critical": true, "final": "pass" }, { "id": "AC08", "capability": "Quantity validation", "critical": true, "final": "pass" }, { "id": "AC09", "capability": "Date validation", "critical": false, "final": "pass" }, { "id": "AC10", "capability": "Approval", "critical": true, "final": "pass" }, { "id": "AC11", "capability": "Rejection", "critical": true, "final": "pass" }, { "id": "AC12", "capability": "Oversell prevention", "critical": true, "final": "pass" }, { "id": "AC13", "capability": "Single state transition", "critical": true, "final": "pass" }, { "id": "AC14", "capability": "Managed persistence", "critical": false, "final": "not-required" }, { "id": "AC15", "capability": "Requester scoping", "critical": false, "final": "pass" }, { "id": "AC16", "capability": "Safe repeatable reset", "critical": false, "final": "pass" }, { "id": "AC17", "capability": "Keyboard and labels", "critical": false, "final": "pass" }, { "id": "AC18", "capability": "Responsive layout", "critical": false, "final": "pass" }, { "id": "AC19", "capability": "Honest demo boundary", "critical": false, "final": "pass" }, { "id": "AC20", "capability": "Actual private managed app", "critical": true, "final": "pass" } ] } ], "comparisonEligible": false, "reviewMode": "build-and-native-review", "acceptanceCheckCount": 20, "acceptanceScope": { "virtualDataAllowed": true, "notRequired": [ "AC14" ], "note": "Virtual/local data is allowed. Managed-store persistence is not required; recorded scores are unchanged." }, "protocolSha256": "b4a373aa7348f0a09964a1313dbb779dc033e74ec33fb62f070ba6ac1100e460" } ], "validation": { "title": "Additional external-review measurements", "note": "Original cohorts are preserved, not pooled with these observations. The four common settings provide a like-for-like model mix, but virtual approvals and SharePoint request tracking are still different app scopes.", "promptNote": "The virtual sample spans two prompt versions: the same business specification, with private owner-only publication explicitly permitted in version 2. All five additional SharePoint builds use one identical amended prompt. Publication permission and review method differ from the original cohorts; the differences do not isolate connector pricing.", "cohorts": [ { "test": "test-01", "reviewMode": "external-only", "runs": [ { "id": "run-01", "model": "Sonnet 5", "effort": "Medium", "firstCost": { "status": "observed", "value": 215, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 215, "unit": "credits", "scope": "Cowork task" }, "final": { "points": 100, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [] }, "reviews": 0, "corrections": 0, "clarifications": 0, "timing": { "build": { "seconds": 591.786, "lowerBoundSeconds": 583.64, "observation": "interval-bounded", "authoringPrompts": 1, "observedPrompts": 1 }, "firstBuildResponseSeconds": 591.786, "firstBuildResponseLowerBoundSeconds": 583.64, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1269.205, "timeToAcceptanceSeconds": 1269.205, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 8.146, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "promptSha256": "371fc49875e5c19170df827535fd11dc075b8c6080660f5b4d4cac898c21e506", "promptVersion": "virtual-v1", "screenshotFiles": 94, "archived": true, "checks": [ { "id": "AC01", "capability": "Exact catalogue", "critical": false, "final": "pass" }, { "id": "AC02", "capability": "Search and empty state", "critical": false, "final": "pass" }, { "id": "AC03", "capability": "Combined filtering", "critical": false, "final": "pass" }, { "id": "AC04", "capability": "Unavailable stock", "critical": true, "final": "pass" }, { "id": "AC05", "capability": "Dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "AC06", "capability": "Valid request", "critical": true, "final": "pass" }, { "id": "AC07", "capability": "Required fields", "critical": true, "final": "pass" }, { "id": "AC08", "capability": "Quantity validation", "critical": true, "final": "pass" }, { "id": "AC09", "capability": "Date validation", "critical": false, "final": "pass" }, { "id": "AC10", "capability": "Approval", "critical": true, "final": "pass" }, { "id": "AC11", "capability": "Rejection", "critical": true, "final": "pass" }, { "id": "AC12", "capability": "Oversell prevention", "critical": true, "final": "pass" }, { "id": "AC13", "capability": "Single state transition", "critical": true, "final": "pass" }, { "id": "AC14", "capability": "Virtual/local reopen persistence", "critical": true, "final": "pass" }, { "id": "AC15", "capability": "Requester scoping", "critical": false, "final": "pass" }, { "id": "AC16", "capability": "Safe repeatable reset", "critical": false, "final": "pass" }, { "id": "AC17", "capability": "Keyboard and labels", "critical": false, "final": "pass" }, { "id": "AC18", "capability": "Responsive layout", "critical": false, "final": "pass" }, { "id": "AC19", "capability": "Honest demo boundary", "critical": false, "final": "pass" }, { "id": "AC20", "capability": "Actual private managed app", "critical": true, "final": "pass" } ] }, { "id": "run-02", "model": "Opus 5", "effort": "High", "firstCost": { "status": "observed", "value": 611, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 826, "unit": "credits", "scope": "Cowork task" }, "final": { "points": 100, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [] }, "reviews": 0, "corrections": 1, "clarifications": 0, "timing": { "build": { "seconds": 901.309, "lowerBoundSeconds": 896.781, "observation": "interval-bounded", "authoringPrompts": 2, "observedPrompts": 2 }, "firstBuildResponseSeconds": 731.296, "firstBuildResponseLowerBoundSeconds": 728.986, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1684.242, "timeToAcceptanceSeconds": null, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.31, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "promptSha256": "371fc49875e5c19170df827535fd11dc075b8c6080660f5b4d4cac898c21e506", "promptVersion": "virtual-v1", "screenshotFiles": 137, "archived": true, "checks": [ { "id": "AC01", "capability": "Exact catalogue", "critical": false, "final": "pass" }, { "id": "AC02", "capability": "Search and empty state", "critical": false, "final": "pass" }, { "id": "AC03", "capability": "Combined filtering", "critical": false, "final": "pass" }, { "id": "AC04", "capability": "Unavailable stock", "critical": true, "final": "pass" }, { "id": "AC05", "capability": "Dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "AC06", "capability": "Valid request", "critical": true, "final": "pass" }, { "id": "AC07", "capability": "Required fields", "critical": true, "final": "pass" }, { "id": "AC08", "capability": "Quantity validation", "critical": true, "final": "pass" }, { "id": "AC09", "capability": "Date validation", "critical": false, "final": "pass" }, { "id": "AC10", "capability": "Approval", "critical": true, "final": "pass" }, { "id": "AC11", "capability": "Rejection", "critical": true, "final": "pass" }, { "id": "AC12", "capability": "Oversell prevention", "critical": true, "final": "pass" }, { "id": "AC13", "capability": "Single state transition", "critical": true, "final": "pass" }, { "id": "AC14", "capability": "Virtual/local reopen persistence", "critical": true, "final": "pass" }, { "id": "AC15", "capability": "Requester scoping", "critical": false, "final": "pass" }, { "id": "AC16", "capability": "Safe repeatable reset", "critical": false, "final": "pass" }, { "id": "AC17", "capability": "Keyboard and labels", "critical": false, "final": "pass" }, { "id": "AC18", "capability": "Responsive layout", "critical": false, "final": "pass" }, { "id": "AC19", "capability": "Honest demo boundary", "critical": false, "final": "pass" }, { "id": "AC20", "capability": "Actual private managed app", "critical": true, "final": "pass" } ] }, { "id": "run-04", "model": "Sonnet 5", "effort": "High", "firstCost": { "status": "observed", "value": 220, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 220, "unit": "credits", "scope": "Cowork task" }, "final": { "points": 100, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [] }, "reviews": 0, "corrections": 0, "clarifications": 0, "timing": { "build": { "seconds": 673.43, "lowerBoundSeconds": 671.255, "observation": "interval-bounded", "authoringPrompts": 1, "observedPrompts": 1 }, "firstBuildResponseSeconds": 673.43, "firstBuildResponseLowerBoundSeconds": 671.255, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 841.612, "timeToAcceptanceSeconds": 841.612, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.175, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "promptSha256": "15e2426b3b9e06c35706e48a928e70a454859b3fcf6e4d2dec8d6255b01a9e4a", "promptVersion": "virtual-v2", "screenshotFiles": 110, "archived": true, "checks": [ { "id": "AC01", "capability": "Exact catalogue", "critical": false, "final": "pass" }, { "id": "AC02", "capability": "Search and empty state", "critical": false, "final": "pass" }, { "id": "AC03", "capability": "Combined filtering", "critical": false, "final": "pass" }, { "id": "AC04", "capability": "Unavailable stock", "critical": true, "final": "pass" }, { "id": "AC05", "capability": "Dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "AC06", "capability": "Valid request", "critical": true, "final": "pass" }, { "id": "AC07", "capability": "Required fields", "critical": true, "final": "pass" }, { "id": "AC08", "capability": "Quantity validation", "critical": true, "final": "pass" }, { "id": "AC09", "capability": "Date validation", "critical": false, "final": "pass" }, { "id": "AC10", "capability": "Approval", "critical": true, "final": "pass" }, { "id": "AC11", "capability": "Rejection", "critical": true, "final": "pass" }, { "id": "AC12", "capability": "Oversell prevention", "critical": true, "final": "pass" }, { "id": "AC13", "capability": "Single state transition", "critical": true, "final": "pass" }, { "id": "AC14", "capability": "Virtual/local reopen persistence", "critical": true, "final": "pass" }, { "id": "AC15", "capability": "Requester scoping", "critical": false, "final": "pass" }, { "id": "AC16", "capability": "Safe repeatable reset", "critical": false, "final": "pass" }, { "id": "AC17", "capability": "Keyboard and labels", "critical": false, "final": "pass" }, { "id": "AC18", "capability": "Responsive layout", "critical": false, "final": "pass" }, { "id": "AC19", "capability": "Honest demo boundary", "critical": false, "final": "pass" }, { "id": "AC20", "capability": "Actual private managed app", "critical": true, "final": "pass" } ] }, { "id": "run-05", "model": "GPT 5.5", "effort": "Medium", "firstCost": { "status": "observed", "value": 199, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 269, "unit": "credits", "scope": "Cowork task" }, "final": { "points": 100, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [] }, "reviews": 0, "corrections": 1, "clarifications": 0, "timing": { "build": { "seconds": 545.105, "lowerBoundSeconds": 539.408, "observation": "interval-bounded", "authoringPrompts": 2, "observedPrompts": 2 }, "firstBuildResponseSeconds": 366.222, "firstBuildResponseLowerBoundSeconds": 362.637, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1154.629, "timeToAcceptanceSeconds": 1154.629, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 3.585, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "promptSha256": "15e2426b3b9e06c35706e48a928e70a454859b3fcf6e4d2dec8d6255b01a9e4a", "promptVersion": "virtual-v2", "screenshotFiles": 120, "archived": true, "checks": [ { "id": "AC01", "capability": "Exact catalogue", "critical": false, "final": "pass" }, { "id": "AC02", "capability": "Search and empty state", "critical": false, "final": "pass" }, { "id": "AC03", "capability": "Combined filtering", "critical": false, "final": "pass" }, { "id": "AC04", "capability": "Unavailable stock", "critical": true, "final": "pass" }, { "id": "AC05", "capability": "Dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "AC06", "capability": "Valid request", "critical": true, "final": "pass" }, { "id": "AC07", "capability": "Required fields", "critical": true, "final": "pass" }, { "id": "AC08", "capability": "Quantity validation", "critical": true, "final": "pass" }, { "id": "AC09", "capability": "Date validation", "critical": false, "final": "pass" }, { "id": "AC10", "capability": "Approval", "critical": true, "final": "pass" }, { "id": "AC11", "capability": "Rejection", "critical": true, "final": "pass" }, { "id": "AC12", "capability": "Oversell prevention", "critical": true, "final": "pass" }, { "id": "AC13", "capability": "Single state transition", "critical": true, "final": "pass" }, { "id": "AC14", "capability": "Virtual/local reopen persistence", "critical": true, "final": "pass" }, { "id": "AC15", "capability": "Requester scoping", "critical": false, "final": "pass" }, { "id": "AC16", "capability": "Safe repeatable reset", "critical": false, "final": "pass" }, { "id": "AC17", "capability": "Keyboard and labels", "critical": false, "final": "pass" }, { "id": "AC18", "capability": "Responsive layout", "critical": false, "final": "pass" }, { "id": "AC19", "capability": "Honest demo boundary", "critical": false, "final": "pass" }, { "id": "AC20", "capability": "Actual private managed app", "critical": true, "final": "pass" } ] } ], "meanFinalCredits": 382.5, "note": "Four reviewed virtual-data observations, one per shown setting. Browser-local persistence is explicitly allowed and independently verified; no native self-review." }, { "test": "test-02", "reviewMode": "external-only", "runs": [ { "id": "run-01", "model": "Sonnet 5", "effort": "Medium", "firstCost": { "status": "observed", "value": 301, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 301, "unit": "credits", "scope": "Cowork task" }, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "reviews": 0, "corrections": 0, "clarifications": 0, "timing": { "build": { "seconds": 715.385, "lowerBoundSeconds": 712.917, "observation": "interval-bounded", "authoringPrompts": 1, "observedPrompts": 1 }, "firstBuildResponseSeconds": 715.385, "firstBuildResponseLowerBoundSeconds": 712.917, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1084.126, "timeToAcceptanceSeconds": 1084.126, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.468, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "promptSha256": "77f27f229570d44ae0c3114b5ca3efc644cd03718bf2b03a07711468592f31a8", "promptVersion": "sharepoint-v2", "screenshotFiles": 92, "archived": true, "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] }, { "id": "run-02", "model": "Opus 5", "effort": "High", "firstCost": { "status": "observed", "value": 868, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 1130, "unit": "credits", "scope": "Cowork task" }, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "reviews": 0, "corrections": 1, "clarifications": 0, "timing": { "build": { "seconds": 912.876, "lowerBoundSeconds": 908.157, "observation": "interval-bounded", "authoringPrompts": 2, "observedPrompts": 2 }, "firstBuildResponseSeconds": 759.285, "firstBuildResponseLowerBoundSeconds": 757.064, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1179.952, "timeToAcceptanceSeconds": 1179.952, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.221, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "promptSha256": "77f27f229570d44ae0c3114b5ca3efc644cd03718bf2b03a07711468592f31a8", "promptVersion": "sharepoint-v2", "screenshotFiles": 124, "archived": true, "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] }, { "id": "run-03", "model": "Opus 5", "effort": "Medium", "firstCost": { "status": "observed", "value": 544, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 842, "unit": "credits", "scope": "Cowork task" }, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "reviews": 0, "corrections": 1, "clarifications": 0, "timing": { "build": { "seconds": 875.964, "lowerBoundSeconds": 871.543, "observation": "interval-bounded", "authoringPrompts": 2, "observedPrompts": 2 }, "firstBuildResponseSeconds": 598.961, "firstBuildResponseLowerBoundSeconds": 596.819, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1208.886, "timeToAcceptanceSeconds": 1208.886, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.142, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "promptSha256": "77f27f229570d44ae0c3114b5ca3efc644cd03718bf2b03a07711468592f31a8", "promptVersion": "sharepoint-v2", "screenshotFiles": 123, "archived": true, "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] }, { "id": "run-04", "model": "Sonnet 5", "effort": "High", "firstCost": { "status": "observed", "value": 586, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 1025, "unit": "credits", "scope": "Cowork task" }, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "reviews": 0, "corrections": 2, "clarifications": 0, "timing": { "build": { "seconds": 1656.779, "lowerBoundSeconds": 1645.937, "observation": "interval-bounded", "authoringPrompts": 3, "observedPrompts": 3 }, "firstBuildResponseSeconds": 941.877, "firstBuildResponseLowerBoundSeconds": 939.669, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 2678.46, "timeToAcceptanceSeconds": 2678.46, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.208, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "promptSha256": "77f27f229570d44ae0c3114b5ca3efc644cd03718bf2b03a07711468592f31a8", "promptVersion": "sharepoint-v2", "screenshotFiles": 176, "archived": true, "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] }, { "id": "run-05", "model": "GPT 5.5", "effort": "Medium", "firstCost": { "status": "observed", "value": 314, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 314, "unit": "credits", "scope": "Cowork task" }, "final": { "points": 90, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "BC09" ] }, "reviews": 0, "corrections": 0, "clarifications": 0, "timing": { "build": { "seconds": 468.877, "lowerBoundSeconds": 466.578, "observation": "interval-bounded", "authoringPrompts": 1, "observedPrompts": 1 }, "firstBuildResponseSeconds": 468.877, "firstBuildResponseLowerBoundSeconds": 466.578, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 1335.266, "timeToAcceptanceSeconds": 1335.266, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.299, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "promptSha256": "77f27f229570d44ae0c3114b5ca3efc644cd03718bf2b03a07711468592f31a8", "promptVersion": "sharepoint-v2", "screenshotFiles": 109, "archived": true, "checks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "blocked" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ] } ], "meanFinalCredits": 722.4, "note": "Five additional SharePoint observations, one per setting, using the same four-view business scope, owned fixture and external-only review." } ], "matchedConfigurations": [ "Sonnet 5 / Medium", "Opus 5 / High", "Sonnet 5 / High", "GPT 5.5 / Medium" ], "matchedMeans": { "test-01": 382.5, "test-02": 692.5 } }, "complexValidation": { "test": "test-03", "title": "Additional complex-app measurements", "reviewMode": "external-only", "promptPath": "benchmark/test-03-repeat/stage-01-build.txt", "runs": [ { "id": "run-06", "model": "GPT 5.5", "effort": "Medium", "replicate": 2, "firstCost": { "status": "observed", "value": 183, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 573, "unit": "credits", "scope": "Cowork task" }, "final": { "points": 85, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "CX14", "CX16" ] }, "corrections": 1, "clarifications": 0, "reviews": 0, "plannedChanges": 3, "timing": { "build": { "seconds": 1268.978, "lowerBoundSeconds": 1253.919, "observation": "interval-bounded", "authoringPrompts": 5, "observedPrompts": 5 }, "firstBuildResponseSeconds": 339.929, "firstBuildResponseLowerBoundSeconds": 337.225, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 5143.882, "timeToAcceptanceSeconds": 5143.882, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 2.704, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "stages": [ { "id": "stage-01", "title": "Build the relational virtual-data app first", "cost": { "status": "observed", "value": 183, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": null, "increaseSincePrecedingReading": null }, { "id": "stage-02", "title": "Connect the existing managed app to Office 365 Users", "cost": { "status": "observed", "value": 276, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 183, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 93, "unit": "credits" } }, { "id": "stage-03", "title": "Virtual approvals, asset lifecycle, themes and preferences", "cost": { "status": "observed", "value": 397, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 276, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 121, "unit": "credits" } }, { "id": "stage-04", "title": "Approval policy change, revisions, cost centres and responsive refinement", "cost": { "status": "observed", "value": 573, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 442, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 131, "unit": "credits" } } ], "promptSha256": "00e76f1c01c59953a44e6199dedb1bad741dbb4ec4f5051c18e150d32f9a175c", "promptVersion": "complex-private-v2", "archived": true, "screenshotFiles": 378, "profileProof": { "verifiedInApp": true, "reverifiedAfterFinalReopen": true, "directoryWritesObserved": 0 }, "preferences": { "saveCancelAndPersistence": "verified", "requestRowDensity": "not-demonstrated", "rowHeightDifferenceCssPixels": 0, "gapDifferenceCssPixels": 0, "defaultDepartmentConsumption": "not-demonstrated" }, "checks": [ { "id": "CX01", "capability": "Same private app opens with all seven navigable screens", "critical": true, "final": "pass" }, { "id": "CX02", "capability": "Exact six-device/four-request relational seed and derived initial metrics", "critical": true, "final": "pass" }, { "id": "CX03", "capability": "Catalogue search, combined category/stock filters and zero-stock protection", "critical": false, "final": "pass" }, { "id": "CX04", "capability": "Multi-line draft creation, line edits/removal and accurate GBP totals", "critical": true, "final": "pass" }, { "id": "CX05", "capability": "Required fields, dates, whole positive quantities, duplicates and stock validation", "critical": true, "final": "pass" }, { "id": "CX06", "capability": "Draft/persona isolation, edits and exact-once submission with stable IDs", "critical": true, "final": "pass" }, { "id": "CX07", "capability": "Controller-created draft, IDs, lines and timeline survive all three planned changes and reopen", "critical": true, "final": "pass" }, { "id": "CX08", "capability": "Virtual manager/finance role gates and approval transitions at the final threshold", "critical": true, "final": "pass" }, { "id": "CX09", "capability": "Rejection comments, append-only actor/timestamp history and terminal action guards", "critical": false, "final": "pass" }, { "id": "CX10", "capability": "Request changes, requester edit/resubmit and cancellation preserve identity/history", "critical": false, "final": "pass" }, { "id": "CX11", "capability": "Atomic fulfilment, stock revalidation, per-unit assets and duplicate-fulfilment protection", "critical": true, "final": "pass" }, { "id": "CX12", "capability": "Good/damaged asset returns and duplicate-return protection", "critical": false, "final": "pass" }, { "id": "CX13", "capability": "Current-user Office 365 Users profile actually reads through supported app binding", "critical": true, "final": "pass" }, { "id": "CX14", "capability": "Profile loading/error/retry and genuinely missing optional fields are honest", "critical": false, "final": "blocked" }, { "id": "CX15", "capability": "Explicit light/dark mode applies across all screens and persists on reopen", "critical": true, "final": "pass" }, { "id": "CX16", "capability": "Local preference Save/Cancel, density and default department work without directory writes", "critical": false, "final": "blocked" }, { "id": "CX17", "capability": "Cost-centre migration, persistence and department mismatch validation", "critical": false, "final": "pass" }, { "id": "CX18", "capability": "Derived status/asset/stock metrics, low-stock navigation and combined request filters", "critical": false, "final": "pass" }, { "id": "CX19", "capability": "All screens and key forms are usable at actual wide desktop and 390px mobile widths", "critical": true, "final": "pass" }, { "id": "CX20", "capability": "Labels, visible keyboard focus, readable contrast and reachable form errors", "critical": false, "final": "fail" } ] }, { "id": "run-07", "model": "Sonnet 5", "effort": "High", "replicate": 2, "firstCost": { "status": "observed", "value": 433, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 2134, "unit": "credits", "scope": "Cowork task" }, "final": { "points": 85, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "CX14", "CX16" ] }, "corrections": 1, "clarifications": 0, "reviews": 0, "plannedChanges": 3, "timing": { "build": { "seconds": 2950.698, "lowerBoundSeconds": 2923.852, "observation": "interval-bounded", "authoringPrompts": 5, "observedPrompts": 5 }, "firstBuildResponseSeconds": 1037.248, "firstBuildResponseLowerBoundSeconds": 1033.201, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 5135.595, "timeToAcceptanceSeconds": 5135.595, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 4.047, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "stages": [ { "id": "stage-01", "title": "Build the relational virtual-data app first", "cost": { "status": "observed", "value": 433, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": null, "increaseSincePrecedingReading": null }, { "id": "stage-02", "title": "Connect the existing managed app to Office 365 Users", "cost": { "status": "observed", "value": 654, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 497, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 157, "unit": "credits" } }, { "id": "stage-03", "title": "Virtual approvals, asset lifecycle, themes and preferences", "cost": { "status": "observed", "value": 1232, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 654, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 578, "unit": "credits" } }, { "id": "stage-04", "title": "Approval policy change, revisions, cost centres and responsive refinement", "cost": { "status": "observed", "value": 2134, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 1232, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 902, "unit": "credits" } } ], "promptSha256": "00e76f1c01c59953a44e6199dedb1bad741dbb4ec4f5051c18e150d32f9a175c", "promptVersion": "complex-private-v2", "archived": true, "screenshotFiles": 401, "profileProof": { "verifiedInApp": true, "reverifiedAfterFinalReopen": true, "directoryWritesObserved": 0 }, "preferences": { "saveCancelAndPersistence": "verified", "requestRowDensity": "verified", "rowHeightDifferenceCssPixels": 4, "gapDifferenceCssPixels": 0, "defaultDepartmentConsumption": "not-demonstrated" }, "checks": [ { "id": "CX01", "capability": "Same private app opens with all seven navigable screens", "critical": true, "final": "pass" }, { "id": "CX02", "capability": "Exact six-device/four-request relational seed and derived initial metrics", "critical": true, "final": "pass" }, { "id": "CX03", "capability": "Catalogue search, combined category/stock filters and zero-stock protection", "critical": false, "final": "pass" }, { "id": "CX04", "capability": "Multi-line draft creation, line edits/removal and accurate GBP totals", "critical": true, "final": "pass" }, { "id": "CX05", "capability": "Required fields, dates, whole positive quantities, duplicates and stock validation", "critical": true, "final": "pass" }, { "id": "CX06", "capability": "Draft/persona isolation, edits and exact-once submission with stable IDs", "critical": true, "final": "pass" }, { "id": "CX07", "capability": "Controller-created draft, IDs, lines and timeline survive all three planned changes and reopen", "critical": true, "final": "pass" }, { "id": "CX08", "capability": "Virtual manager/finance role gates and approval transitions at the final threshold", "critical": true, "final": "pass" }, { "id": "CX09", "capability": "Rejection comments, append-only actor/timestamp history and terminal action guards", "critical": false, "final": "pass" }, { "id": "CX10", "capability": "Request changes, requester edit/resubmit and cancellation preserve identity/history", "critical": false, "final": "pass" }, { "id": "CX11", "capability": "Atomic fulfilment, stock revalidation, per-unit assets and duplicate-fulfilment protection", "critical": true, "final": "pass" }, { "id": "CX12", "capability": "Good/damaged asset returns and duplicate-return protection", "critical": false, "final": "pass" }, { "id": "CX13", "capability": "Current-user Office 365 Users profile actually reads through supported app binding", "critical": true, "final": "pass" }, { "id": "CX14", "capability": "Profile loading/error/retry and genuinely missing optional fields are honest", "critical": false, "final": "blocked" }, { "id": "CX15", "capability": "Explicit light/dark mode applies across all screens and persists on reopen", "critical": true, "final": "pass" }, { "id": "CX16", "capability": "Local preference Save/Cancel, density and default department work without directory writes", "critical": false, "final": "blocked" }, { "id": "CX17", "capability": "Cost-centre migration, persistence and department mismatch validation", "critical": false, "final": "pass" }, { "id": "CX18", "capability": "Derived status/asset/stock metrics, low-stock navigation and combined request filters", "critical": false, "final": "pass" }, { "id": "CX19", "capability": "All screens and key forms are usable at actual wide desktop and 390px mobile widths", "critical": true, "final": "pass" }, { "id": "CX20", "capability": "Labels, visible keyboard focus, readable contrast and reachable form errors", "critical": false, "final": "fail" } ] }, { "id": "run-09", "model": "Opus 5", "effort": "High", "replicate": 2, "firstCost": { "status": "observed", "value": 521, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 2860, "unit": "credits", "scope": "Cowork task" }, "final": { "points": 95, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [ "CX14" ] }, "corrections": 2, "clarifications": 0, "reviews": 0, "plannedChanges": 3, "timing": { "build": { "seconds": 2565.492, "lowerBoundSeconds": 2541.55, "observation": "interval-bounded", "authoringPrompts": 6, "observedPrompts": 6 }, "firstBuildResponseSeconds": 824.791, "firstBuildResponseLowerBoundSeconds": 821.099, "firstCompletionObservation": "interval-bounded", "finalReviewedResultSeconds": 26070.834, "timeToAcceptanceSeconds": 26070.834, "observedApprovalWaitSeconds": 0, "recordedInterruptionSeconds": 0, "approvalWaitStillOpen": false, "interruptionStillOpen": false, "firstCompletionDetectionWindowSeconds": 3.692, "basis": "Observed wall-clock time, including waits; not isolated model execution. A completion first seen after it happened is an upper bound. Missing endpoints stay null." }, "stages": [ { "id": "stage-01", "title": "Build the relational virtual-data app first", "cost": { "status": "observed", "value": 521, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": null, "increaseSincePrecedingReading": null }, { "id": "stage-02", "title": "Connect the existing managed app to Office 365 Users", "cost": { "status": "observed", "value": 1219, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 832, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 387, "unit": "credits" } }, { "id": "stage-03", "title": "Virtual approvals, asset lifecycle, themes and preferences", "cost": { "status": "observed", "value": 1860, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 1219, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 641, "unit": "credits" } }, { "id": "stage-04", "title": "Approval policy change, revisions, cost centres and responsive refinement", "cost": { "status": "observed", "value": 2431, "unit": "credits", "scope": "Cowork task" }, "costBeforePrompt": { "status": "observed", "value": 1860, "unit": "credits", "scope": "Cowork task" }, "increaseSincePrecedingReading": { "value": 571, "unit": "credits" } } ], "promptSha256": "00e76f1c01c59953a44e6199dedb1bad741dbb4ec4f5051c18e150d32f9a175c", "promptVersion": "complex-private-v2", "archived": true, "screenshotFiles": 465, "profileProof": { "verifiedInApp": true, "reverifiedAfterFinalReopen": true, "directoryWritesObserved": 0 }, "preferences": { "saveCancelAndPersistence": "verified", "requestRowDensity": "verified", "rowHeightDifferenceCssPixels": 8, "gapDifferenceCssPixels": 0, "defaultDepartmentConsumption": "verified", "defaultDepartmentSurfaces": [ "Approvals department filter", "All-requests department filter" ] }, "checks": [ { "id": "CX01", "capability": "Same private app opens with all seven navigable screens", "critical": true, "final": "pass" }, { "id": "CX02", "capability": "Exact six-device/four-request relational seed and derived initial metrics", "critical": true, "final": "pass" }, { "id": "CX03", "capability": "Catalogue search, combined category/stock filters and zero-stock protection", "critical": false, "final": "pass" }, { "id": "CX04", "capability": "Multi-line draft creation, line edits/removal and accurate GBP totals", "critical": true, "final": "pass" }, { "id": "CX05", "capability": "Required fields, dates, whole positive quantities, duplicates and stock validation", "critical": true, "final": "pass" }, { "id": "CX06", "capability": "Draft/persona isolation, edits and exact-once submission with stable IDs", "critical": true, "final": "pass" }, { "id": "CX07", "capability": "Controller-created draft, IDs, lines and timeline survive all three planned changes and reopen", "critical": true, "final": "pass" }, { "id": "CX08", "capability": "Virtual manager/finance role gates and approval transitions at the final threshold", "critical": true, "final": "pass" }, { "id": "CX09", "capability": "Rejection comments, append-only actor/timestamp history and terminal action guards", "critical": false, "final": "pass" }, { "id": "CX10", "capability": "Request changes, requester edit/resubmit and cancellation preserve identity/history", "critical": false, "final": "pass" }, { "id": "CX11", "capability": "Atomic fulfilment, stock revalidation, per-unit assets and duplicate-fulfilment protection", "critical": true, "final": "pass" }, { "id": "CX12", "capability": "Good/damaged asset returns and duplicate-return protection", "critical": false, "final": "pass" }, { "id": "CX13", "capability": "Current-user Office 365 Users profile actually reads through supported app binding", "critical": true, "final": "pass" }, { "id": "CX14", "capability": "Profile loading/error/retry and genuinely missing optional fields are honest", "critical": false, "final": "blocked" }, { "id": "CX15", "capability": "Explicit light/dark mode applies across all screens and persists on reopen", "critical": true, "final": "pass" }, { "id": "CX16", "capability": "Local preference Save/Cancel, density and default department work without directory writes", "critical": false, "final": "pass" }, { "id": "CX17", "capability": "Cost-centre migration, persistence and department mismatch validation", "critical": false, "final": "pass" }, { "id": "CX18", "capability": "Derived status/asset/stock metrics, low-stock navigation and combined request filters", "critical": false, "final": "pass" }, { "id": "CX19", "capability": "All screens and key forms are usable at actual wide desktop and 390px mobile widths", "critical": true, "final": "pass" }, { "id": "CX20", "capability": "Labels, visible keyboard focus, readable contrast and reachable form errors", "critical": false, "final": "pass" } ] } ], "pairs": [ { "model": "GPT 5.5", "effort": "Medium", "observations": 2, "originalCredits": 944, "additionalCredits": 573, "meanFinalCredits": 758.5, "originalCorrections": 0, "additionalCorrections": 1 }, { "model": "Sonnet 5", "effort": "High", "observations": 2, "originalCredits": 2289, "additionalCredits": 2134, "meanFinalCredits": 2211.5, "originalCorrections": 0, "additionalCorrections": 1 }, { "model": "Opus 5", "effort": "High", "observations": 2, "originalCredits": 3073, "additionalCredits": 2860, "meanFinalCredits": 2966.5, "originalCorrections": 1, "additionalCorrections": 2 } ], "matchedConfigurationCount": 3, "originalMeanFinalCredits": 1998, "meanFinalCredits": 1855.6666666666667, "matchedOriginalMeanFinalCredits": 2102, "matchedAdditionalMeanFinalCredits": 1855.6666666666667, "matchedCombinedMeanFinalCredits": 1978.8333333333333, "note": "Completed externally reviewed applications only. Original results remain unchanged. Compare original and additional observations within the same model/effort settings shown.", "comparisonNote": "The paired means use exactly the same settings on both sides. The original overall average covers all five settings; do not attribute a different model mix to a model-price change.", "promptNote": "All four business stages and the acceptance rubric are unchanged. The additional foundation prompt permits private owner-only publication. This scope difference and two observations per paired setting prevent causal pricing or reliability claims.", "stageNote": "Stage readings are cumulative and must not be added together. An increase uses the last reading before that prompt, including any earlier repair; it is not an isolated connector or token price." }, "stressTest": { "test": "test-04", "route": "test-04", "title": "Test 4: capability stress test", "reviewMode": "external-only", "runs": [ { "id": "run-03", "model": "GPT 5.5", "effort": "Medium", "replicate": 1, "firstCompleteTargetCost": { "status": "observed", "value": 1095, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 1573, "unit": "credits", "scope": "Cowork task" }, "firstCore": { "points": 100, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [] }, "final": { "points": 100, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [] }, "firstAdvancedPoints": 80, "finalAdvancedPoints": 90, "incrementalGuidanceCredits": 478, "earlyCorrections": 0, "laterCorrections": 1, "corrections": 1, "clarifications": 0, "nativeReviews": 0, "plannedChanges": 3, "timing": { "build": { "seconds": 2092.087, "lowerBoundSeconds": 2078.532, "observation": "interval-bounded", "authoringPrompts": 5, "observedPrompts": 5 } }, "coreChecks": [ { "id": "TB01", "capability": "Same private managed app opens with all eight navigable areas", "critical": true, "first": "pass", "final": "pass" }, { "id": "TB02", "capability": "App-isolated persistence and exact four-device/two-request seed with correct derived metrics", "critical": true, "first": "pass", "final": "pass" }, { "id": "TB03", "capability": "Multi-line draft creation, save, quantity/removal/edit and stable unchanged IDs with accurate simple totals", "critical": true, "first": "pass", "final": "pass" }, { "id": "TB04", "capability": "Required fields, dates, whole quantities, duplicate lines and useful field errors", "critical": false, "first": "pass", "final": "pass" }, { "id": "TB05", "capability": "Requester draft isolation and an ordinary submission enters the correct manager queue", "critical": true, "first": "pass", "final": "pass" }, { "id": "TB06", "capability": "Ordinary manager/finance approval and storekeeper fulfilment complete with matching visible stock", "critical": true, "first": "pass", "final": "pass" }, { "id": "TB07", "capability": "Required rejection reason, correct role actions and visible decision/audit history", "critical": false, "first": "pass", "final": "pass" }, { "id": "TB08", "capability": "Controller-created draft, unchanged line IDs and simple values survive all stages and native reopen", "critical": true, "first": "pass", "final": "pass" }, { "id": "TB09", "capability": "Light/dark modes apply consistently and persist without altering business records", "critical": false, "first": "pass", "final": "pass" }, { "id": "TB10", "capability": "All eight areas and core forms usable at wide desktop and actual 390px without page-wide overflow", "critical": true, "first": "pass", "final": "pass" } ], "advancedChecks": [ { "id": "TA01", "capability": "Historical price snapshots and exact integer discount/VAT rounding, including a half-penny tie", "critical": false, "first": "pass", "final": "pass", "recheckedAfterCorrection": false }, { "id": "TA02", "capability": "Pending and approved V1 decisions survive V2; revised/new/boundary requests follow the correct version", "critical": false, "first": "pass", "final": "pass", "recheckedAfterCorrection": false }, { "id": "TA03", "capability": "Self-approval and same-person manager/finance decisions are blocked across synthetic dual roles", "critical": false, "first": "pass", "final": "pass", "recheckedAfterCorrection": false }, { "id": "TA04", "capability": "Budget reservation is atomic at the insufficient-funds boundary and releases correctly on revision/cancellation", "critical": false, "first": "pass", "final": "pass", "recheckedAfterCorrection": false }, { "id": "TA05", "capability": "Partial fulfilment and closing the remainder conserve stock, quantities, reserved/spent value and every penny", "critical": false, "first": "pass", "final": "pass", "recheckedAfterCorrection": false }, { "id": "TA06", "capability": "Multi-line fulfilment with one insufficient-stock line makes no partial business-state changes", "critical": false, "first": "pass", "final": "pass", "recheckedAfterCorrection": false }, { "id": "TA07", "capability": "Double confirmation and replay from a stale tab do not duplicate or overwrite a request operation", "critical": false, "first": "fail", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TA08", "capability": "Two stale request views competing for one remaining department budget cannot both reserve it", "critical": false, "first": "pass", "final": "pass", "recheckedAfterCorrection": false }, { "id": "TA09", "capability": "Price-import preview, cancellation, all-row validation, atomic apply and stale-preview rejection work", "critical": false, "first": "pass", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TA10", "capability": "Keyboard dialogs/focus/errors and sampled control/error contrast meet the specified desktop/mobile targets in both themes", "critical": false, "first": "fail", "final": "untested", "recheckedAfterCorrection": true } ], "archived": true, "reviewSurface": "Native owner preview of the same privately published app; later edits are not claimed republished.", "stages": [ { "id": "stage-01", "title": "Isolated relational foundation and stable request editing", "cost": { "status": "observed", "value": 210, "unit": "credits", "scope": "Cowork task" } }, { "id": "stage-02", "title": "Versioned approvals, budget reservation and partial fulfilment", "cost": { "status": "observed", "value": 431, "unit": "credits", "scope": "Cowork task" } }, { "id": "stage-03", "title": "Change policy and money rules while preserving existing business state", "cost": { "status": "observed", "value": 677, "unit": "credits", "scope": "Cowork task" } }, { "id": "stage-04", "title": "Competing updates, atomic imports and accessible interaction", "cost": { "status": "observed", "value": 1095, "unit": "credits", "scope": "Cowork task" } } ] }, { "id": "run-08", "model": "Opus 5", "effort": "Medium", "replicate": 2, "firstCompleteTargetCost": { "status": "observed", "value": 2426, "unit": "credits", "scope": "Cowork task" }, "finalCost": { "status": "observed", "value": 2938, "unit": "credits", "scope": "Cowork task" }, "firstCore": { "points": 70, "maximum": 100, "accepted": false, "criticalFailures": [ "TB05", "TB06" ], "incomplete": [] }, "final": { "points": 100, "maximum": 100, "accepted": true, "criticalFailures": [], "incomplete": [] }, "firstAdvancedPoints": 0, "finalAdvancedPoints": 70, "incrementalGuidanceCredits": 512, "earlyCorrections": 1, "laterCorrections": 1, "corrections": 2, "clarifications": 0, "nativeReviews": 0, "plannedChanges": 3, "timing": { "build": { "seconds": 3098.346, "lowerBoundSeconds": 3013.07, "observation": "interval-bounded", "authoringPrompts": 6, "observedPrompts": 6 } }, "coreChecks": [ { "id": "TB01", "capability": "Same private managed app opens with all eight navigable areas", "critical": true, "first": "pass", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TB02", "capability": "App-isolated persistence and exact four-device/two-request seed with correct derived metrics", "critical": true, "first": "pass", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TB03", "capability": "Multi-line draft creation, save, quantity/removal/edit and stable unchanged IDs with accurate simple totals", "critical": true, "first": "pass", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TB04", "capability": "Required fields, dates, whole quantities, duplicate lines and useful field errors", "critical": false, "first": "pass", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TB05", "capability": "Requester draft isolation and an ordinary submission enters the correct manager queue", "critical": true, "first": "fail", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TB06", "capability": "Ordinary manager/finance approval and storekeeper fulfilment complete with matching visible stock", "critical": true, "first": "fail", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TB07", "capability": "Required rejection reason, correct role actions and visible decision/audit history", "critical": false, "first": "fail", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TB08", "capability": "Controller-created draft, unchanged line IDs and simple values survive all stages and native reopen", "critical": true, "first": "pass", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TB09", "capability": "Light/dark modes apply consistently and persist without altering business records", "critical": false, "first": "pass", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TB10", "capability": "All eight areas and core forms usable at wide desktop and actual 390px without page-wide overflow", "critical": true, "first": "pass", "final": "pass", "recheckedAfterCorrection": true } ], "advancedChecks": [ { "id": "TA01", "capability": "Historical price snapshots and exact integer discount/VAT rounding, including a half-penny tie", "critical": false, "first": "fail", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TA02", "capability": "Pending and approved V1 decisions survive V2; revised/new/boundary requests follow the correct version", "critical": false, "first": "fail", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TA03", "capability": "Self-approval and same-person manager/finance decisions are blocked across synthetic dual roles", "critical": false, "first": "blocked", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TA04", "capability": "Budget reservation is atomic at the insufficient-funds boundary and releases correctly on revision/cancellation", "critical": false, "first": "blocked", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TA05", "capability": "Partial fulfilment and closing the remainder conserve stock, quantities, reserved/spent value and every penny", "critical": false, "first": "blocked", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TA06", "capability": "Multi-line fulfilment with one insufficient-stock line makes no partial business-state changes", "critical": false, "first": "blocked", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TA07", "capability": "Double confirmation and replay from a stale tab do not duplicate or overwrite a request operation", "critical": false, "first": "fail", "final": "blocked", "recheckedAfterCorrection": true }, { "id": "TA08", "capability": "Two stale request views competing for one remaining department budget cannot both reserve it", "critical": false, "first": "blocked", "final": "blocked", "recheckedAfterCorrection": true }, { "id": "TA09", "capability": "Price-import preview, cancellation, all-row validation, atomic apply and stale-preview rejection work", "critical": false, "first": "fail", "final": "pass", "recheckedAfterCorrection": true }, { "id": "TA10", "capability": "Keyboard dialogs/focus/errors and sampled control/error contrast meet the specified desktop/mobile targets in both themes", "critical": false, "first": "fail", "final": "fail", "recheckedAfterCorrection": true } ], "archived": true, "reviewSurface": "Native owner preview of the same privately published app; later edits are not claimed republished.", "stages": [ { "id": "stage-01", "title": "Isolated relational foundation and stable request editing", "cost": { "status": "observed", "value": 585, "unit": "credits", "scope": "Cowork task" } }, { "id": "stage-02", "title": "Versioned approvals, budget reservation and partial fulfilment", "cost": { "status": "observed", "value": 1392, "unit": "credits", "scope": "Cowork task" } }, { "id": "stage-03", "title": "Change policy and money rules while preserving existing business state", "cost": { "status": "observed", "value": 1928, "unit": "credits", "scope": "Cowork task" } }, { "id": "stage-04", "title": "Competing updates, atomic imports and accessible interaction", "cost": { "status": "observed", "value": 2426, "unit": "credits", "scope": "Cowork task" } } ] } ], "meanFinalCredits": 2255.5, "note": "All five model/effort settings have actual final task readings. Only accepted, independently reviewed completed apps enter the accepted-app average and capability matrix. Spend-only readings are actual task costs, not accepted-app prices or zero capability scores.", "scope": "Eight areas; multi-line virtual requests; versioned approvals; budget reservations; partial fulfilment; exact-pence pricing; stale-view guards; atomic price imports; light/dark desktop and mobile interaction. No live business or profile connector.", "scoreNote": "Core acceptance and ten advanced challenges are separate. Each check is worth ten points; every frozen subcase must be verified for an advanced pass. Fail, blocked and untested each earn zero. No combined quality-per-credit score.", "guidanceNote": "Four identical planned prompts per app. Early corrections are included in first-complete-target cost; it is not necessarily a no-guidance first pass. Later guidance is final minus first-complete-target credits. Cumulative readings must not be added.", "coverageNote": "Every frozen subcase is required for a pass. Automatic cross-tab refresh can prevent an intentionally stale form from remaining available; an unexercised stale prerequisite is a coverage gap, not a demonstrated race defect. The keyboard scenario specifies request search and a rejection-reason dialog, whereas the authoring prompts allow other interfaces. A different interface is not automatically a requested-feature defect. Earlier historical evidence and post-correction rechecks remain distinct; no source or data is altered to force a test precondition.", "limitation": "Controlled same-context local simulations and sampled accessibility do not establish production concurrency, real authorization or accessibility certification. The selected Cowork model does not prove every internal app-building worker used that model. Compare only within this test, not against Tests 1-3 scores or averages.", "registeredSettings": [ { "runId": "run-03", "model": "GPT 5.5", "effort": "Medium", "cohort": "Original", "status": "accepted", "finalSpend": { "status": "observed", "value": 1573, "unit": "credits", "scope": "Cowork task" } }, { "runId": "run-06", "model": "Sonnet 5", "effort": "Medium", "status": "not-accepted", "finalSpend": { "status": "observed", "value": 1992, "unit": "credits", "scope": "Cowork task" }, "cohort": "Follow-up" }, { "runId": "run-07", "model": "Opus 5", "effort": "High", "status": "not-accepted", "finalSpend": { "status": "observed", "value": 8144, "unit": "credits", "scope": "Cowork task" }, "cohort": "Follow-up" }, { "runId": "run-08", "model": "Opus 5", "effort": "Medium", "status": "accepted", "finalSpend": { "status": "observed", "value": 2938, "unit": "credits", "scope": "Cowork task" }, "cohort": "Follow-up" }, { "runId": "run-09", "model": "Sonnet 5", "effort": "High", "status": "not-accepted", "finalSpend": { "status": "observed", "value": 4229, "unit": "credits", "scope": "Cowork task" }, "cohort": "Follow-up" } ], "followupNote": "The original GPT Medium observation is retained alongside four separately registered follow-ups using the identical original prompts, fixtures, rubrics and authoring/correction limits. Original outcomes are not overwritten. All four follow-ups are closed. This is a disclosed follow-up comparison, not five original first attempts; unfinished or unaccepted spend is excluded from accepted-app averages." }, "htmlReferenceTest": { "test": "test-05", "route": "test-05", "cohort": "original-html-reference", "title": "Test 5: recreate an HTML reference", "reviewMode": "external-only", "protocolSha256": "f93b8aaf313977049c80c523ce9cf0ddd75b5b9921bb2073eaece9fe6f99183c", "buildPromptSha256": "c3c9e4e9de95e488fde6ecd99c7ba443be63302664b587c86ffc2bb6f1ef17ff", "referenceSha256": "b04d8b8bb71e18b2429a4ddf995b698fcaebfbeac1f9926c804f437d256b0d28", "referenceBytes": 188501, "costMode": "complete-app-only", "runs": [ { "id": "run-01", "model": "Sonnet 5", "effort": "Medium", "replicate": 1, "status": "accepted", "finalCost": { "status": "observed", "value": 2517, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 8, "corrections": 7, "clarifications": 0, "freeCostCommands": 2, "final": { "accepted": true, "functionalPassed": 17, "functionalTotal": 20, "visualPassed": 8, "visualTotal": 10, "allCriticalPassed": true }, "functionalChecks": [ { "id": "HF01", "capability": "Working managed app, complete role-aware scope and reference fixture", "critical": true, "final": "pass" }, { "id": "HF02", "capability": "Derived dashboards and balances", "critical": true, "final": "pass" }, { "id": "HF03", "capability": "My Leave filters, sorting and empty states", "critical": false, "final": "pass" }, { "id": "HF04", "capability": "Request detail and retained history", "critical": false, "final": "pass" }, { "id": "HF05", "capability": "Four-step request wizard", "critical": true, "final": "pass" }, { "id": "HF06", "capability": "Working-day and half-day arithmetic", "critical": true, "final": "pass" }, { "id": "HF07", "capability": "Blocking validation", "critical": true, "final": "pass" }, { "id": "HF08", "capability": "Warnings and policy-dependent validation", "critical": false, "final": "pass" }, { "id": "HF09", "capability": "Draft, edit and duplicate lifecycle", "critical": false, "final": "pass" }, { "id": "HF10", "capability": "Submission, approval routing and notifications", "critical": true, "final": "pass" }, { "id": "HF11", "capability": "Withdraw and cancel with balance reconciliation", "critical": false, "final": "pass" }, { "id": "HF12", "capability": "Manager decisions and reasons", "critical": true, "final": "pass" }, { "id": "HF13", "capability": "HR second-step and no-manager routing", "critical": true, "final": "pass" }, { "id": "HF14", "capability": "Bulk decisions, delegation and escalation", "critical": false, "final": "fail" }, { "id": "HF15", "capability": "Team calendars", "critical": false, "final": "pass" }, { "id": "HF16", "capability": "Scoped reports and actual CSV exports", "critical": false, "final": "fail" }, { "id": "HF17", "capability": "Live policy and pro-rata calculator", "critical": false, "final": "pass" }, { "id": "HF18", "capability": "Six working HR settings tabs", "critical": true, "final": "pass" }, { "id": "HF19", "capability": "Search, notifications, themes and keyboard interaction", "critical": false, "final": "fail" }, { "id": "HF20", "capability": "Faithful demo lifecycle and isolation", "critical": true, "final": "pass" } ], "visualChecks": [ { "id": "HV01", "capability": "Reference identity and shell", "critical": true, "final": "pass" }, { "id": "HV02", "capability": "Dashboard composition", "critical": false, "final": "pass" }, { "id": "HV03", "capability": "Wizard and form hierarchy", "critical": false, "final": "pass" }, { "id": "HV04", "capability": "Tables, filters and status treatment", "critical": false, "final": "pass" }, { "id": "HV05", "capability": "Calendar visual encoding", "critical": false, "final": "pass" }, { "id": "HV06", "capability": "Reports, charts and settings layouts", "critical": false, "final": "fail" }, { "id": "HV07", "capability": "Light/dark palette and typography", "critical": false, "final": "pass" }, { "id": "HV08", "capability": "Cards, spacing and content density", "critical": false, "final": "fail" }, { "id": "HV09", "capability": "Dialogs, drawers and feedback", "critical": false, "final": "pass" }, { "id": "HV10", "capability": "Responsive correspondence", "critical": true, "final": "pass" } ], "timing": { "build": { "seconds": 5529.797, "lowerBoundSeconds": 5453.061, "observation": "interval-bounded", "authoringPrompts": 8, "observedPrompts": 8 } }, "archived": true }, { "id": "run-02", "model": "Opus 5", "effort": "High", "replicate": 1, "status": "accepted", "finalCost": { "status": "observed", "value": 3197, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 3, "corrections": 2, "clarifications": 0, "freeCostCommands": 1, "final": { "accepted": true, "functionalPassed": 19, "functionalTotal": 20, "visualPassed": 8, "visualTotal": 10, "allCriticalPassed": true }, "functionalChecks": [ { "id": "HF01", "capability": "Working managed app, complete role-aware scope and reference fixture", "critical": true, "final": "pass" }, { "id": "HF02", "capability": "Derived dashboards and balances", "critical": true, "final": "pass" }, { "id": "HF03", "capability": "My Leave filters, sorting and empty states", "critical": false, "final": "pass" }, { "id": "HF04", "capability": "Request detail and retained history", "critical": false, "final": "pass" }, { "id": "HF05", "capability": "Four-step request wizard", "critical": true, "final": "pass" }, { "id": "HF06", "capability": "Working-day and half-day arithmetic", "critical": true, "final": "pass" }, { "id": "HF07", "capability": "Blocking validation", "critical": true, "final": "pass" }, { "id": "HF08", "capability": "Warnings and policy-dependent validation", "critical": false, "final": "pass" }, { "id": "HF09", "capability": "Draft, edit and duplicate lifecycle", "critical": false, "final": "pass" }, { "id": "HF10", "capability": "Submission, approval routing and notifications", "critical": true, "final": "pass" }, { "id": "HF11", "capability": "Withdraw and cancel with balance reconciliation", "critical": false, "final": "pass" }, { "id": "HF12", "capability": "Manager decisions and reasons", "critical": true, "final": "pass" }, { "id": "HF13", "capability": "HR second-step and no-manager routing", "critical": true, "final": "pass" }, { "id": "HF14", "capability": "Bulk decisions, delegation and escalation", "critical": false, "final": "pass" }, { "id": "HF15", "capability": "Team calendars", "critical": false, "final": "pass" }, { "id": "HF16", "capability": "Scoped reports and actual CSV exports", "critical": false, "final": "fail" }, { "id": "HF17", "capability": "Live policy and pro-rata calculator", "critical": false, "final": "pass" }, { "id": "HF18", "capability": "Six working HR settings tabs", "critical": true, "final": "pass" }, { "id": "HF19", "capability": "Search, notifications, themes and keyboard interaction", "critical": false, "final": "pass" }, { "id": "HF20", "capability": "Faithful demo lifecycle and isolation", "critical": true, "final": "pass" } ], "visualChecks": [ { "id": "HV01", "capability": "Reference identity and shell", "critical": true, "final": "pass" }, { "id": "HV02", "capability": "Dashboard composition", "critical": false, "final": "pass" }, { "id": "HV03", "capability": "Wizard and form hierarchy", "critical": false, "final": "pass" }, { "id": "HV04", "capability": "Tables, filters and status treatment", "critical": false, "final": "pass" }, { "id": "HV05", "capability": "Calendar visual encoding", "critical": false, "final": "pass" }, { "id": "HV06", "capability": "Reports, charts and settings layouts", "critical": false, "final": "pass" }, { "id": "HV07", "capability": "Light/dark palette and typography", "critical": false, "final": "fail" }, { "id": "HV08", "capability": "Cards, spacing and content density", "critical": false, "final": "fail" }, { "id": "HV09", "capability": "Dialogs, drawers and feedback", "critical": false, "final": "pass" }, { "id": "HV10", "capability": "Responsive correspondence", "critical": true, "final": "pass" } ], "timing": { "build": { "seconds": 3179, "lowerBoundSeconds": 3140.938, "observation": "interval-bounded", "authoringPrompts": 3, "observedPrompts": 3 } }, "archived": true }, { "id": "run-03", "model": "GPT 5.5", "effort": "Medium", "replicate": 1, "status": "accepted", "finalCost": { "status": "observed", "value": 659, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 7, "corrections": 6, "clarifications": 0, "freeCostCommands": 1, "final": { "accepted": true, "functionalPassed": 17, "functionalTotal": 20, "visualPassed": 8, "visualTotal": 10, "allCriticalPassed": true }, "functionalChecks": [ { "id": "HF01", "capability": "Working managed app, complete role-aware scope and reference fixture", "critical": true, "final": "pass" }, { "id": "HF02", "capability": "Derived dashboards and balances", "critical": true, "final": "pass" }, { "id": "HF03", "capability": "My Leave filters, sorting and empty states", "critical": false, "final": "pass" }, { "id": "HF04", "capability": "Request detail and retained history", "critical": false, "final": "pass" }, { "id": "HF05", "capability": "Four-step request wizard", "critical": true, "final": "pass" }, { "id": "HF06", "capability": "Working-day and half-day arithmetic", "critical": true, "final": "pass" }, { "id": "HF07", "capability": "Blocking validation", "critical": true, "final": "pass" }, { "id": "HF08", "capability": "Warnings and policy-dependent validation", "critical": false, "final": "pass" }, { "id": "HF09", "capability": "Draft, edit and duplicate lifecycle", "critical": false, "final": "fail" }, { "id": "HF10", "capability": "Submission, approval routing and notifications", "critical": true, "final": "pass" }, { "id": "HF11", "capability": "Withdraw and cancel with balance reconciliation", "critical": false, "final": "pass" }, { "id": "HF12", "capability": "Manager decisions and reasons", "critical": true, "final": "pass" }, { "id": "HF13", "capability": "HR second-step and no-manager routing", "critical": true, "final": "pass" }, { "id": "HF14", "capability": "Bulk decisions, delegation and escalation", "critical": false, "final": "fail" }, { "id": "HF15", "capability": "Team calendars", "critical": false, "final": "pass" }, { "id": "HF16", "capability": "Scoped reports and actual CSV exports", "critical": false, "final": "pass" }, { "id": "HF17", "capability": "Live policy and pro-rata calculator", "critical": false, "final": "pass" }, { "id": "HF18", "capability": "Six working HR settings tabs", "critical": true, "final": "pass" }, { "id": "HF19", "capability": "Search, notifications, themes and keyboard interaction", "critical": false, "final": "fail" }, { "id": "HF20", "capability": "Faithful demo lifecycle and isolation", "critical": true, "final": "pass" } ], "visualChecks": [ { "id": "HV01", "capability": "Reference identity and shell", "critical": true, "final": "pass" }, { "id": "HV02", "capability": "Dashboard composition", "critical": false, "final": "pass" }, { "id": "HV03", "capability": "Wizard and form hierarchy", "critical": false, "final": "pass" }, { "id": "HV04", "capability": "Tables, filters and status treatment", "critical": false, "final": "pass" }, { "id": "HV05", "capability": "Calendar visual encoding", "critical": false, "final": "pass" }, { "id": "HV06", "capability": "Reports, charts and settings layouts", "critical": false, "final": "pass" }, { "id": "HV07", "capability": "Light/dark palette and typography", "critical": false, "final": "fail" }, { "id": "HV08", "capability": "Cards, spacing and content density", "critical": false, "final": "fail" }, { "id": "HV09", "capability": "Dialogs, drawers and feedback", "critical": false, "final": "pass" }, { "id": "HV10", "capability": "Responsive correspondence", "critical": true, "final": "pass" } ], "timing": { "build": { "seconds": 2909.171, "lowerBoundSeconds": 2820.289, "observation": "interval-bounded", "authoringPrompts": 7, "observedPrompts": 7 } }, "archived": true }, { "id": "run-04", "model": "Opus 5", "effort": "Medium", "replicate": 1, "status": "accepted", "finalCost": { "status": "observed", "value": 3220, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 2, "corrections": 1, "clarifications": 0, "freeCostCommands": 1, "final": { "accepted": true, "functionalPassed": 20, "functionalTotal": 20, "visualPassed": 8, "visualTotal": 10, "allCriticalPassed": true }, "functionalChecks": [ { "id": "HF01", "capability": "Working managed app, complete role-aware scope and reference fixture", "critical": true, "final": "pass" }, { "id": "HF02", "capability": "Derived dashboards and balances", "critical": true, "final": "pass" }, { "id": "HF03", "capability": "My Leave filters, sorting and empty states", "critical": false, "final": "pass" }, { "id": "HF04", "capability": "Request detail and retained history", "critical": false, "final": "pass" }, { "id": "HF05", "capability": "Four-step request wizard", "critical": true, "final": "pass" }, { "id": "HF06", "capability": "Working-day and half-day arithmetic", "critical": true, "final": "pass" }, { "id": "HF07", "capability": "Blocking validation", "critical": true, "final": "pass" }, { "id": "HF08", "capability": "Warnings and policy-dependent validation", "critical": false, "final": "pass" }, { "id": "HF09", "capability": "Draft, edit and duplicate lifecycle", "critical": false, "final": "pass" }, { "id": "HF10", "capability": "Submission, approval routing and notifications", "critical": true, "final": "pass" }, { "id": "HF11", "capability": "Withdraw and cancel with balance reconciliation", "critical": false, "final": "pass" }, { "id": "HF12", "capability": "Manager decisions and reasons", "critical": true, "final": "pass" }, { "id": "HF13", "capability": "HR second-step and no-manager routing", "critical": true, "final": "pass" }, { "id": "HF14", "capability": "Bulk decisions, delegation and escalation", "critical": false, "final": "pass" }, { "id": "HF15", "capability": "Team calendars", "critical": false, "final": "pass" }, { "id": "HF16", "capability": "Scoped reports and actual CSV exports", "critical": false, "final": "pass" }, { "id": "HF17", "capability": "Live policy and pro-rata calculator", "critical": false, "final": "pass" }, { "id": "HF18", "capability": "Six working HR settings tabs", "critical": true, "final": "pass" }, { "id": "HF19", "capability": "Search, notifications, themes and keyboard interaction", "critical": false, "final": "pass" }, { "id": "HF20", "capability": "Faithful demo lifecycle and isolation", "critical": true, "final": "pass" } ], "visualChecks": [ { "id": "HV01", "capability": "Reference identity and shell", "critical": true, "final": "pass" }, { "id": "HV02", "capability": "Dashboard composition", "critical": false, "final": "pass" }, { "id": "HV03", "capability": "Wizard and form hierarchy", "critical": false, "final": "pass" }, { "id": "HV04", "capability": "Tables, filters and status treatment", "critical": false, "final": "pass" }, { "id": "HV05", "capability": "Calendar visual encoding", "critical": false, "final": "pass" }, { "id": "HV06", "capability": "Reports, charts and settings layouts", "critical": false, "final": "fail" }, { "id": "HV07", "capability": "Light/dark palette and typography", "critical": false, "final": "fail" }, { "id": "HV08", "capability": "Cards, spacing and content density", "critical": false, "final": "pass" }, { "id": "HV09", "capability": "Dialogs, drawers and feedback", "critical": false, "final": "pass" }, { "id": "HV10", "capability": "Responsive correspondence", "critical": true, "final": "pass" } ], "timing": { "build": { "seconds": 3060.46, "lowerBoundSeconds": 3021.219, "observation": "interval-bounded", "authoringPrompts": 2, "observedPrompts": 2 } }, "archived": true }, { "id": "run-05", "model": "Sonnet 5", "effort": "High", "replicate": 1, "status": "accepted", "finalCost": { "status": "observed", "value": 2523, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 3, "corrections": 2, "clarifications": 0, "freeCostCommands": 1, "final": { "accepted": true, "functionalPassed": 18, "functionalTotal": 20, "visualPassed": 8, "visualTotal": 10, "allCriticalPassed": true }, "functionalChecks": [ { "id": "HF01", "capability": "Working managed app, complete role-aware scope and reference fixture", "critical": true, "final": "pass" }, { "id": "HF02", "capability": "Derived dashboards and balances", "critical": true, "final": "pass" }, { "id": "HF03", "capability": "My Leave filters, sorting and empty states", "critical": false, "final": "pass" }, { "id": "HF04", "capability": "Request detail and retained history", "critical": false, "final": "pass" }, { "id": "HF05", "capability": "Four-step request wizard", "critical": true, "final": "pass" }, { "id": "HF06", "capability": "Working-day and half-day arithmetic", "critical": true, "final": "pass" }, { "id": "HF07", "capability": "Blocking validation", "critical": true, "final": "pass" }, { "id": "HF08", "capability": "Warnings and policy-dependent validation", "critical": false, "final": "pass" }, { "id": "HF09", "capability": "Draft, edit and duplicate lifecycle", "critical": false, "final": "pass" }, { "id": "HF10", "capability": "Submission, approval routing and notifications", "critical": true, "final": "pass" }, { "id": "HF11", "capability": "Withdraw and cancel with balance reconciliation", "critical": false, "final": "pass" }, { "id": "HF12", "capability": "Manager decisions and reasons", "critical": true, "final": "pass" }, { "id": "HF13", "capability": "HR second-step and no-manager routing", "critical": true, "final": "pass" }, { "id": "HF14", "capability": "Bulk decisions, delegation and escalation", "critical": false, "final": "pass" }, { "id": "HF15", "capability": "Team calendars", "critical": false, "final": "fail" }, { "id": "HF16", "capability": "Scoped reports and actual CSV exports", "critical": false, "final": "pass" }, { "id": "HF17", "capability": "Live policy and pro-rata calculator", "critical": false, "final": "pass" }, { "id": "HF18", "capability": "Six working HR settings tabs", "critical": true, "final": "pass" }, { "id": "HF19", "capability": "Search, notifications, themes and keyboard interaction", "critical": false, "final": "fail" }, { "id": "HF20", "capability": "Faithful demo lifecycle and isolation", "critical": true, "final": "pass" } ], "visualChecks": [ { "id": "HV01", "capability": "Reference identity and shell", "critical": true, "final": "pass" }, { "id": "HV02", "capability": "Dashboard composition", "critical": false, "final": "pass" }, { "id": "HV03", "capability": "Wizard and form hierarchy", "critical": false, "final": "pass" }, { "id": "HV04", "capability": "Tables, filters and status treatment", "critical": false, "final": "pass" }, { "id": "HV05", "capability": "Calendar visual encoding", "critical": false, "final": "fail" }, { "id": "HV06", "capability": "Reports, charts and settings layouts", "critical": false, "final": "pass" }, { "id": "HV07", "capability": "Light/dark palette and typography", "critical": false, "final": "fail" }, { "id": "HV08", "capability": "Cards, spacing and content density", "critical": false, "final": "pass" }, { "id": "HV09", "capability": "Dialogs, drawers and feedback", "critical": false, "final": "pass" }, { "id": "HV10", "capability": "Responsive correspondence", "critical": true, "final": "pass" } ], "timing": { "build": { "seconds": 3651.981, "lowerBoundSeconds": 3632.183, "observation": "interval-bounded", "authoringPrompts": 3, "observedPrompts": 3 } }, "archived": true } ], "registeredSettings": [ { "runId": "run-01", "model": "Sonnet 5", "effort": "Medium", "status": "accepted", "finalCost": { "status": "observed", "value": 2517, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 8 }, { "runId": "run-02", "model": "Opus 5", "effort": "High", "status": "accepted", "finalCost": { "status": "observed", "value": 3197, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 3 }, { "runId": "run-03", "model": "GPT 5.5", "effort": "Medium", "status": "accepted", "finalCost": { "status": "observed", "value": 659, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 7 }, { "runId": "run-04", "model": "Opus 5", "effort": "Medium", "status": "accepted", "finalCost": { "status": "observed", "value": 3220, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 2 }, { "runId": "run-05", "model": "Sonnet 5", "effort": "High", "status": "accepted", "finalCost": { "status": "observed", "value": 2523, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 3 } ], "meanFinalCredits": 2423.2, "complete": true, "scope": "Annual Leave Manager: role-aware leave requests, approvals, calendars, reports and six HR settings tabs; fictional in-memory data only.", "method": "Identical initial prompt and frozen HTML bytes in five fresh tasks. Independent native functional and visual review; targeted repairs in the same app. No arbitrary authoring-time or credit cutoff and no paid self-review.", "acceptanceNote": "The frozen minimum requires 17 of 20 functional groups and 8 of 10 visual groups, including every critical group. Passing that minimum does not mean complete parity; all remaining failed groups are shown.", "costNote": "Only cumulative task credits after full independent review and joint acceptance. No interim, repair-only or initial-output credit figures. Free cost commands do not count as authoring prompts.", "limitation": "One original observation per setting; selected model and effort, not a proven internal worker model or a causal ranking. Guided replay is not included in this original cohort.", "privacy": "The exact HTML reference and selected final app-only screenshots are available in the authorized Test 5 showcase. The fixture is fictional. Transcripts, generated source, account/task/resource identifiers and other raw evidence remain private." }, "htmlReferenceRepeats": { "test": "test-05", "route": "test-05", "cohort": "matched-adaptive-repeat", "title": "Test 5: matched adaptive repeats", "reviewMode": "external-only", "protocolSha256": "8ca487b03edfacf06a51444cb384af9488135dc2cc5a7332002c6ba41d3cd018", "buildPromptSha256": "c3c9e4e9de95e488fde6ecd99c7ba443be63302664b587c86ffc2bb6f1ef17ff", "referenceSha256": "b04d8b8bb71e18b2429a4ddf995b698fcaebfbeac1f9926c804f437d256b0d28", "referenceBytes": 188501, "costMode": "complete-app-only", "runs": [ { "id": "run-06", "model": "Sonnet 5", "effort": "High", "replicate": 2, "matchedOriginalRunId": "run-05", "status": "accepted", "finalCost": { "status": "observed", "value": 3153, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 6, "corrections": 5, "clarifications": 0, "freeCostCommands": 1, "final": { "accepted": true, "functionalPassed": 17, "functionalTotal": 20, "visualPassed": 8, "visualTotal": 10, "allCriticalPassed": true }, "functionalChecks": [ { "id": "HF01", "capability": "Working managed app, complete role-aware scope and reference fixture", "critical": true, "final": "pass" }, { "id": "HF02", "capability": "Derived dashboards and balances", "critical": true, "final": "pass" }, { "id": "HF03", "capability": "My Leave filters, sorting and empty states", "critical": false, "final": "pass" }, { "id": "HF04", "capability": "Request detail and retained history", "critical": false, "final": "fail" }, { "id": "HF05", "capability": "Four-step request wizard", "critical": true, "final": "pass" }, { "id": "HF06", "capability": "Working-day and half-day arithmetic", "critical": true, "final": "pass" }, { "id": "HF07", "capability": "Blocking validation", "critical": true, "final": "pass" }, { "id": "HF08", "capability": "Warnings and policy-dependent validation", "critical": false, "final": "pass" }, { "id": "HF09", "capability": "Draft, edit and duplicate lifecycle", "critical": false, "final": "fail" }, { "id": "HF10", "capability": "Submission, approval routing and notifications", "critical": true, "final": "pass" }, { "id": "HF11", "capability": "Withdraw and cancel with balance reconciliation", "critical": false, "final": "pass" }, { "id": "HF12", "capability": "Manager decisions and reasons", "critical": true, "final": "pass" }, { "id": "HF13", "capability": "HR second-step and no-manager routing", "critical": true, "final": "pass" }, { "id": "HF14", "capability": "Bulk decisions, delegation and escalation", "critical": false, "final": "pass" }, { "id": "HF15", "capability": "Team calendars", "critical": false, "final": "pass" }, { "id": "HF16", "capability": "Scoped reports and actual CSV exports", "critical": false, "final": "pass" }, { "id": "HF17", "capability": "Live policy and pro-rata calculator", "critical": false, "final": "pass" }, { "id": "HF18", "capability": "Six working HR settings tabs", "critical": true, "final": "pass" }, { "id": "HF19", "capability": "Search, notifications, themes and keyboard interaction", "critical": false, "final": "fail" }, { "id": "HF20", "capability": "Faithful demo lifecycle and isolation", "critical": true, "final": "pass" } ], "visualChecks": [ { "id": "HV01", "capability": "Reference identity and shell", "critical": true, "final": "pass" }, { "id": "HV02", "capability": "Dashboard composition", "critical": false, "final": "pass" }, { "id": "HV03", "capability": "Wizard and form hierarchy", "critical": false, "final": "pass" }, { "id": "HV04", "capability": "Tables, filters and status treatment", "critical": false, "final": "pass" }, { "id": "HV05", "capability": "Calendar visual encoding", "critical": false, "final": "pass" }, { "id": "HV06", "capability": "Reports, charts and settings layouts", "critical": false, "final": "pass" }, { "id": "HV07", "capability": "Light/dark palette and typography", "critical": false, "final": "fail" }, { "id": "HV08", "capability": "Cards, spacing and content density", "critical": false, "final": "fail" }, { "id": "HV09", "capability": "Dialogs, drawers and feedback", "critical": false, "final": "pass" }, { "id": "HV10", "capability": "Responsive correspondence", "critical": true, "final": "pass" } ], "timing": { "build": { "seconds": 4536.348, "lowerBoundSeconds": null, "observation": "upper-bound-only", "authoringPrompts": 6, "observedPrompts": 6 } }, "archived": true }, { "id": "run-08", "model": "GPT 5.5", "effort": "Medium", "replicate": 2, "matchedOriginalRunId": "run-03", "status": "accepted", "finalCost": { "status": "observed", "value": 7464, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 16, "corrections": 8, "clarifications": 7, "freeCostCommands": 1, "final": { "accepted": true, "functionalPassed": 20, "functionalTotal": 20, "visualPassed": 8, "visualTotal": 10, "allCriticalPassed": true }, "functionalChecks": [ { "id": "HF01", "capability": "Working managed app, complete role-aware scope and reference fixture", "critical": true, "final": "pass" }, { "id": "HF02", "capability": "Derived dashboards and balances", "critical": true, "final": "pass" }, { "id": "HF03", "capability": "My Leave filters, sorting and empty states", "critical": false, "final": "pass" }, { "id": "HF04", "capability": "Request detail and retained history", "critical": false, "final": "pass" }, { "id": "HF05", "capability": "Four-step request wizard", "critical": true, "final": "pass" }, { "id": "HF06", "capability": "Working-day and half-day arithmetic", "critical": true, "final": "pass" }, { "id": "HF07", "capability": "Blocking validation", "critical": true, "final": "pass" }, { "id": "HF08", "capability": "Warnings and policy-dependent validation", "critical": false, "final": "pass" }, { "id": "HF09", "capability": "Draft, edit and duplicate lifecycle", "critical": false, "final": "pass" }, { "id": "HF10", "capability": "Submission, approval routing and notifications", "critical": true, "final": "pass" }, { "id": "HF11", "capability": "Withdraw and cancel with balance reconciliation", "critical": false, "final": "pass" }, { "id": "HF12", "capability": "Manager decisions and reasons", "critical": true, "final": "pass" }, { "id": "HF13", "capability": "HR second-step and no-manager routing", "critical": true, "final": "pass" }, { "id": "HF14", "capability": "Bulk decisions, delegation and escalation", "critical": false, "final": "pass" }, { "id": "HF15", "capability": "Team calendars", "critical": false, "final": "pass" }, { "id": "HF16", "capability": "Scoped reports and actual CSV exports", "critical": false, "final": "pass" }, { "id": "HF17", "capability": "Live policy and pro-rata calculator", "critical": false, "final": "pass" }, { "id": "HF18", "capability": "Six working HR settings tabs", "critical": true, "final": "pass" }, { "id": "HF19", "capability": "Search, notifications, themes and keyboard interaction", "critical": false, "final": "pass" }, { "id": "HF20", "capability": "Faithful demo lifecycle and isolation", "critical": true, "final": "pass" } ], "visualChecks": [ { "id": "HV01", "capability": "Reference identity and shell", "critical": true, "final": "pass" }, { "id": "HV02", "capability": "Dashboard composition", "critical": false, "final": "pass" }, { "id": "HV03", "capability": "Wizard and form hierarchy", "critical": false, "final": "pass" }, { "id": "HV04", "capability": "Tables, filters and status treatment", "critical": false, "final": "pass" }, { "id": "HV05", "capability": "Calendar visual encoding", "critical": false, "final": "pass" }, { "id": "HV06", "capability": "Reports, charts and settings layouts", "critical": false, "final": "fail" }, { "id": "HV07", "capability": "Light/dark palette and typography", "critical": false, "final": "pass" }, { "id": "HV08", "capability": "Cards, spacing and content density", "critical": false, "final": "fail" }, { "id": "HV09", "capability": "Dialogs, drawers and feedback", "critical": false, "final": "pass" }, { "id": "HV10", "capability": "Responsive correspondence", "critical": true, "final": "pass" } ], "timing": { "build": { "seconds": 8266.903, "lowerBoundSeconds": null, "observation": "upper-bound-only", "authoringPrompts": 16, "observedPrompts": 16 } }, "archived": true }, { "id": "run-10", "model": "Sonnet 5", "effort": "Medium", "replicate": 2, "matchedOriginalRunId": "run-01", "status": "accepted", "finalCost": { "status": "observed", "value": 3309, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 3, "corrections": 2, "clarifications": 0, "freeCostCommands": 1, "final": { "accepted": true, "functionalPassed": 18, "functionalTotal": 20, "visualPassed": 8, "visualTotal": 10, "allCriticalPassed": true }, "functionalChecks": [ { "id": "HF01", "capability": "Working managed app, complete role-aware scope and reference fixture", "critical": true, "final": "pass" }, { "id": "HF02", "capability": "Derived dashboards and balances", "critical": true, "final": "pass" }, { "id": "HF03", "capability": "My Leave filters, sorting and empty states", "critical": false, "final": "pass" }, { "id": "HF04", "capability": "Request detail and retained history", "critical": false, "final": "pass" }, { "id": "HF05", "capability": "Four-step request wizard", "critical": true, "final": "pass" }, { "id": "HF06", "capability": "Working-day and half-day arithmetic", "critical": true, "final": "pass" }, { "id": "HF07", "capability": "Blocking validation", "critical": true, "final": "pass" }, { "id": "HF08", "capability": "Warnings and policy-dependent validation", "critical": false, "final": "pass" }, { "id": "HF09", "capability": "Draft, edit and duplicate lifecycle", "critical": false, "final": "pass" }, { "id": "HF10", "capability": "Submission, approval routing and notifications", "critical": true, "final": "pass" }, { "id": "HF11", "capability": "Withdraw and cancel with balance reconciliation", "critical": false, "final": "pass" }, { "id": "HF12", "capability": "Manager decisions and reasons", "critical": true, "final": "pass" }, { "id": "HF13", "capability": "HR second-step and no-manager routing", "critical": true, "final": "pass" }, { "id": "HF14", "capability": "Bulk decisions, delegation and escalation", "critical": false, "final": "pass" }, { "id": "HF15", "capability": "Team calendars", "critical": false, "final": "pass" }, { "id": "HF16", "capability": "Scoped reports and actual CSV exports", "critical": false, "final": "fail" }, { "id": "HF17", "capability": "Live policy and pro-rata calculator", "critical": false, "final": "pass" }, { "id": "HF18", "capability": "Six working HR settings tabs", "critical": true, "final": "pass" }, { "id": "HF19", "capability": "Search, notifications, themes and keyboard interaction", "critical": false, "final": "fail" }, { "id": "HF20", "capability": "Faithful demo lifecycle and isolation", "critical": true, "final": "pass" } ], "visualChecks": [ { "id": "HV01", "capability": "Reference identity and shell", "critical": true, "final": "pass" }, { "id": "HV02", "capability": "Dashboard composition", "critical": false, "final": "pass" }, { "id": "HV03", "capability": "Wizard and form hierarchy", "critical": false, "final": "pass" }, { "id": "HV04", "capability": "Tables, filters and status treatment", "critical": false, "final": "pass" }, { "id": "HV05", "capability": "Calendar visual encoding", "critical": false, "final": "pass" }, { "id": "HV06", "capability": "Reports, charts and settings layouts", "critical": false, "final": "pass" }, { "id": "HV07", "capability": "Light/dark palette and typography", "critical": false, "final": "fail" }, { "id": "HV08", "capability": "Cards, spacing and content density", "critical": false, "final": "fail" }, { "id": "HV09", "capability": "Dialogs, drawers and feedback", "critical": false, "final": "pass" }, { "id": "HV10", "capability": "Responsive correspondence", "critical": true, "final": "pass" } ], "timing": { "build": { "seconds": 4992.702, "lowerBoundSeconds": 4962.343, "observation": "interval-bounded", "authoringPrompts": 3, "observedPrompts": 3 } }, "archived": true } ], "registeredSettings": [ { "runId": "run-06", "model": "Sonnet 5", "effort": "High", "status": "accepted", "finalCost": { "status": "observed", "value": 3153, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 6 }, { "runId": "run-07", "model": "Opus 5", "effort": "Medium", "status": "pending", "finalCost": null, "authoringPrompts": null }, { "runId": "run-08", "model": "GPT 5.5", "effort": "Medium", "status": "accepted", "finalCost": { "status": "observed", "value": 7464, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 16 }, { "runId": "run-09", "model": "Opus 5", "effort": "High", "status": "pending", "finalCost": null, "authoringPrompts": null }, { "runId": "run-10", "model": "Sonnet 5", "effort": "Medium", "status": "accepted", "finalCost": { "status": "observed", "value": 3309, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 3 } ], "meanFinalCredits": 4642, "complete": false, "scope": "Annual Leave Manager: role-aware leave requests, approvals, calendars, reports and six HR settings tabs; fictional in-memory data only.", "method": "Identical initial prompt and frozen HTML bytes in five fresh tasks. Independent native functional and visual review; targeted repairs in the same app. No arbitrary authoring-time or credit cutoff and no paid self-review.", "acceptanceNote": "The frozen minimum requires 17 of 20 functional groups and 8 of 10 visual groups, including every critical group. Passing that minimum does not mean complete parity; all remaining failed groups are shown.", "costNote": "Only cumulative task credits after full independent review and joint acceptance. No interim, repair-only or initial-output credit figures. Free cost commands do not count as authoring prompts.", "limitation": "One additional adaptive observation per original setting, in reverse order. Two observations remain exploratory; compare both delivered capability results. The separately guided replay is not pooled with these runs.", "privacy": "Only allowlisted measurements and capability outcomes are public. All repeat screenshots, transcripts, generated source and identifiers remain private. The original showcase permission is not extended.", "comparison": { "method": "matched-adaptive-repeat", "completePairs": 3, "complete": false, "pairs": [ { "model": "Sonnet 5", "effort": "Medium", "originalRunId": "run-01", "repeatRunId": "run-10", "observations": 2, "completePair": true, "credits": { "original": 2517, "repeat": 3309, "mean": 2913, "minimum": 2517, "maximum": 3309 }, "authoringPrompts": { "original": 8, "repeat": 3, "mean": 5.5, "minimum": 3, "maximum": 8 }, "capabilityChanges": { "functional": [ { "id": "HF14", "original": "fail", "repeat": "pass" } ], "visual": [ { "id": "HV06", "original": "fail", "repeat": "pass" }, { "id": "HV07", "original": "pass", "repeat": "fail" } ] } }, { "model": "Opus 5", "effort": "High", "originalRunId": "run-02", "repeatRunId": null, "observations": 1, "completePair": false, "credits": { "original": 3197, "repeat": null, "mean": null, "minimum": null, "maximum": null }, "authoringPrompts": { "original": 3, "repeat": null, "mean": null, "minimum": null, "maximum": null }, "capabilityChanges": { "functional": null, "visual": null } }, { "model": "GPT 5.5", "effort": "Medium", "originalRunId": "run-03", "repeatRunId": "run-08", "observations": 2, "completePair": true, "credits": { "original": 659, "repeat": 7464, "mean": 4061.5, "minimum": 659, "maximum": 7464 }, "authoringPrompts": { "original": 7, "repeat": 16, "mean": 11.5, "minimum": 7, "maximum": 16 }, "capabilityChanges": { "functional": [ { "id": "HF09", "original": "fail", "repeat": "pass" }, { "id": "HF14", "original": "fail", "repeat": "pass" }, { "id": "HF19", "original": "fail", "repeat": "pass" } ], "visual": [ { "id": "HV06", "original": "pass", "repeat": "fail" }, { "id": "HV07", "original": "fail", "repeat": "pass" } ] } }, { "model": "Opus 5", "effort": "Medium", "originalRunId": "run-04", "repeatRunId": null, "observations": 1, "completePair": false, "credits": { "original": 3220, "repeat": null, "mean": null, "minimum": null, "maximum": null }, "authoringPrompts": { "original": 2, "repeat": null, "mean": null, "minimum": null, "maximum": null }, "capabilityChanges": { "functional": null, "visual": null } }, { "model": "Sonnet 5", "effort": "High", "originalRunId": "run-05", "repeatRunId": "run-06", "observations": 2, "completePair": true, "credits": { "original": 2523, "repeat": 3153, "mean": 2838, "minimum": 2523, "maximum": 3153 }, "authoringPrompts": { "original": 3, "repeat": 6, "mean": 4.5, "minimum": 3, "maximum": 6 }, "capabilityChanges": { "functional": [ { "id": "HF04", "original": "pass", "repeat": "fail" }, { "id": "HF09", "original": "pass", "repeat": "fail" }, { "id": "HF15", "original": "fail", "repeat": "pass" } ], "visual": [ { "id": "HV05", "original": "fail", "repeat": "pass" }, { "id": "HV08", "original": "pass", "repeat": "fail" } ] } } ], "limitation": "Two observations per setting are exploratory, not a stable ranking or guaranteed budget. The acceptance target is fixed, but passed capability groups can differ; inspect both results. Guided replay is excluded." } }, "newModelSuite": { "id": "opus55-suite-20260922", "cohort": "new-model-adaptive", "model": "Opus 5.5", "effort": "Medium", "reviewMode": "external-only", "costMode": "complete-app-only", "registeredBuilds": 6, "reportedBuilds": 5, "reportingScope": { "includedTests": [ "test-05", "test-01", "test-02", "test-03" ], "omittedIncompleteTests": [ "test-04" ] }, "completedBuilds": 5, "registeredSettings": [ { "test": "test-05", "runId": "run-01", "model": "Opus 5.5", "effort": "Medium", "replicate": 1, "status": "accepted", "finalCost": { "status": "observed", "value": 365, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 2 }, { "test": "test-01", "runId": "run-02", "model": "Opus 5.5", "effort": "Medium", "replicate": 1, "status": "accepted", "finalCost": { "status": "observed", "value": 159, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 1 }, { "test": "test-02", "runId": "run-03", "model": "Opus 5.5", "effort": "Medium", "replicate": 1, "status": "accepted", "finalCost": { "status": "observed", "value": 610, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 3 }, { "test": "test-02", "runId": "run-04", "model": "Opus 5.5", "effort": "Medium", "replicate": 2, "status": "accepted", "finalCost": { "status": "observed", "value": 419, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 2 }, { "test": "test-03", "runId": "run-05", "model": "Opus 5.5", "effort": "Medium", "replicate": 1, "status": "accepted", "finalCost": { "status": "observed", "value": 920, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 4 } ], "cohorts": [ { "test": "test-05", "title": "HTML reference recreation", "runs": [ { "id": "run-01", "model": "Opus 5.5", "effort": "Medium", "replicate": 1, "status": "accepted", "finalCost": { "status": "observed", "value": 365, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 2, "corrections": 1, "clarifications": 0, "freeCostCommands": 1, "final": { "accepted": true, "functionalPassed": 20, "functionalTotal": 20, "visualPassed": 10, "visualTotal": 10, "allCriticalPassed": true }, "functionalChecks": [ { "id": "HF01", "capability": "Working managed app, complete role-aware scope and reference fixture", "critical": true, "final": "pass" }, { "id": "HF02", "capability": "Derived dashboards and balances", "critical": true, "final": "pass" }, { "id": "HF03", "capability": "My Leave filters, sorting and empty states", "critical": false, "final": "pass" }, { "id": "HF04", "capability": "Request detail and retained history", "critical": false, "final": "pass" }, { "id": "HF05", "capability": "Four-step request wizard", "critical": true, "final": "pass" }, { "id": "HF06", "capability": "Working-day and half-day arithmetic", "critical": true, "final": "pass" }, { "id": "HF07", "capability": "Blocking validation", "critical": true, "final": "pass" }, { "id": "HF08", "capability": "Warnings and policy-dependent validation", "critical": false, "final": "pass" }, { "id": "HF09", "capability": "Draft, edit and duplicate lifecycle", "critical": false, "final": "pass" }, { "id": "HF10", "capability": "Submission, approval routing and notifications", "critical": true, "final": "pass" }, { "id": "HF11", "capability": "Withdraw and cancel with balance reconciliation", "critical": false, "final": "pass" }, { "id": "HF12", "capability": "Manager decisions and reasons", "critical": true, "final": "pass" }, { "id": "HF13", "capability": "HR second-step and no-manager routing", "critical": true, "final": "pass" }, { "id": "HF14", "capability": "Bulk decisions, delegation and escalation", "critical": false, "final": "pass" }, { "id": "HF15", "capability": "Team calendars", "critical": false, "final": "pass" }, { "id": "HF16", "capability": "Scoped reports and actual CSV exports", "critical": false, "final": "pass" }, { "id": "HF17", "capability": "Live policy and pro-rata calculator", "critical": false, "final": "pass" }, { "id": "HF18", "capability": "Six working HR settings tabs", "critical": true, "final": "pass" }, { "id": "HF19", "capability": "Search, notifications, themes and keyboard interaction", "critical": false, "final": "pass" }, { "id": "HF20", "capability": "Faithful demo lifecycle and isolation", "critical": true, "final": "pass" } ], "visualChecks": [ { "id": "HV01", "capability": "Reference identity and shell", "critical": true, "final": "pass" }, { "id": "HV02", "capability": "Dashboard composition", "critical": false, "final": "pass" }, { "id": "HV03", "capability": "Wizard and form hierarchy", "critical": false, "final": "pass" }, { "id": "HV04", "capability": "Tables, filters and status treatment", "critical": false, "final": "pass" }, { "id": "HV05", "capability": "Calendar visual encoding", "critical": false, "final": "pass" }, { "id": "HV06", "capability": "Reports, charts and settings layouts", "critical": false, "final": "pass" }, { "id": "HV07", "capability": "Light/dark palette and typography", "critical": false, "final": "pass" }, { "id": "HV08", "capability": "Cards, spacing and content density", "critical": false, "final": "pass" }, { "id": "HV09", "capability": "Dialogs, drawers and feedback", "critical": false, "final": "pass" }, { "id": "HV10", "capability": "Responsive correspondence", "critical": true, "final": "pass" } ], "timing": { "build": { "seconds": 633.225, "lowerBoundSeconds": 608.26, "observation": "interval-bounded", "authoringPrompts": 2, "observedPrompts": 2 } }, "archived": true } ], "registeredBuilds": 1, "complete": true, "meanFinalCredits": 365, "strategy": "Reused the supplied reference screens and business logic. One targeted correction fixed inherited theme styles; final review verified all functional and visual groups.", "strategyCaveat": "Reuse was observed, not separately randomized. This does not isolate a model-only or reuse-only saving." }, { "test": "test-01", "title": "Virtual procurement", "runs": [ { "id": "run-02", "model": "Opus 5.5", "effort": "Medium", "replicate": 1, "status": "accepted", "finalCost": { "status": "observed", "value": 159, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 1, "corrections": 0, "clarifications": 0, "final": { "accepted": true, "functionalPassed": 20, "functionalTotal": 20, "allCriticalPassed": true }, "functionalChecks": [ { "id": "AC01", "capability": "Exact catalogue", "critical": false, "final": "pass" }, { "id": "AC02", "capability": "Search and empty state", "critical": false, "final": "pass" }, { "id": "AC03", "capability": "Combined filtering", "critical": false, "final": "pass" }, { "id": "AC04", "capability": "Unavailable stock", "critical": true, "final": "pass" }, { "id": "AC05", "capability": "Dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "AC06", "capability": "Valid request", "critical": true, "final": "pass" }, { "id": "AC07", "capability": "Required fields", "critical": true, "final": "pass" }, { "id": "AC08", "capability": "Quantity validation", "critical": true, "final": "pass" }, { "id": "AC09", "capability": "Date validation", "critical": false, "final": "pass" }, { "id": "AC10", "capability": "Approval", "critical": true, "final": "pass" }, { "id": "AC11", "capability": "Rejection", "critical": true, "final": "pass" }, { "id": "AC12", "capability": "Oversell prevention", "critical": true, "final": "pass" }, { "id": "AC13", "capability": "Single state transition", "critical": true, "final": "pass" }, { "id": "AC14", "capability": "Virtual/local reopen persistence", "critical": true, "final": "pass" }, { "id": "AC15", "capability": "Requester scoping", "critical": false, "final": "pass" }, { "id": "AC16", "capability": "Safe repeatable reset", "critical": false, "final": "pass" }, { "id": "AC17", "capability": "Keyboard and labels", "critical": false, "final": "pass" }, { "id": "AC18", "capability": "Responsive layout", "critical": false, "final": "pass" }, { "id": "AC19", "capability": "Honest demo boundary", "critical": false, "final": "pass" }, { "id": "AC20", "capability": "Actual private managed app", "critical": true, "final": "pass" } ], "timing": { "build": { "seconds": 361.767, "lowerBoundSeconds": 350.923, "observation": "interval-bounded", "authoringPrompts": 1, "observedPrompts": 1 } }, "archived": true, "reviewedRevisionScope": "owner-preview" } ], "registeredBuilds": 1, "complete": true, "meanFinalCredits": 159 }, { "test": "test-02", "title": "Live SharePoint requests", "runs": [ { "id": "run-03", "model": "Opus 5.5", "effort": "Medium", "replicate": 1, "status": "accepted", "finalCost": { "status": "observed", "value": 610, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 3, "corrections": 1, "clarifications": 1, "final": { "accepted": true, "functionalPassed": 10, "functionalTotal": 10, "allCriticalPassed": true }, "functionalChecks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "pass" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ], "timing": { "build": { "seconds": null, "status": "observer-recovery", "note": "Completion confirmation was delayed by the external observer; no model-speed comparison is reported." } }, "archived": true, "reviewedRevisionScope": "owner-preview" }, { "id": "run-04", "model": "Opus 5.5", "effort": "Medium", "replicate": 2, "status": "accepted", "finalCost": { "status": "observed", "value": 419, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 2, "corrections": 1, "clarifications": 0, "final": { "accepted": true, "functionalPassed": 10, "functionalTotal": 10, "allCriticalPassed": true }, "functionalChecks": [ { "id": "BC01", "capability": "Runnable private managed app", "critical": true, "final": "pass" }, { "id": "BC02", "capability": "Live catalogue connection", "critical": true, "final": "pass" }, { "id": "BC03", "capability": "Live dashboard arithmetic", "critical": false, "final": "pass" }, { "id": "BC04", "capability": "Catalogue search and filtering", "critical": false, "final": "pass" }, { "id": "BC05", "capability": "Complete saved request", "critical": true, "final": "pass" }, { "id": "BC06", "capability": "Basic request validation", "critical": true, "final": "pass" }, { "id": "BC07", "capability": "Live request listing", "critical": false, "final": "pass" }, { "id": "BC08", "capability": "Reopen persistence", "critical": true, "final": "pass" }, { "id": "BC09", "capability": "Honest data and error states", "critical": false, "final": "pass" }, { "id": "BC10", "capability": "Usable desktop and mobile build", "critical": false, "final": "pass" } ], "timing": { "build": { "seconds": 683.114, "lowerBoundSeconds": 656.306, "observation": "interval-bounded", "authoringPrompts": 2, "observedPrompts": 2 } }, "archived": true, "reviewedRevisionScope": "owner-preview" } ], "registeredBuilds": 2, "complete": true, "meanFinalCredits": 514.5 }, { "test": "test-03", "title": "Complex lifecycle", "runs": [ { "id": "run-05", "model": "Opus 5.5", "effort": "Medium", "replicate": 1, "status": "accepted", "finalCost": { "status": "observed", "value": 920, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 4, "corrections": 0, "clarifications": 0, "plannedStages": 4, "plannedChanges": 3, "firstCompleteTarget": { "authoringPrompts": 4, "functionalPassed": 20, "functionalTotal": 20, "cost": null }, "final": { "accepted": true, "functionalPassed": 20, "functionalTotal": 20, "allCriticalPassed": true }, "functionalChecks": [ { "id": "CX01", "capability": "Same private app opens with all seven navigable screens", "critical": true, "first": "pass", "final": "pass" }, { "id": "CX02", "capability": "Exact six-device/four-request relational seed and derived initial metrics", "critical": true, "first": "pass", "final": "pass" }, { "id": "CX03", "capability": "Catalogue search, combined category/stock filters and zero-stock protection", "critical": false, "first": "pass", "final": "pass" }, { "id": "CX04", "capability": "Multi-line draft creation, line edits/removal and accurate GBP totals", "critical": true, "first": "pass", "final": "pass" }, { "id": "CX05", "capability": "Required fields, dates, whole positive quantities, duplicates and stock validation", "critical": true, "first": "pass", "final": "pass" }, { "id": "CX06", "capability": "Draft/persona isolation, edits and exact-once submission with stable IDs", "critical": true, "first": "pass", "final": "pass" }, { "id": "CX07", "capability": "Controller-created draft, IDs, lines and timeline survive all three planned changes and reopen", "critical": true, "first": "pass", "final": "pass" }, { "id": "CX08", "capability": "Virtual manager/finance role gates and approval transitions at the final threshold", "critical": true, "first": "pass", "final": "pass" }, { "id": "CX09", "capability": "Rejection comments, append-only actor/timestamp history and terminal action guards", "critical": false, "first": "pass", "final": "pass" }, { "id": "CX10", "capability": "Request changes, requester edit/resubmit and cancellation preserve identity/history", "critical": false, "first": "pass", "final": "pass" }, { "id": "CX11", "capability": "Atomic fulfilment, stock revalidation, per-unit assets and duplicate-fulfilment protection", "critical": true, "first": "pass", "final": "pass" }, { "id": "CX12", "capability": "Good/damaged asset returns and duplicate-return protection", "critical": false, "first": "pass", "final": "pass" }, { "id": "CX13", "capability": "Current-user Office 365 Users profile actually reads through supported app binding", "critical": true, "first": "pass", "final": "pass" }, { "id": "CX14", "capability": "Profile loading/error/retry and genuinely missing optional fields are honest", "critical": false, "first": "pass", "final": "pass" }, { "id": "CX15", "capability": "Explicit light/dark mode applies across all screens and persists on reopen", "critical": true, "first": "pass", "final": "pass" }, { "id": "CX16", "capability": "Local preference Save/Cancel, density and default department work without directory writes", "critical": false, "first": "pass", "final": "pass" }, { "id": "CX17", "capability": "Cost-centre migration, persistence and department mismatch validation", "critical": false, "first": "pass", "final": "pass" }, { "id": "CX18", "capability": "Derived status/asset/stock metrics, low-stock navigation and combined request filters", "critical": false, "first": "pass", "final": "pass" }, { "id": "CX19", "capability": "All screens and key forms are usable at actual wide desktop and 390px mobile widths", "critical": true, "first": "pass", "final": "pass" }, { "id": "CX20", "capability": "Labels, visible keyboard focus, readable contrast and reachable form errors", "critical": false, "first": "pass", "final": "pass" } ], "timing": { "build": { "seconds": 1473.303, "lowerBoundSeconds": 1379.479, "observation": "interval-bounded", "authoringPrompts": 4, "observedPrompts": 4 } }, "archived": true, "reviewedRevisionScope": "owner-preview" } ], "registeredBuilds": 1, "complete": true, "meanFinalCredits": 920, "reviewCaveat": "All optional profile values were present, so genuinely missing-value behavior was not exercised. One native low-stock review justification differed only by a hyphen; all numeric, date, persona and workflow inputs matched. The frozen authoring prompts were unchanged." } ], "method": "Six fresh tasks were originally registered: Test 5 first, then Tests 1–4, with two SharePoint builds. The original frozen business inputs and scope-specific criteria were retained. Medium was the observed native default. Independent review, adaptive repairs in the same app and one actual completed-app cost. Reporting is limited to the requested scopes; omitted incomplete builds contribute no cost, prompt total or capability result.", "limitation": "New model version, not an Opus 5 repeat. No pooled cross-version or cross-scope average. One observation per scope, or two SharePoint observations when complete, is exploratory rather than a stable price or causal ranking.", "privacy": "Only allowlisted measurements and capability verdicts are public. New-version screenshots, transcripts, generated source and account/resource identifiers remain private." }, "testOneRepeatPair": { "test": "test-01", "cohort": "same-tenant-adaptive-pair", "model": "GPT 5.5", "effort": "Medium", "date": "2026-09-28", "sampleSize": 2, "complete": true, "reviewMode": "external-only", "costMode": "complete-app-only", "tenantScope": "One demo tenant", "runs": [ { "id": "run-01", "model": "GPT 5.5", "effort": "Medium", "replicate": 1, "status": "accepted", "final": { "points": 100, "accepted": true, "allCriticalPassed": true, "passed": 20, "total": 20 }, "finalCost": { "status": "observed", "value": 281, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 2, "corrections": 1, "clarifications": 0, "reviews": 0, "archived": true, "timing": { "build": { "seconds": 866.943, "lowerBoundSeconds": 832.146, "observation": "interval-bounded", "authoringPrompts": 2, "observedPrompts": 2 } }, "responsiveLayout": { "status": "pass", "overflowScenes": [] }, "firstOutput": { "cost": null, "passed": 0, "incomplete": 19 }, "checks": [ { "id": "AC01", "capability": "Exact catalogue", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC02", "capability": "Search and empty state", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC03", "capability": "Combined filtering", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC04", "capability": "Unavailable stock", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC05", "capability": "Dashboard arithmetic", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC06", "capability": "Valid request", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC07", "capability": "Required fields", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC08", "capability": "Quantity validation", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC09", "capability": "Date validation", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC10", "capability": "Approval", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC11", "capability": "Rejection", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC12", "capability": "Oversell prevention", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC13", "capability": "Single state transition", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC14", "capability": "Virtual/local reopen persistence", "critical": true, "first": "fail", "final": "pass" }, { "id": "AC15", "capability": "Requester scoping", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC16", "capability": "Safe repeatable reset", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC17", "capability": "Keyboard and labels", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC18", "capability": "Responsive layout", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC19", "capability": "Honest demo boundary", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC20", "capability": "Actual private managed app", "critical": true, "first": "blocked", "final": "pass" } ], "reviewedRevisionScope": "owner-preview", "latestRevisionRepublished": false }, { "id": "run-02", "model": "GPT 5.5", "effort": "Medium", "replicate": 2, "status": "accepted", "final": { "points": 95, "accepted": true, "allCriticalPassed": true, "passed": 19, "total": 20 }, "finalCost": { "status": "observed", "value": 221, "unit": "credits", "scope": "Cowork task" }, "authoringPrompts": 2, "corrections": 1, "clarifications": 0, "reviews": 0, "archived": true, "timing": { "build": { "seconds": 513.598, "lowerBoundSeconds": 466.893, "observation": "interval-bounded", "authoringPrompts": 2, "observedPrompts": 2 } }, "responsiveLayout": { "status": "fail", "overflowScenes": [ { "view": "Dashboard", "viewportWidth": 390, "contentWidth": 669 } ] }, "firstOutput": { "cost": null, "passed": 0, "incomplete": 19 }, "checks": [ { "id": "AC01", "capability": "Exact catalogue", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC02", "capability": "Search and empty state", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC03", "capability": "Combined filtering", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC04", "capability": "Unavailable stock", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC05", "capability": "Dashboard arithmetic", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC06", "capability": "Valid request", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC07", "capability": "Required fields", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC08", "capability": "Quantity validation", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC09", "capability": "Date validation", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC10", "capability": "Approval", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC11", "capability": "Rejection", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC12", "capability": "Oversell prevention", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC13", "capability": "Single state transition", "critical": true, "first": "blocked", "final": "pass" }, { "id": "AC14", "capability": "Virtual/local reopen persistence", "critical": true, "first": "fail", "final": "pass" }, { "id": "AC15", "capability": "Requester scoping", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC16", "capability": "Safe repeatable reset", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC17", "capability": "Keyboard and labels", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC18", "capability": "Responsive layout", "critical": false, "first": "blocked", "final": "fail" }, { "id": "AC19", "capability": "Honest demo boundary", "critical": false, "first": "blocked", "final": "pass" }, { "id": "AC20", "capability": "Actual private managed app", "critical": true, "first": "blocked", "final": "pass" } ], "reviewedRevisionScope": "owner-preview", "latestRevisionRepublished": false } ], "meanFinalCredits": 251, "minimumCredits": 221, "maximumCredits": 281, "rangeCredits": 60, "note": "Two fresh apps with the identical Test 1 business prompt and setting. Each final cumulative cost includes all guidance; no paid self-review or interim cost.", "limitation": "Separate from historical cohorts: review and persistence scope differ from original Test 1. Two observations do not establish stable pricing, a confidence interval or model superiority. Owner preview was reviewed; no separately verified deployment is claimed." } }