{
  "title": "ArchiLabs 10-task Revit AI evaluation scorecard",
  "publisher": "ArchiLabs",
  "version": "1.0",
  "date": "2026-11-04",
  "status": "Blank evaluation template; no product benchmark results.",
  "instructions": [
    "Record product/client/model versions, integration versions, permissions, input access and reviewer interventions.",
    "Use test copies. Only attempt writes after supported tools and explicit test scope are established.",
    "Score each applicable task: 0 incorrect/unsupported result, 1 requires material correction, 2 accepted against stated checks. Use unsupported, not_run, or not_applicable explicitly; do not turn unknown into a scored win.",
    "Record generation, correction and review effort separately; preserve native evidence and omissions.",
    "Do not compare a PDF-only run with full-model access as equivalent input conditions."
  ],
  "tasks": [
    {
      "task_id": 1,
      "task": "Known-element read",
      "inputs": "Three native element IDs in a named test document plus independently recorded Mark values",
      "acceptance": "Exact document/ID membership and values; unsupported fields explicit",
      "run_status": "not_run",
      "score": null,
      "tool_and_version": null,
      "evidence": [],
      "corrections": null,
      "review_minutes": null
    },
    {
      "task_id": 2,
      "task": "Schedule reconciliation",
      "inputs": "Itemized export with scope and units plus a register containing a duplicate key",
      "acceptance": "Duplicate ambiguity retained; explicit unit conversion; no row-count membership assumption",
      "run_status": "not_run",
      "score": null,
      "tool_and_version": null,
      "evidence": [],
      "corrections": null,
      "review_minutes": null
    },
    {
      "task_id": 3,
      "task": "Instance value update",
      "inputs": "Approved change to one verified writable instance field in a test copy",
      "acceptance": "Only approved instances/field changed; native comparison verifies exclusions",
      "run_status": "not_run",
      "score": null,
      "tool_and_version": null,
      "evidence": [],
      "corrections": null,
      "review_minutes": null
    },
    {
      "task_id": 4,
      "task": "Type change scope",
      "inputs": "Approved hypothetical type-field change and known instances beyond initial selection",
      "acceptance": "Full affected-instance scope assessed before any write",
      "run_status": "not_run",
      "score": null,
      "tool_and_version": null,
      "evidence": [],
      "corrections": null,
      "review_minutes": null
    },
    {
      "task_id": 5,
      "task": "View naming",
      "inputs": "Views with prefix rules, collision cases and blank Title on Sheet",
      "acceptance": "Proposed collisions and printed-title effects surfaced before rename",
      "run_status": "not_run",
      "score": null,
      "tool_and_version": null,
      "evidence": [],
      "corrections": null,
      "review_minutes": null
    },
    {
      "task_id": 6,
      "task": "Sheet register",
      "inputs": "Issued sheet index with unreadable and repeated pages",
      "acceptance": "Unreadable not labeled absent; PDF packaging distinct from model action",
      "run_status": "not_run",
      "score": null,
      "tool_and_version": null,
      "evidence": [],
      "corrections": null,
      "review_minutes": null
    },
    {
      "task_id": 7,
      "task": "Annotation review",
      "inputs": "Bounded model change with expected affected tags/views",
      "acceptance": "Native verification of relevant annotations; unresolved dependencies identified",
      "run_status": "not_run",
      "score": null,
      "tool_and_version": null,
      "evidence": [],
      "corrections": null,
      "review_minutes": null
    },
    {
      "task_id": 8,
      "task": "PDF source question",
      "inputs": "Manufacturer source pages with one missing value",
      "acceptance": "Exact evidence references; missing value remains unknown",
      "run_status": "not_run",
      "score": null,
      "tool_and_version": null,
      "evidence": [],
      "corrections": null,
      "review_minutes": null
    },
    {
      "task_id": 9,
      "task": "Decision handoff",
      "inputs": "Approved decision plus a newer unapproved suggestion",
      "acceptance": "Authority/conditions preserved; latest message does not override decision",
      "run_status": "not_run",
      "score": null,
      "tool_and_version": null,
      "evidence": [],
      "corrections": null,
      "review_minutes": null
    },
    {
      "task_id": 10,
      "task": "Native deliverable",
      "inputs": "Agreed supported edit in a test copy plus original baseline",
      "acceptance": "Expected artifact opens in native app; values and scope verified beyond screenshot",
      "run_status": "not_run",
      "score": null,
      "tool_and_version": null,
      "evidence": [],
      "corrections": null,
      "review_minutes": null
    }
  ]
}