{ "schema_version": "forge.evidence-research-pack/v1", "cut_at": "2026-07-17T23:24:33Z", "methodology": { "mode": "zero-copy-human-synthesis", "raw_persisted": false, "semantic_groundedness": "human-reviewed-structural", "minimum_supporting_sources": 2, "minimum_independence_groups_per_research": 3, "reviewed_by": "codex-SQ117", "limitations": "Receipts prove acquisition, policy checks and source binding; semantic support remains an explicit human-reviewed mapping, not an automated truth claim." }, "researches": [ { "id": "R1-RELIABLE-HARNESS", "question": "Which public practices are necessary for a reliable agentic engineering harness?", "status": "reproducible", "method": "Triangulate primary harness, sandbox, delivery, orchestration and decision-record guidance; require independent support for each claim.", "source_ids": [ "OPENAI-SYMPHONY-SPEC", "OPENAI-CODEX-SANDBOX", "DORA-2025", "GOOGLE-SMALL-CLS", "AUTOGEN-AGENTCHAT", "AWS-ADR", "AZURE-ADR" ], "claims": [ { "id": "R1-C1", "statement": "Reliable agentic delivery is a system property: the workflow, workspace lifecycle, feedback loop and organizational platform must be explicit rather than left inside a model prompt.", "support": [ { "source_id": "OPENAI-SYMPHONY-SPEC", "location_hint": "Core model; workspace and orchestration sections", "evidence_note": "The specification binds tracked work to isolated workspaces and an explicit orchestration lifecycle.", "relation": "supports" }, { "source_id": "DORA-2025", "location_hint": "AI capabilities and organizational system findings", "evidence_note": "DORA treats AI as an amplifier whose outcome depends on the surrounding delivery system and capabilities.", "relation": "supports" } ], "confidence": "high", "limitations": "Public descriptions do not expose internal reliability distributions or comparable production SLOs." }, { "id": "R1-C2", "statement": "Autonomy should be bounded by isolation and small reviewable changes; a worktree alone is not a security boundary and a large change weakens review quality.", "support": [ { "source_id": "OPENAI-CODEX-SANDBOX", "location_hint": "Linux sandbox architecture and syscall restrictions", "evidence_note": "Codex uses a dedicated Linux sandbox and kernel controls rather than treating source-control isolation as containment.", "relation": "supports" }, { "source_id": "GOOGLE-SMALL-CLS", "location_hint": "Why write small CLs", "evidence_note": "Google links small changes to faster, more effective and less error-prone review.", "relation": "supports" } ], "confidence": "high", "limitations": "Change-size guidance is human-review guidance; it does not by itself define an agent sandbox policy." }, { "id": "R1-C3", "statement": "Multi-agent teams and evolving architecture require explicit orchestration roles plus durable decision records with status, consequences and supersession.", "support": [ { "source_id": "AUTOGEN-AGENTCHAT", "location_hint": "AgentChat overview and team abstractions", "evidence_note": "AutoGen exposes agents and teams as explicit orchestration primitives rather than an implicit prompt convention.", "relation": "supports" }, { "source_id": "AWS-ADR", "location_hint": "ADR process", "evidence_note": "AWS guidance records architectural context, decision and consequences as a durable engineering artifact.", "relation": "supports" }, { "source_id": "AZURE-ADR", "location_hint": "Architecture decision record guidance", "evidence_note": "Azure guidance treats decisions as versioned records whose state and replacement remain visible over time.", "relation": "supports" } ], "confidence": "high", "limitations": "The sources do not prescribe one universal team topology or ADR template." } ], "verdict": "FORGE should retain spec-driven worktrees, explicit agent roles and ADRs, while treating sandbox, feedback and bounded changes as enforceable harness controls.", "limitations": "This research establishes a public design baseline; operational superiority still requires product observations and stable-window metrics." }, { "id": "R2-SECURE-AUDITABLE-DELIVERY", "question": "What is the minimum public control set for secure and auditable agentic delivery?", "status": "reproducible", "method": "Cross-check AI measurement and verification guidance with supply-chain provenance, attestations and execution isolation.", "source_ids": [ "NIST-DIOPTRA-AI-RMF", "OWASP-AISVS", "SLSA-1.2", "GITHUB-ATTESTATIONS", "OPENAI-CODEX-SANDBOX" ], "claims": [ { "id": "R2-C1", "statement": "AI assurance needs reproducible tests and measurable evidence mapped to explicit verification requirements; policy prose alone is insufficient.", "support": [ { "source_id": "NIST-DIOPTRA-AI-RMF", "location_hint": "Project purpose and AI RMF Measure relationship", "evidence_note": "NIST Dioptra operationalizes trustworthy-AI measurement through managed, repeatable experiments and tracked results.", "relation": "supports" }, { "source_id": "OWASP-AISVS", "location_hint": "Purpose, levels and verification requirements", "evidence_note": "AISVS frames AI security as testable implementation requirements rather than awareness-only risk labels.", "relation": "supports" } ], "confidence": "high", "limitations": "Dioptra and AISVS cover different scopes; neither is a turnkey certification of a FORGE deployment." }, { "id": "R2-C2", "statement": "Auditable delivery must bind an artifact to its build provenance and allow independent verification; a successful CI status without source binding is weaker evidence.", "support": [ { "source_id": "SLSA-1.2", "location_hint": "Provenance and build track requirements", "evidence_note": "SLSA defines verifiable provenance and graduated supply-chain guarantees for produced artifacts.", "relation": "supports" }, { "source_id": "GITHUB-ATTESTATIONS", "location_hint": "Artifact attestations overview", "evidence_note": "GitHub attestations cryptographically bind provenance claims to built artifacts and support verification.", "relation": "supports" } ], "confidence": "high", "limitations": "An attestation proves the declared provenance chain, not the semantic correctness of the artifact." }, { "id": "R2-C3", "statement": "Least privilege, isolated execution and supply-chain provenance are complementary controls; none substitutes for the others in an agentic system.", "support": [ { "source_id": "OPENAI-CODEX-SANDBOX", "location_hint": "Sandbox threat boundary", "evidence_note": "The Codex sandbox limits the execution environment and reachable system surface.", "relation": "supports" }, { "source_id": "OWASP-AISVS", "location_hint": "Agentic AI security scope", "evidence_note": "AISVS includes agent-specific verification concerns beyond process or artifact provenance alone.", "relation": "supports" }, { "source_id": "SLSA-1.2", "location_hint": "Supply-chain threats and provenance", "evidence_note": "SLSA addresses build and provenance risks that runtime isolation does not cover.", "relation": "supports" } ], "confidence": "high", "limitations": "The exact control profile remains environment-specific and must be threat-modeled per product." } ], "verdict": "FORGE needs deny-by-default execution, measurable AI verification, source-bound receipts and independently verifiable artifact provenance as separate gates.", "limitations": "The selected public standards are authoritative design inputs, not proof that every FORGE product has operated each control." }, { "id": "R3-EVALS-OBSERVABILITY-UNIT-ECONOMICS", "question": "How should evals, observability and unit economics be combined without inventing cost or quality evidence?", "status": "reproducible", "method": "Map evaluation, tracing, telemetry, unit-cost and official provider-billing sources; separate measured intake cost from unconfigured provider reconciliation.", "source_ids": [ "ANTHROPIC-EVAL-COOKBOOK", "OPENAI-AGENTS-TRACING", "OTEL-GENAI-OBSERVABILITY", "FINOPS-UNIT-COST", "GITHUB-BILLING-USAGE-API", "GITHUB-BILLING-REPORTS", "OPENAI-COST-API", "ANTHROPIC-USAGE-COST-API", "GOOGLE-CLOUD-BILLING-EXPORT" ], "claims": [ { "id": "R3-C1", "statement": "An AI quality claim requires explicit eval criteria and observable execution evidence; traces alone and subjective inspection alone are incomplete.", "support": [ { "source_id": "ANTHROPIC-EVAL-COOKBOOK", "location_hint": "Summarization evaluation metrics and test configuration", "evidence_note": "The cookbook combines defined evaluation criteria and repeatable tests for generated summaries.", "relation": "supports" }, { "source_id": "OPENAI-AGENTS-TRACING", "location_hint": "Tracing spans and run instrumentation", "evidence_note": "The Agents SDK records structured execution traces that make agent runs observable and diagnosable.", "relation": "supports" } ], "confidence": "high", "limitations": "Neither source defines FORGE-specific pass thresholds; baselines must be calibrated per use case." }, { "id": "R3-C2", "statement": "Useful AI observability must connect technical telemetry to a unit of value and its cost, instead of optimizing aggregate spend without an outcome denominator.", "support": [ { "source_id": "OTEL-GENAI-OBSERVABILITY", "location_hint": "GenAI metrics, traces and dashboards", "evidence_note": "OpenTelemetry guidance connects GenAI execution signals such as spans and metrics to operational analysis.", "relation": "supports" }, { "source_id": "FINOPS-UNIT-COST", "location_hint": "Measure unit costs capability", "evidence_note": "FinOps defines unit cost as the relationship between spend and a business or engineering value unit.", "relation": "supports" } ], "confidence": "high", "limitations": "A unit must be product-specific; this framework-only run measures intake cost but not customer outcome." }, { "id": "R3-C3", "statement": "Financial reconciliation should use official provider usage or cost interfaces and preserve unknown as unknown; token estimates are not an invoice substitute.", "support": [ { "source_id": "OPENAI-COST-API", "location_hint": "Admin organization usage costs resource", "evidence_note": "The official OpenAI SDK exposes an organization costs endpoint for provider-side cost records.", "relation": "supports" }, { "source_id": "ANTHROPIC-USAGE-COST-API", "location_hint": "Usage and cost report workflow", "evidence_note": "The official Anthropic cookbook queries organization usage and cost reports through the provider API.", "relation": "supports" }, { "source_id": "GOOGLE-CLOUD-BILLING-EXPORT", "location_hint": "Requirements and billing export workflow", "evidence_note": "The official Google Cloud blueprint consumes standard or detailed usage-cost exports for billing analysis.", "relation": "qualifies" }, { "source_id": "GITHUB-BILLING-REPORTS", "location_hint": "Billing reports and usage export", "evidence_note": "GitHub documentation distinguishes provider-generated billing reports from locally inferred usage.", "relation": "supports" } ], "confidence": "high", "limitations": "No provider admin credential or nonzero model call was used in this framework-only run, so live model spend remains not configured rather than zero." } ], "verdict": "FORGE should gate quality with evals plus traces, measure cost per outcome, and reconcile nonzero provider spend only from official billing sources.", "limitations": "The evidence intake itself is no-LLM and measured at zero model cost; human time and future provider spend are outside this receipt set." } ], "billing": [ { "provider": "openai", "source_ids": ["OPENAI-COST-API"], "official_mechanism": "Organization Admin Usage Costs API exposed by the official OpenAI SDK.", "operational_status": "documented-not-configured", "limitation": "This intake made no OpenAI model call and no organization admin credential was authorized; nonzero spend must be reconciled later, not recorded as zero." }, { "provider": "anthropic", "source_ids": ["ANTHROPIC-USAGE-COST-API"], "official_mechanism": "Organization Usage and Cost API cost report demonstrated by the official Anthropic cookbook.", "operational_status": "documented-not-configured", "limitation": "This intake made no Anthropic model call and no admin API credential was authorized; live organization cost remains unavailable." }, { "provider": "google", "source_ids": ["GOOGLE-CLOUD-BILLING-EXPORT"], "official_mechanism": "Google Cloud standard or detailed usage-cost export to BigQuery, consumed by the official billing-dashboard blueprint; model cost must be scoped to project and SKU.", "operational_status": "documented-not-configured", "limitation": "This intake made no Gemini model call and no billing-account access was authorized; a future export must preserve SKU and project attribution." } ], "aggregate": { "source_count": 20, "independence_group_count": 11, "receipt_count": 20, "bytes_received": 423265, "elapsed_ms": 37675, "http_calls": 21, "model_calls": 0, "model_tokens": 0, "cost_usd": 0, "research_count": 3, "claim_count": 9 } }