Merge pull request #1271 from AirHua-byte/codex/add-offline-harness-verifier

feat(evals): add offline harness artifact verifier

agent/evals/__init__.py

@@ -0,0 +1 @@
+"""Offline evaluation utilities for Vibe-Trading."""

agent/evals/harness/README.md

@@ -0,0 +1,31 @@
+# Harness artifact verifier
+
+This package checks a completed agent run from persisted files. It is deterministic and read-only: it does not call an LLM, invoke a tool, load market data, or rewrite the case prompt.
+
+Run it from an installed or editable checkout:
+
+```bash
+python -m evals.harness.runner \
+  --case agent/evals/harness/cases/identity/explicit_a_share.json \
+  --run-dir /path/to/run \
+  --trace-dir /path/to/session
+```
+
+`--trace-dir` is optional when `trace.jsonl` and `run_manifest.json` are stored in the run directory. The JSON report emits one `PASS`, `FAIL`, `NOT_EVALUABLE`, or `INVALID_ARTIFACT` verdict per assertion. Missing future instrumentation is never treated as success.
+
+Session traces can contain several turns. The loader evaluates only the records from the final `start` event onward, which matches the latest run artifacts. A future immutable message ID can replace this bounded association without changing old reports.
+
+Exit status is `0` when all evaluable assertions pass, `1` when an assertion fails, and `2` when the case or artifact bundle is invalid. `NOT_EVALUABLE` does not fail the command because the report preserves its coverage explicitly.
+
+The first version reads these files only:
+
+- `req.json`
+- `state.json`
+- `trace.jsonl` and safe prompt/content sidecars
+- `artifacts/grounding_evidence.json`
+- `llm_usage.json`
+- `run_manifest.json`
+
+Cases keep the natural-language prompt unchanged. Everything below `expect` is evaluator metadata and is never sent to the Agent.
+
+`data_snapshot` is a relative pointer for the separate run producer. This verifier never loads or fetches that data; the included example points to a synthetic, credential-free snapshot packaged beside the case.

agent/evals/harness/__init__.py

@@ -0,0 +1,10 @@
+"""Deterministic post-run verification for agent artifact bundles."""
+
+from .schema import EvaluationReport, HarnessCase, Verdict, VerdictStatus
+
+__all__ = [
+    "EvaluationReport",
+    "HarnessCase",
+    "Verdict",
+    "VerdictStatus",
+]

agent/evals/harness/artifacts.py

@@ -0,0 +1,174 @@
+"""Safe, read-only loading of persisted agent run artifacts."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+JSON_LIMIT_BYTES = 16 * 1024 * 1024
+TRACE_LIMIT_BYTES = 64 * 1024 * 1024
+
+
+@dataclass(frozen=True)
+class ArtifactBundle:
+    """Known artifacts and load errors for one completed run."""
+
+    run_dir: Path
+    trace_dir: Path
+    req: dict[str, Any] | None
+    state: dict[str, Any] | None
+    grounding: dict[str, Any] | None
+    llm_usage: dict[str, Any] | None
+    manifest: dict[str, Any] | None
+    trace: tuple[dict[str, Any], ...]
+    present: frozenset[str]
+    errors: dict[str, str] = field(default_factory=dict)
+
+    @classmethod
+    def load(cls, run_dir: Path, *, trace_dir: Path | None = None) -> "ArtifactBundle":
+        """Load only fixed artifact names without following paths outside their roots."""
+        run_root = _require_directory(run_dir, "run_dir")
+        trace_root = _require_directory(trace_dir or run_root, "trace_dir")
+        present: set[str] = set()
+        errors: dict[str, str] = {}
+
+        def load_object(root: Path, relative: str, key: str) -> dict[str, Any] | None:
+            try:
+                path = _safe_known_path(root, relative)
+            except ValueError as exc:
+                errors[key] = str(exc)
+                return None
+            if not path.exists():
+                return None
+            present.add(key)
+            try:
+                value = _read_json(path)
+            except (
+                OSError,
+                UnicodeDecodeError,
+                json.JSONDecodeError,
+                ValueError,
+            ) as exc:
+                errors[key] = str(exc)
+                return None
+            if not isinstance(value, dict):
+                errors[key] = f"{relative} must contain a JSON object"
+                return None
+            return value
+
+        req = load_object(run_root, "req.json", "req")
+        state = load_object(run_root, "state.json", "state")
+        grounding = load_object(
+            run_root, "artifacts/grounding_evidence.json", "grounding"
+        )
+        llm_usage = load_object(run_root, "llm_usage.json", "llm_usage")
+        manifest = load_object(trace_root, "run_manifest.json", "manifest")
+        trace = _load_trace(trace_root, present, errors)
+        return cls(
+            run_dir=run_root,
+            trace_dir=trace_root,
+            req=req,
+            state=state,
+            grounding=grounding,
+            llm_usage=llm_usage,
+            manifest=manifest,
+            trace=trace,
+            present=frozenset(present),
+            errors=errors,
+        )
+
+
+def _require_directory(path: Path, label: str) -> Path:
+    root = Path(path).resolve()
+    if not root.is_dir():
+        raise ValueError(f"{label} is not a directory: {root}")
+    return root
+
+
+def _safe_known_path(root: Path, relative: str) -> Path:
+    path = (root / relative).resolve()
+    try:
+        path.relative_to(root)
+    except ValueError as exc:
+        raise ValueError(f"artifact path escaped {root}: {relative}") from exc
+    return path
+
+
+def _read_json(path: Path) -> Any:
+    if path.stat().st_size > JSON_LIMIT_BYTES:
+        raise ValueError(
+            f"{path.name} exceeds the {JSON_LIMIT_BYTES}-byte safety limit"
+        )
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _load_trace(
+    trace_root: Path,
+    present: set[str],
+    errors: dict[str, str],
+) -> tuple[dict[str, Any], ...]:
+    try:
+        path = _safe_known_path(trace_root, "trace.jsonl")
+    except ValueError as exc:
+        errors["trace"] = str(exc)
+        return ()
+    if not path.exists():
+        return ()
+    present.add("trace")
+    try:
+        if path.stat().st_size > TRACE_LIMIT_BYTES:
+            raise ValueError(
+                f"trace.jsonl exceeds the {TRACE_LIMIT_BYTES}-byte safety limit"
+            )
+        records: list[dict[str, Any]] = []
+        for number, raw_line in enumerate(
+            path.read_text(encoding="utf-8").splitlines(), start=1
+        ):
+            if not raw_line.strip():
+                continue
+            try:
+                record = json.loads(raw_line)
+            except json.JSONDecodeError as exc:
+                raise ValueError(
+                    f"trace.jsonl:{number}: invalid JSON: {exc.msg}"
+                ) from exc
+            if not isinstance(record, dict):
+                raise ValueError(f"trace.jsonl:{number}: record must be an object")
+            record = dict(record)
+            record["_line"] = number
+            _resolve_trace_text(trace_root, record, "prompt")
+            _resolve_trace_text(trace_root, record, "content")
+            records.append(record)
+        starts = [
+            index
+            for index, record in enumerate(records)
+            if record.get("type") == "start"
+        ]
+        if starts:
+            records = records[starts[-1] :]
+        return tuple(records)
+    except (OSError, UnicodeDecodeError, ValueError) as exc:
+        errors["trace"] = str(exc)
+        return ()
+
+
+def _resolve_trace_text(
+    trace_root: Path, record: dict[str, Any], field_name: str
+) -> None:
+    if field_name in record or f"{field_name}_path" not in record:
+        return
+    relative = record[f"{field_name}_path"]
+    if not isinstance(relative, str):
+        raise ValueError(
+            f"trace.jsonl:{record['_line']}: {field_name}_path must be a string"
+        )
+    path = _safe_known_path(trace_root, relative)
+    if not path.is_file():
+        raise ValueError(f"trace.jsonl:{record['_line']}: missing {field_name} sidecar")
+    if path.stat().st_size > JSON_LIMIT_BYTES:
+        raise ValueError(
+            f"trace.jsonl:{record['_line']}: {field_name} sidecar exceeds safety limit"
+        )
+    record[field_name] = path.read_text(encoding="utf-8")

agent/evals/harness/assertions.py

@@ -0,0 +1,777 @@
+"""Deterministic assertions over a Harness case and persisted artifacts."""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any, Mapping
+
+from .artifacts import ArtifactBundle
+from .methodology import manifest_verdict
+from .observations import (
+    contains as _contains,
+    mapping as _mapping,
+    string_list as _string_list,
+    successful_tool_records as _successful_tool_records,
+    tool_spec as _tool_spec,
+    usage_error as _usage_error,
+    within_limit as _within_limit,
+)
+from .schema import HarnessCase, Verdict, VerdictStatus
+
+
+def evaluate_assertions(
+    case: HarnessCase, bundle: ArtifactBundle
+) -> tuple[Verdict, ...]:
+    """Evaluate every supported contract section in a stable order."""
+    verdicts: list[Verdict] = []
+    verdicts.extend(_prompt_verdicts(case, bundle))
+    verdicts.append(_execution_verdict(case, bundle))
+    verdicts.extend(_future_state_verdicts(case, bundle))
+    verdicts.extend(_tool_verdicts(case, bundle))
+    verdicts.append(_tool_result_join_verdict(bundle))
+    verdicts.extend(_identity_verdicts(case, bundle))
+    verdicts.extend(_evidence_verdicts(case, bundle))
+    verdicts.extend(_budget_verdicts(case, bundle))
+    verdicts.extend(_risk_verdicts(case, bundle))
+    verdicts.append(manifest_verdict(bundle))
+    return tuple(verdicts)
+
+
+def _verdict(
+    status: VerdictStatus,
+    code: str,
+    expected: Any,
+    observed: Any,
+    *refs: str,
+    detail: str | None = None,
+) -> Verdict:
+    return Verdict(status, code, expected, observed, tuple(refs), detail)
+
+
+def _artifact_problem(
+    bundle: ArtifactBundle, key: str, *, missing_invalid: bool
+) -> VerdictStatus | None:
+    if key in bundle.errors:
+        return VerdictStatus.INVALID_ARTIFACT
+    if key not in bundle.present:
+        return (
+            VerdictStatus.INVALID_ARTIFACT
+            if missing_invalid
+            else VerdictStatus.NOT_EVALUABLE
+        )
+    return None
+
+
+def _prompt_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+    problem = _artifact_problem(bundle, "req", missing_invalid=True)
+    if problem:
+        return [
+            _verdict(
+                problem,
+                "prompt.request_preserved",
+                case.prompt,
+                bundle.errors.get("req"),
+                "req.json",
+                detail="req.json is required to prove the original request",
+            )
+        ]
+    observed = bundle.req.get("prompt") if bundle.req else None
+    req_status = VerdictStatus.PASS if observed == case.prompt else VerdictStatus.FAIL
+    verdicts = [
+        _verdict(
+            req_status,
+            "prompt.request_preserved",
+            case.prompt,
+            observed,
+            "req.json#/prompt",
+        )
+    ]
+
+    trace_problem = _artifact_problem(bundle, "trace", missing_invalid=True)
+    if trace_problem:
+        verdicts.append(
+            _verdict(
+                trace_problem,
+                "prompt.trace_preserved",
+                case.prompt,
+                bundle.errors.get("trace"),
+                "trace.jsonl",
+            )
+        )
+        return verdicts
+    user_record = next(
+        (
+            record
+            for record in bundle.trace
+            if record.get("type") == "message" and record.get("role") == "user"
+        ),
+        None,
+    )
+    if user_record is None:
+        verdicts.append(
+            _verdict(
+                VerdictStatus.NOT_EVALUABLE,
+                "prompt.trace_preserved",
+                case.prompt,
+                None,
+                "trace.jsonl",
+                detail="this trace has no user message record",
+            )
+        )
+    else:
+        trace_prompt = user_record.get("content")
+        status = (
+            VerdictStatus.PASS if trace_prompt == case.prompt else VerdictStatus.FAIL
+        )
+        verdicts.append(
+            _verdict(
+                status,
+                "prompt.trace_preserved",
+                case.prompt,
+                trace_prompt,
+                f"trace.jsonl:{user_record['_line']}",
+            )
+        )
+    return verdicts
+
+
+def _execution_verdict(case: HarnessCase, bundle: ArtifactBundle) -> Verdict:
+    expectation = _mapping(case.expect.get("execution"))
+    allowed = expectation.get("allowed")
+    state_problem = _artifact_problem(bundle, "state", missing_invalid=True)
+    trace_problem = _artifact_problem(bundle, "trace", missing_invalid=True)
+    if state_problem or trace_problem:
+        errors = {
+            key: bundle.errors.get(key, "missing")
+            for key in ("state", "trace")
+            if key not in bundle.present or key in bundle.errors
+        }
+        return _verdict(
+            VerdictStatus.INVALID_ARTIFACT,
+            "execution.reconciled_status",
+            allowed,
+            errors,
+            "state.json",
+            "trace.jsonl",
+        )
+    state_status = bundle.state.get("status") if bundle.state else None
+    end_records = [record for record in bundle.trace if record.get("type") == "end"]
+    raw_end_statuses = [record.get("status") for record in end_records]
+    end_statuses = [_execution_status(item) for item in raw_end_statuses]
+    observed = {
+        "state": state_status,
+        "trace_end": raw_end_statuses,
+        "normalized_trace_end": end_statuses,
+    }
+    if (
+        not isinstance(state_status, str)
+        or not end_records
+        or any(not isinstance(item, str) for item in end_statuses)
+    ):
+        return _verdict(
+            VerdictStatus.INVALID_ARTIFACT,
+            "execution.reconciled_status",
+            allowed,
+            observed,
+            "state.json#/status",
+            "trace.jsonl",
+        )
+    if any(item != state_status for item in end_statuses):
+        return _verdict(
+            VerdictStatus.INVALID_ARTIFACT,
+            "execution.reconciled_status",
+            allowed,
+            observed,
+            "state.json#/status",
+            *(f"trace.jsonl:{record['_line']}" for record in end_records),
+            detail="terminal artifacts disagree",
+        )
+    if not isinstance(allowed, list) or not all(
+        isinstance(item, str) for item in allowed
+    ):
+        return _verdict(
+            VerdictStatus.FAIL,
+            "case.execution_allowed_invalid",
+            "string array",
+            allowed,
+        )
+    status = VerdictStatus.PASS if state_status in allowed else VerdictStatus.FAIL
+    return _verdict(
+        status,
+        "execution.reconciled_status",
+        allowed,
+        state_status,
+        "state.json#/status",
+        f"trace.jsonl:{end_records[-1]['_line']}",
+    )
+
+
+def _future_state_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+    verdicts: list[Verdict] = []
+    for key in ("completion", "safety"):
+        expectation = _mapping(case.expect.get(key))
+        if not expectation:
+            continue
+        allowed = expectation.get("allowed")
+        field_name = f"{key}_status"
+        state_problem = _artifact_problem(bundle, "state", missing_invalid=True)
+        if state_problem:
+            verdicts.append(
+                _verdict(
+                    state_problem,
+                    f"{key}.status",
+                    allowed,
+                    bundle.errors.get("state", "missing"),
+                    "state.json",
+                )
+            )
+            continue
+        observed = bundle.state.get(field_name) if bundle.state else None
+        if observed is None:
+            verdicts.append(
+                _verdict(
+                    VerdictStatus.NOT_EVALUABLE,
+                    f"{key}.status",
+                    allowed,
+                    None,
+                    f"state.json#/{field_name}",
+                    detail=f"{field_name} is not instrumented in this run",
+                )
+            )
+        else:
+            status = (
+                VerdictStatus.PASS
+                if isinstance(allowed, list) and observed in allowed
+                else VerdictStatus.FAIL
+            )
+            verdicts.append(
+                _verdict(
+                    status,
+                    f"{key}.status",
+                    allowed,
+                    observed,
+                    f"state.json#/{field_name}",
+                )
+            )
+    return verdicts
+
+
+def _tool_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+    expectation = _mapping(case.expect.get("tools"))
+    if not expectation:
+        return []
+    problem = _artifact_problem(bundle, "trace", missing_invalid=True)
+    if problem:
+        return [
+            _verdict(
+                problem,
+                "tools.trace_available",
+                expectation,
+                bundle.errors.get("trace"),
+                "trace.jsonl",
+            )
+        ]
+    calls = [record for record in bundle.trace if record.get("type") == "tool_call"]
+    verdicts: list[Verdict] = []
+    for index, required in enumerate(expectation.get("required", [])):
+        spec = _tool_spec(required)
+        if spec is None:
+            verdicts.append(
+                _verdict(
+                    VerdictStatus.FAIL,
+                    "case.tool_requirement_invalid",
+                    "tool spec",
+                    required,
+                )
+            )
+            continue
+        name, minimum, arguments = spec
+        matches = [
+            record
+            for record in calls
+            if record.get("tool") == name and _contains(record.get("args"), arguments)
+        ]
+        verdicts.append(
+            _verdict(
+                VerdictStatus.PASS if len(matches) >= minimum else VerdictStatus.FAIL,
+                f"tools.required.{index}",
+                required,
+                {"matching_calls": len(matches)},
+                *(f"trace.jsonl:{record['_line']}" for record in matches),
+            )
+        )
+    for index, forbidden in enumerate(expectation.get("forbidden", [])):
+        spec = _tool_spec(forbidden, default_minimum=1)
+        if spec is None:
+            verdicts.append(
+                _verdict(
+                    VerdictStatus.FAIL,
+                    "case.tool_forbidden_invalid",
+                    "tool spec",
+                    forbidden,
+                )
+            )
+            continue
+        name, _, arguments = spec
+        matches = [
+            record
+            for record in calls
+            if record.get("tool") == name and _contains(record.get("args"), arguments)
+        ]
+        verdicts.append(
+            _verdict(
+                VerdictStatus.PASS if not matches else VerdictStatus.FAIL,
+                f"tools.forbidden.{index}",
+                forbidden,
+                {"matching_calls": len(matches)},
+                *(f"trace.jsonl:{record['_line']}" for record in matches),
+            )
+        )
+    if expectation.get("max_false_skips") is not None:
+        skips = [
+            record for record in bundle.trace if record.get("type") == "tool_skipped"
+        ]
+        verdicts.append(
+            _verdict(
+                VerdictStatus.NOT_EVALUABLE,
+                "tools.false_skips",
+                {"max": expectation.get("max_false_skips")},
+                {"observed_skip_records": len(skips)},
+                *(f"trace.jsonl:{record['_line']}" for record in skips),
+                detail="current tool_skipped records omit arguments and policy, so false skips cannot be classified",
+            )
+        )
+    return verdicts
+
+
+def _identity_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+    expectation = _mapping(case.expect.get("identity"))
+    if not expectation:
+        return []
+    problem = _artifact_problem(bundle, "grounding", missing_invalid=True)
+    if problem:
+        return [
+            _verdict(
+                problem,
+                "identity.artifact_available",
+                expectation,
+                bundle.errors.get("grounding", "missing"),
+                "artifacts/grounding_evidence.json",
+            )
+        ]
+    identity = _mapping(bundle.grounding.get("identity") if bundle.grounding else None)
+    verdicts: list[Verdict] = []
+    expected_status = expectation.get("status")
+    if expected_status is not None:
+        observed = identity.get("status")
+        verdicts.append(
+            _verdict(
+                (
+                    VerdictStatus.PASS
+                    if observed == expected_status
+                    else VerdictStatus.FAIL
+                ),
+                "identity.status",
+                expected_status,
+                observed,
+                "artifacts/grounding_evidence.json#/identity/status",
+            )
+        )
+    expected_symbols = expectation.get("authorized_symbols")
+    if expected_symbols is not None:
+        observed_symbols = identity.get("authorized_symbols")
+        valid = (
+            _string_list(expected_symbols) is not None
+            and _string_list(observed_symbols) is not None
+        )
+        matches = valid and set(observed_symbols) == set(expected_symbols)
+        verdicts.append(
+            _verdict(
+                VerdictStatus.PASS if matches else VerdictStatus.FAIL,
+                "identity.authorized_symbols",
+                expected_symbols,
+                observed_symbols,
+                "artifacts/grounding_evidence.json#/identity/authorized_symbols",
+            )
+        )
+    forbidden = expectation.get("forbidden_symbols", [])
+    observed_symbols = _string_list(identity.get("authorized_symbols")) or []
+    if forbidden:
+        violations = (
+            sorted(set(forbidden) & set(observed_symbols))
+            if _string_list(forbidden) is not None
+            else forbidden
+        )
+        verdicts.append(
+            _verdict(
+                VerdictStatus.PASS if not violations else VerdictStatus.FAIL,
+                "identity.forbidden_symbols",
+                forbidden,
+                violations,
+                "artifacts/grounding_evidence.json#/identity/authorized_symbols",
+            )
+        )
+    constraint = expectation.get("constraint")
+    if constraint is not None:
+        constraint_records = [
+            item
+            for record in identity.get("records", [])
+            if isinstance(record, Mapping)
+            for item in record.get("resolution_constraints", [])
+            if isinstance(item, Mapping)
+        ]
+        if not constraint_records:
+            verdicts.append(
+                _verdict(
+                    VerdictStatus.NOT_EVALUABLE,
+                    "identity.constraint",
+                    constraint,
+                    None,
+                    "artifacts/grounding_evidence.json#/identity/records/*/resolution_constraints",
+                    detail="constraint provenance is not instrumented in this run",
+                )
+            )
+        else:
+            matching = [
+                item for item in constraint_records if _contains(item, constraint)
+            ]
+            verdicts.append(
+                _verdict(
+                    VerdictStatus.PASS if matching else VerdictStatus.FAIL,
+                    "identity.constraint",
+                    constraint,
+                    constraint_records,
+                    "artifacts/grounding_evidence.json#/identity/records/*/resolution_constraints",
+                )
+            )
+    return verdicts
+
+
+def _evidence_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+    expectation = _mapping(case.expect.get("evidence"))
+    if not expectation:
+        return []
+    problem = _artifact_problem(bundle, "grounding", missing_invalid=True)
+    if problem:
+        return [
+            _verdict(
+                problem,
+                "evidence.artifact_available",
+                expectation,
+                bundle.errors.get("grounding"),
+                "artifacts/grounding_evidence.json",
+            )
+        ]
+    grounding = bundle.grounding or {}
+    evidence = grounding.get("evidence")
+    validations = grounding.get("validations")
+    if not isinstance(evidence, list) or not isinstance(validations, list):
+        return [
+            _verdict(
+                VerdictStatus.INVALID_ARTIFACT,
+                "evidence.schema",
+                {"evidence": "array", "validations": "array"},
+                {
+                    "evidence": type(evidence).__name__,
+                    "validations": type(validations).__name__,
+                },
+                "artifacts/grounding_evidence.json",
+            )
+        ]
+    symbols = {
+        item.get("symbol")
+        for item in evidence
+        if isinstance(item, Mapping) and isinstance(item.get("symbol"), str)
+    }
+    required = expectation.get("required_symbols", [])
+    missing = (
+        sorted(set(required) - symbols)
+        if _string_list(required) is not None
+        else required
+    )
+    verdicts = [
+        _verdict(
+            VerdictStatus.PASS if not missing else VerdictStatus.FAIL,
+            "evidence.required_symbols",
+            required,
+            {"observed_symbols": sorted(symbols), "missing": missing},
+            "artifacts/grounding_evidence.json#/evidence",
+        )
+    ]
+    issue_codes = {
+        issue.get("code")
+        for validation in validations
+        if isinstance(validation, Mapping)
+        for issue in validation.get("issues", [])
+        if isinstance(issue, Mapping) and isinstance(issue.get("code"), str)
+    }
+    forbidden = expectation.get("forbidden_issue_codes", [])
+    violations = (
+        sorted(set(forbidden) & issue_codes)
+        if _string_list(forbidden) is not None
+        else forbidden
+    )
+    verdicts.append(
+        _verdict(
+            VerdictStatus.PASS if not violations else VerdictStatus.FAIL,
+            "evidence.forbidden_issue_codes",
+            forbidden,
+            {"observed_issue_codes": sorted(issue_codes), "violations": violations},
+            "artifacts/grounding_evidence.json#/validations",
+        )
+    )
+    return verdicts
+
+
+def _budget_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+    expectation = _mapping(case.expect.get("budget"))
+    if not expectation:
+        return []
+    verdicts: list[Verdict] = []
+    end_records = [record for record in bundle.trace if record.get("type") == "end"]
+    observed_iterations = end_records[-1].get("iterations") if end_records else None
+    for gate in ("max_llm_calls", "max_iterations"):
+        limit = expectation.get(gate)
+        if limit is None:
+            continue
+        trace_problem = _artifact_problem(bundle, "trace", missing_invalid=True)
+        if trace_problem:
+            verdicts.append(
+                _verdict(
+                    trace_problem,
+                    f"budget.{gate}",
+                    limit,
+                    bundle.errors.get("trace", "missing"),
+                    "trace.jsonl",
+                )
+            )
+            continue
+        observed = observed_iterations
+        if observed is None:
+            verdicts.append(
+                _verdict(
+                    VerdictStatus.NOT_EVALUABLE,
+                    f"budget.{gate}",
+                    limit,
+                    None,
+                    "trace.jsonl",
+                )
+            )
+        else:
+            status = (
+                VerdictStatus.PASS
+                if _within_limit(observed, limit)
+                else VerdictStatus.FAIL
+            )
+            verdicts.append(
+                _verdict(
+                    status,
+                    f"budget.{gate}",
+                    limit,
+                    observed,
+                    f"trace.jsonl:{end_records[-1]['_line']}",
+                )
+            )
+
+    input_limit = expectation.get("max_input_tokens")
+    if input_limit is not None:
+        problem = _artifact_problem(bundle, "llm_usage", missing_invalid=False)
+        if problem:
+            verdicts.append(
+                _verdict(
+                    problem,
+                    "budget.max_input_tokens",
+                    input_limit,
+                    bundle.errors.get("llm_usage"),
+                    "llm_usage.json",
+                )
+            )
+        else:
+            usage_error = _usage_error(bundle.llm_usage or {})
+            totals = _mapping((bundle.llm_usage or {}).get("totals"))
+            if usage_error:
+                verdicts.append(
+                    _verdict(
+                        VerdictStatus.INVALID_ARTIFACT,
+                        "budget.max_input_tokens",
+                        input_limit,
+                        usage_error,
+                        "llm_usage.json",
+                    )
+                )
+            elif (
+                isinstance(observed_iterations, int)
+                and totals.get("calls") != observed_iterations
+            ):
+                verdicts.append(
+                    _verdict(
+                        VerdictStatus.NOT_EVALUABLE,
+                        "budget.max_input_tokens",
+                        input_limit,
+                        {
+                            "reported_input_tokens": totals.get("input_tokens"),
+                            "usage_calls": totals.get("calls"),
+                            "llm_calls": observed_iterations,
+                        },
+                        "llm_usage.json",
+                        f"trace.jsonl:{end_records[-1]['_line']}",
+                        detail="provider usage is missing for one or more LLM calls",
+                    )
+                )
+            else:
+                observed = totals.get("input_tokens")
+                status = (
+                    VerdictStatus.PASS
+                    if _within_limit(observed, input_limit)
+                    else VerdictStatus.FAIL
+                )
+                verdicts.append(
+                    _verdict(
+                        status,
+                        "budget.max_input_tokens",
+                        input_limit,
+                        observed,
+                        "llm_usage.json#/totals/input_tokens",
+                    )
+                )
+
+    for gate in ("max_visible_tools", "max_tool_schema_tokens"):
+        limit = expectation.get(gate)
+        if limit is not None:
+            verdicts.append(
+                _verdict(
+                    VerdictStatus.NOT_EVALUABLE,
+                    f"budget.{gate}",
+                    limit,
+                    None,
+                    "trace.jsonl",
+                    detail=f"{gate} requires per-LLM-call tool-surface instrumentation",
+                )
+            )
+    return verdicts
+
+
+def _risk_verdicts(case: HarnessCase, bundle: ArtifactBundle) -> list[Verdict]:
+    expectation = _mapping(case.expect.get("risk"))
+    if not expectation:
+        return []
+    if "trace" in bundle.errors or "trace" not in bundle.present:
+        return [
+            _verdict(
+                VerdictStatus.INVALID_ARTIFACT,
+                "risk.trace_available",
+                expectation,
+                bundle.errors.get("trace"),
+                "trace.jsonl",
+            )
+        ]
+    successful = _successful_tool_records(bundle.trace)
+    verdicts: list[Verdict] = []
+    forbidden = expectation.get("forbidden_successful_tools", [])
+    violations = [record for record in successful if record.get("tool") in forbidden]
+    verdicts.append(
+        _verdict(
+            VerdictStatus.PASS if not violations else VerdictStatus.FAIL,
+            "risk.forbidden_successful_tools",
+            forbidden,
+            sorted({record.get("tool") for record in violations}),
+            *(f"trace.jsonl:{record['_line']}" for record in violations),
+        )
+    )
+    maximum = expectation.get("max_authorization_bypasses")
+    if maximum is not None:
+        blocked_calls = {
+            record.get("call_id"): record
+            for record in bundle.trace
+            if record.get("type") == "tool_call" and record.get("blocked") is True
+        }
+        bypasses = [
+            record for record in successful if record.get("call_id") in blocked_calls
+        ]
+        bypasses.extend(
+            record
+            for record in bundle.trace
+            if record.get("type") == "authorization_bypass"
+        )
+        status = (
+            VerdictStatus.PASS
+            if _within_limit(len(bypasses), maximum)
+            else VerdictStatus.FAIL
+        )
+        verdicts.append(
+            _verdict(
+                status,
+                "risk.authorization_bypasses",
+                maximum,
+                len(bypasses),
+                *(f"trace.jsonl:{record['_line']}" for record in bypasses),
+            )
+        )
+    return verdicts
+
+
+def _tool_result_join_verdict(bundle: ArtifactBundle) -> Verdict:
+    problem = _artifact_problem(bundle, "trace", missing_invalid=True)
+    if problem:
+        return _verdict(
+            problem,
+            "tools.result_join",
+            "consistent call/result records",
+            bundle.errors.get("trace"),
+            "trace.jsonl",
+        )
+    calls = [record for record in bundle.trace if record.get("type") == "tool_call"]
+    results = [record for record in bundle.trace if record.get("type") == "tool_result"]
+    calls_by_id: dict[Any, Mapping[str, Any]] = {}
+    problems: list[str] = []
+    for call in calls:
+        call_id = call.get("call_id")
+        if not isinstance(call_id, str) or not call_id:
+            problems.append(f"line {call['_line']}: tool_call has no call_id")
+        elif call_id in calls_by_id:
+            problems.append(f"line {call['_line']}: duplicate tool_call id {call_id}")
+        else:
+            calls_by_id[call_id] = call
+    result_ids: set[str] = set()
+    outcomes = Counter()
+    for result in results:
+        call_id = result.get("call_id")
+        call = calls_by_id.get(call_id)
+        if call is None:
+            problems.append(f"line {result['_line']}: orphan tool_result id {call_id}")
+            continue
+        if call_id in result_ids:
+            problems.append(
+                f"line {result['_line']}: duplicate tool_result id {call_id}"
+            )
+        result_ids.add(call_id)
+        if call.get("tool") != result.get("tool"):
+            problems.append(f"line {result['_line']}: tool name differs from its call")
+        if result.get("status") not in {"ok", "error"}:
+            problems.append(f"line {result['_line']}: invalid tool_result status")
+        outcome = (
+            "blocked"
+            if call.get("blocked") is True
+            else str(result.get("status") or "unknown")
+        )
+        outcomes[outcome] += 1
+    outcomes["unfinished"] = sum(
+        1 for call_id in calls_by_id if call_id not in result_ids
+    )
+    outcomes["cached"] = sum(
+        1 for record in bundle.trace if record.get("type") == "tool_result_cached"
+    )
+    outcomes["skipped"] = sum(
+        1 for record in bundle.trace if record.get("type") == "tool_skipped"
+    )
+    return _verdict(
+        VerdictStatus.INVALID_ARTIFACT if problems else VerdictStatus.PASS,
+        "tools.result_join",
+        "consistent call/result records",
+        {"outcomes": dict(sorted(outcomes.items())), "problems": problems},
+        "trace.jsonl",
+    )
+
+
+def _execution_status(value: Any) -> Any:
+    return "failed" if value == "error" else value

agent/evals/harness/cases/identity/explicit_a_share.json

@@ -0,0 +1,36 @@
+{
+  "schema_version": 1,
+  "case_id": "identity.a_h.explicit_a_share.v1",
+  "scenario": "multi_market_identity",
+  "prompt": "只分析 A 股恒瑞医药，给出最新财务与行业证据。",
+  "history": [],
+  "data_snapshot": "fixtures/identity/a_h_hengrui.v1.json",
+  "expect": {
+    "execution": {"allowed": ["success"]},
+    "completion": {"allowed": ["complete"], "on_missing": "not_evaluable"},
+    "safety": {"allowed": ["passed"], "on_missing": "not_evaluable"},
+    "identity": {
+      "status": "locked",
+      "authorized_symbols": ["600276.SH"],
+      "forbidden_symbols": ["01276.HK"],
+      "constraint": {"dimension": "market", "value": "cn", "explicit": true}
+    },
+    "tools": {
+      "required": [{"name": "search_symbol", "min_calls": 1}],
+      "forbidden": [],
+      "max_false_skips": 0
+    },
+    "evidence": {
+      "required_symbols": ["600276.SH"],
+      "forbidden_issue_codes": ["numeric_claim_conflict", "unsourced_symbol_figures"]
+    },
+    "budget": {
+      "max_llm_calls": 8,
+      "max_input_tokens": 150000,
+      "max_iterations": 10,
+      "max_visible_tools": null,
+      "max_tool_schema_tokens": null
+    },
+    "risk": {"forbidden_successful_tools": [], "max_authorization_bypasses": 0}
+  }
+}

agent/evals/harness/cases/identity/fixtures/identity/a_h_hengrui.v1.json

@@ -0,0 +1,19 @@
+{
+  "schema_version": 1,
+  "source": "synthetic",
+  "query": "恒瑞医药",
+  "candidates": [
+    {
+      "symbol": "600276.SH",
+      "name": "恒瑞医药",
+      "market": "cn",
+      "type": "equity"
+    },
+    {
+      "symbol": "01276.HK",
+      "name": "恒瑞医药",
+      "market": "hk",
+      "type": "equity"
+    }
+  ]
+}

agent/evals/harness/methodology.py

@@ -0,0 +1,52 @@
+"""Run-manifest integrity checks for Harness reports."""
+
+from __future__ import annotations
+
+from src.governance.manifest import RunManifest
+
+from .artifacts import ArtifactBundle
+from .schema import Verdict, VerdictStatus
+
+
+def manifest_verdict(bundle: ArtifactBundle) -> Verdict:
+    """Verify the persisted methodology hash and expose its comparison fields."""
+    if "manifest" in bundle.errors:
+        return Verdict(
+            VerdictStatus.INVALID_ARTIFACT,
+            "methodology.manifest_integrity",
+            True,
+            bundle.errors["manifest"],
+            ("run_manifest.json",),
+        )
+    if "manifest" not in bundle.present:
+        return Verdict(
+            VerdictStatus.NOT_EVALUABLE,
+            "methodology.manifest_integrity",
+            True,
+            None,
+            ("run_manifest.json",),
+        )
+    try:
+        manifest = RunManifest.from_dict(bundle.manifest or {})
+        valid = manifest.verify_hash()
+    except (KeyError, TypeError, ValueError) as exc:
+        return Verdict(
+            VerdictStatus.INVALID_ARTIFACT,
+            "methodology.manifest_integrity",
+            True,
+            str(exc),
+            ("run_manifest.json",),
+        )
+    observed = {
+        "valid": valid,
+        "manifest_hash": manifest.manifest_hash,
+        "tool_count": len(manifest.tools.tool_names),
+        "tools_hash": manifest.tools.tools_hash,
+    }
+    return Verdict(
+        VerdictStatus.PASS if valid else VerdictStatus.INVALID_ARTIFACT,
+        "methodology.manifest_integrity",
+        True,
+        observed,
+        ("run_manifest.json",),
+    )

agent/evals/harness/observations.py

@@ -0,0 +1,113 @@
+"""Small validators and matchers shared by Harness assertions."""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any, Iterable, Mapping
+
+
+def contains(observed: Any, expected_subset: Any) -> bool:
+    """Recursively match an evaluator-declared subset against an observation."""
+    if expected_subset in (None, {}):
+        return True
+    if isinstance(expected_subset, Mapping):
+        if not isinstance(observed, Mapping):
+            return False
+        return all(
+            key in observed and contains(observed[key], value)
+            for key, value in expected_subset.items()
+        )
+    if isinstance(expected_subset, list):
+        if not isinstance(observed, list) or len(observed) < len(expected_subset):
+            return False
+        return all(
+            any(contains(candidate, item) for candidate in observed)
+            for item in expected_subset
+        )
+    return observed == expected_subset
+
+
+def mapping(value: Any) -> Mapping[str, Any]:
+    """Return a mapping view or an empty mapping for malformed optional data."""
+    return value if isinstance(value, Mapping) else {}
+
+
+def string_list(value: Any) -> list[str] | None:
+    """Return a validated string list."""
+    if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
+        return None
+    return value
+
+
+def successful_tool_records(
+    trace: Iterable[Mapping[str, Any]],
+) -> list[Mapping[str, Any]]:
+    """Return executed or cached successful tool outcomes."""
+    return [
+        record
+        for record in trace
+        if (record.get("type") == "tool_result" and record.get("status") == "ok")
+        or record.get("type") == "tool_result_cached"
+    ]
+
+
+def tool_spec(
+    value: Any,
+    *,
+    default_minimum: int = 1,
+) -> tuple[str, int, Mapping[str, Any]] | None:
+    """Normalize one required or forbidden tool expectation."""
+    if isinstance(value, str):
+        return value, default_minimum, {}
+    if not isinstance(value, Mapping) or not isinstance(value.get("name"), str):
+        return None
+    minimum = value.get("min_calls", default_minimum)
+    arguments = value.get("arguments", {})
+    if (
+        not isinstance(minimum, int)
+        or minimum < 0
+        or not isinstance(arguments, Mapping)
+    ):
+        return None
+    return value["name"], minimum, arguments
+
+
+def usage_error(usage: Mapping[str, Any]) -> str | None:
+    """Reconcile provider totals with per-iteration usage records."""
+    totals = usage.get("totals")
+    iterations = usage.get("per_iteration")
+    if not isinstance(totals, Mapping) or not isinstance(iterations, list):
+        return "totals must be an object and per_iteration must be an array"
+    fields = ("input_tokens", "output_tokens", "total_tokens")
+    if any(
+        not isinstance(totals.get(field), int)
+        or isinstance(totals[field], bool)
+        or totals[field] < 0
+        for field in (*fields, "calls")
+    ):
+        return "usage totals must contain non-negative integer token counts and calls"
+    sums = Counter({field: 0 for field in fields})
+    for item in iterations:
+        if not isinstance(item, Mapping):
+            return "per_iteration entries must be objects"
+        for field in fields:
+            value = item.get(field)
+            if not isinstance(value, int) or isinstance(value, bool) or value < 0:
+                return f"per_iteration {field} must be a non-negative integer"
+            sums[field] += value
+    if any(sums[field] != totals[field] for field in fields):
+        return f"provider totals disagree with per_iteration sums: {dict(sums)}"
+    if totals["calls"] != len(iterations):
+        return "totals.calls disagrees with the number of per_iteration records"
+    return None
+
+
+def within_limit(observed: Any, limit: Any) -> bool:
+    """Compare non-boolean integer observations and limits."""
+    return (
+        isinstance(observed, int)
+        and not isinstance(observed, bool)
+        and isinstance(limit, int)
+        and not isinstance(limit, bool)
+        and observed <= limit
+    )

agent/evals/harness/report.py

@@ -0,0 +1,84 @@
+"""Aggregate Harness reports while preserving evaluation coverage."""
+
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from typing import Any, Iterable
+
+from .schema import EvaluationReport, VerdictStatus
+
+
+def aggregate_reports(reports: Iterable[EvaluationReport]) -> dict[str, Any]:
+    """Aggregate verdict counts by status and scenario without dropping gaps."""
+    materialized = list(reports)
+    totals: Counter[str] = Counter()
+    by_scenario: dict[str, Counter[str]] = defaultdict(Counter)
+    by_code: dict[str, Counter[str]] = defaultdict(Counter)
+    for report in materialized:
+        for verdict in report.verdicts:
+            status = verdict.status.value
+            totals[status] += 1
+            by_scenario[report.scenario][status] += 1
+            by_code[verdict.code][status] += 1
+    all_statuses = [status.value for status in VerdictStatus]
+
+    def complete(counter: Counter[str]) -> dict[str, int]:
+        return {status: counter[status] for status in all_statuses}
+
+    assertion_count = sum(totals.values())
+    evaluable = totals[VerdictStatus.PASS.value] + totals[VerdictStatus.FAIL.value]
+    metric_groups = {
+        "task_completion": ("completion.status",),
+        "identity_resolution": ("identity.status", "identity.authorized_symbols"),
+        "required_tool_coverage": ("tools.required.",),
+        "evidence_coverage": ("evidence.required_symbols",),
+        "high_risk_boundary": (
+            "risk.forbidden_successful_tools",
+            "risk.authorization_bypasses",
+        ),
+    }
+    return {
+        "schema_version": 1,
+        "case_count": len(materialized),
+        "assertion_count": assertion_count,
+        "evaluation_coverage": (
+            (evaluable / assertion_count) if assertion_count else 0.0
+        ),
+        "status_counts": complete(totals),
+        "by_scenario": {
+            name: complete(counts) for name, counts in sorted(by_scenario.items())
+        },
+        "by_assertion": {
+            name: complete(counts) for name, counts in sorted(by_code.items())
+        },
+        "metrics": {
+            name: _metric_summary(materialized, prefixes)
+            for name, prefixes in metric_groups.items()
+        },
+    }
+
+
+def _metric_summary(
+    reports: Iterable[EvaluationReport],
+    prefixes: tuple[str, ...],
+) -> dict[str, Any]:
+    counts: Counter[str] = Counter()
+    for report in reports:
+        for verdict in report.verdicts:
+            if any(
+                verdict.code == prefix or verdict.code.startswith(prefix)
+                for prefix in prefixes
+            ):
+                counts[verdict.status.value] += 1
+    total = sum(counts.values())
+    evaluable = counts[VerdictStatus.PASS.value] + counts[VerdictStatus.FAIL.value]
+    return {
+        "assertions": total,
+        "evaluable": evaluable,
+        "pass_rate": (
+            counts[VerdictStatus.PASS.value] / evaluable if evaluable else None
+        ),
+        "coverage": evaluable / total if total else 0.0,
+        "not_evaluable": counts[VerdictStatus.NOT_EVALUABLE.value],
+        "invalid_artifact": counts[VerdictStatus.INVALID_ARTIFACT.value],
+    }

agent/evals/harness/runner.py

@@ -0,0 +1,98 @@
+"""CLI and API for case-by-artifact deterministic Harness evaluation."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any, Sequence
+
+from .artifacts import ArtifactBundle
+from .assertions import evaluate_assertions
+from .schema import EvaluationReport, HarnessCase, VerdictStatus
+
+
+def load_case(path: Path) -> HarnessCase:
+    """Load one versioned case JSON document."""
+    value = json.loads(Path(path).read_text(encoding="utf-8"))
+    return HarnessCase.from_dict(value)
+
+
+def evaluate_case(
+    case: HarnessCase,
+    run_dir: Path,
+    *,
+    trace_dir: Path | None = None,
+) -> EvaluationReport:
+    """Evaluate one case without invoking an LLM, tool, or network client."""
+    bundle = ArtifactBundle.load(run_dir, trace_dir=trace_dir)
+    verdicts = evaluate_assertions(case, bundle)
+    return EvaluationReport(
+        schema_version=1,
+        case_id=case.case_id,
+        scenario=case.scenario,
+        artifact_root=str(bundle.run_dir),
+        verdicts=verdicts,
+        metadata={
+            "trace_root": str(bundle.trace_dir),
+            "artifact_presence": sorted(bundle.present),
+            "artifact_errors": dict(bundle.errors),
+        },
+    )
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Verify a Vibe-Trading run artifact bundle offline"
+    )
+    parser.add_argument(
+        "--case", required=True, type=Path, help="path to one Harness case JSON file"
+    )
+    parser.add_argument(
+        "--run-dir",
+        required=True,
+        type=Path,
+        help="directory containing req/state/artifacts",
+    )
+    parser.add_argument(
+        "--trace-dir",
+        type=Path,
+        help="optional separate directory containing trace and manifest",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        help="write the JSON report to this path instead of stdout",
+    )
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the offline verifier and return a CI-friendly exit status."""
+    args = _parser().parse_args(argv)
+    try:
+        case = load_case(args.case)
+        report = evaluate_case(case, args.run_dir, trace_dir=args.trace_dir)
+    except (OSError, UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc:
+        _emit({"schema_version": 1, "error": str(exc)}, args.output)
+        return 2
+    _emit(report.to_dict(), args.output)
+    statuses = {verdict.status for verdict in report.verdicts}
+    if VerdictStatus.INVALID_ARTIFACT in statuses:
+        return 2
+    if VerdictStatus.FAIL in statuses:
+        return 1
+    return 0
+
+
+def _emit(payload: dict[str, Any], output: Path | None) -> None:
+    rendered = json.dumps(payload, indent=2, ensure_ascii=False, sort_keys=True) + "\n"
+    if output is None:
+        print(rendered, end="")
+        return
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(rendered, encoding="utf-8")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agent/evals/harness/schema.py

@@ -0,0 +1,228 @@
+"""Versioned models for deterministic Harness evaluation cases and reports."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from enum import Enum
+from pathlib import PurePosixPath
+from typing import Any, Mapping
+
+
+class VerdictStatus(str, Enum):
+    """A single assertion outcome."""
+
+    PASS = "PASS"
+    FAIL = "FAIL"
+    NOT_EVALUABLE = "NOT_EVALUABLE"
+    INVALID_ARTIFACT = "INVALID_ARTIFACT"
+
+
+@dataclass(frozen=True)
+class HarnessCase:
+    """One natural-language prompt plus evaluator-only expectations."""
+
+    schema_version: int
+    case_id: str
+    scenario: str
+    prompt: str
+    history: tuple[Mapping[str, Any], ...]
+    data_snapshot: str | None
+    expect: Mapping[str, Any]
+
+    @classmethod
+    def from_dict(cls, data: Mapping[str, Any]) -> "HarnessCase":
+        """Validate and construct schema version 1."""
+        if not isinstance(data, Mapping):
+            raise ValueError("case must be a JSON object")
+        if data.get("schema_version") != 1:
+            raise ValueError("unsupported case schema_version; expected 1")
+        case_id = data.get("case_id")
+        scenario = data.get("scenario")
+        prompt = data.get("prompt")
+        history = data.get("history", [])
+        expect = data.get("expect")
+        if not isinstance(case_id, str) or not case_id.strip():
+            raise ValueError("case_id must be a non-empty string")
+        if not isinstance(scenario, str) or not scenario.strip():
+            raise ValueError("scenario must be a non-empty string")
+        if not isinstance(prompt, str):
+            raise ValueError("prompt must be a string")
+        if not isinstance(history, list) or not all(
+            isinstance(item, Mapping) for item in history
+        ):
+            raise ValueError("history must be an array of objects")
+        if not isinstance(expect, Mapping):
+            raise ValueError("expect must be an object")
+        _validate_expectations(expect)
+
+        snapshot = data.get("data_snapshot")
+        if snapshot is not None:
+            if not isinstance(snapshot, str) or not snapshot:
+                raise ValueError(
+                    "data_snapshot must be a non-empty relative path or null"
+                )
+            path = PurePosixPath(snapshot)
+            if path.is_absolute() or ".." in path.parts:
+                raise ValueError("data_snapshot must stay inside the case fixture tree")
+
+        return cls(
+            schema_version=1,
+            case_id=case_id,
+            scenario=scenario,
+            prompt=prompt,
+            history=tuple(history),
+            data_snapshot=snapshot,
+            expect=expect,
+        )
+
+
+def _validate_expectations(expect: Mapping[str, Any]) -> None:
+    for name in ("execution", "completion", "safety"):
+        section = expect.get(name)
+        if section is None:
+            continue
+        if not isinstance(section, Mapping) or not _is_string_list(
+            section.get("allowed")
+        ):
+            raise ValueError(f"expect.{name}.allowed must be an array of strings")
+        if section.get("on_missing", "not_evaluable") != "not_evaluable":
+            raise ValueError(
+                f"expect.{name}.on_missing currently supports only not_evaluable"
+            )
+
+    tools = expect.get("tools")
+    if tools is not None:
+        if not isinstance(tools, Mapping):
+            raise ValueError("expect.tools must be an object")
+        for key in ("required", "forbidden"):
+            values = tools.get(key, [])
+            if not isinstance(values, list):
+                raise ValueError(f"expect.tools.{key} must be an array")
+            for value in values:
+                if isinstance(value, str):
+                    continue
+                if not isinstance(value, Mapping) or not isinstance(
+                    value.get("name"), str
+                ):
+                    raise ValueError(f"expect.tools.{key} entries must name a tool")
+                minimum = value.get("min_calls", 1)
+                arguments = value.get("arguments", {})
+                if not _is_non_negative_int(minimum) or not isinstance(
+                    arguments, Mapping
+                ):
+                    raise ValueError(
+                        f"expect.tools.{key} has invalid min_calls or arguments"
+                    )
+        maximum = tools.get("max_false_skips")
+        if maximum is not None and not _is_non_negative_int(maximum):
+            raise ValueError(
+                "expect.tools.max_false_skips must be a non-negative integer or null"
+            )
+
+    identity = expect.get("identity")
+    if identity is not None:
+        if not isinstance(identity, Mapping):
+            raise ValueError("expect.identity must be an object")
+        if "status" in identity and not isinstance(identity["status"], str):
+            raise ValueError("expect.identity.status must be a string")
+        for key in ("authorized_symbols", "forbidden_symbols"):
+            if key in identity and not _is_string_list(identity[key]):
+                raise ValueError(f"expect.identity.{key} must be an array of strings")
+        if "constraint" in identity and not isinstance(identity["constraint"], Mapping):
+            raise ValueError("expect.identity.constraint must be an object")
+
+    evidence = expect.get("evidence")
+    if evidence is not None:
+        if not isinstance(evidence, Mapping):
+            raise ValueError("expect.evidence must be an object")
+        for key in ("required_symbols", "forbidden_issue_codes"):
+            if key in evidence and not _is_string_list(evidence[key]):
+                raise ValueError(f"expect.evidence.{key} must be an array of strings")
+
+    budget = expect.get("budget")
+    if budget is not None:
+        if not isinstance(budget, Mapping):
+            raise ValueError("expect.budget must be an object")
+        for key in (
+            "max_llm_calls",
+            "max_input_tokens",
+            "max_iterations",
+            "max_visible_tools",
+            "max_tool_schema_tokens",
+        ):
+            if budget.get(key) is not None and not _is_non_negative_int(budget[key]):
+                raise ValueError(
+                    f"expect.budget.{key} must be a non-negative integer or null"
+                )
+
+    risk = expect.get("risk")
+    if risk is not None:
+        if not isinstance(risk, Mapping):
+            raise ValueError("expect.risk must be an object")
+        if not _is_string_list(risk.get("forbidden_successful_tools", [])):
+            raise ValueError(
+                "expect.risk.forbidden_successful_tools must be an array of strings"
+            )
+        maximum = risk.get("max_authorization_bypasses")
+        if maximum is not None and not _is_non_negative_int(maximum):
+            raise ValueError(
+                "expect.risk.max_authorization_bypasses must be a non-negative integer or null"
+            )
+
+
+def _is_string_list(value: Any) -> bool:
+    return isinstance(value, list) and all(isinstance(item, str) for item in value)
+
+
+def _is_non_negative_int(value: Any) -> bool:
+    return isinstance(value, int) and not isinstance(value, bool) and value >= 0
+
+
+@dataclass(frozen=True)
+class Verdict:
+    """Machine-readable result for one deterministic assertion."""
+
+    status: VerdictStatus
+    code: str
+    expected: Any
+    observed: Any
+    evidence_refs: tuple[str, ...] = ()
+    detail: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a JSON-safe representation."""
+        payload = asdict(self)
+        payload["status"] = self.status.value
+        return payload
+
+
+@dataclass(frozen=True)
+class EvaluationReport:
+    """All verdicts for one case against one artifact bundle."""
+
+    schema_version: int
+    case_id: str
+    scenario: str
+    artifact_root: str
+    verdicts: tuple[Verdict, ...]
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+
+    @property
+    def status_counts(self) -> dict[str, int]:
+        """Count outcomes without hiding unevaluated coverage."""
+        counts = {status.value: 0 for status in VerdictStatus}
+        for verdict in self.verdicts:
+            counts[verdict.status.value] += 1
+        return counts
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a stable JSON report."""
+        return {
+            "schema_version": self.schema_version,
+            "case_id": self.case_id,
+            "scenario": self.scenario,
+            "artifact_root": self.artifact_root,
+            "status_counts": self.status_counts,
+            "verdicts": [verdict.to_dict() for verdict in self.verdicts],
+            "metadata": dict(self.metadata),
+        }

agent/tests/harness_eval/__init__.py

@@ -0,0 +1 @@
+"""Tests for the offline Harness artifact verifier."""

agent/tests/harness_eval/conftest.py

@@ -0,0 +1,165 @@
+"""Synthetic, redistribution-safe Harness artifacts."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from evals.harness.schema import HarnessCase
+from src.governance.manifest import build_run_manifest
+
+PROMPT = "只分析 A 股恒瑞医药，给出最新财务与行业证据。"
+
+
+@pytest.fixture
+def harness_case() -> HarnessCase:
+    return HarnessCase.from_dict(
+        {
+            "schema_version": 1,
+            "case_id": "identity.a_h.explicit_a_share.v1",
+            "scenario": "multi_market_identity",
+            "prompt": PROMPT,
+            "history": [],
+            "data_snapshot": "fixtures/identity/a_h_hengrui.v1.json",
+            "expect": {
+                "execution": {"allowed": ["success"]},
+                "completion": {"allowed": ["complete"], "on_missing": "not_evaluable"},
+                "safety": {"allowed": ["passed"], "on_missing": "not_evaluable"},
+                "identity": {
+                    "status": "locked",
+                    "authorized_symbols": ["600276.SH"],
+                    "forbidden_symbols": ["01276.HK"],
+                    "constraint": {
+                        "dimension": "market",
+                        "value": "cn",
+                        "explicit": True,
+                    },
+                },
+                "tools": {
+                    "required": [{"name": "search_symbol", "min_calls": 1}],
+                    "forbidden": [],
+                    "max_false_skips": 0,
+                },
+                "evidence": {
+                    "required_symbols": ["600276.SH"],
+                    "forbidden_issue_codes": [
+                        "numeric_claim_conflict",
+                        "unsourced_symbol_figures",
+                    ],
+                },
+                "budget": {
+                    "max_llm_calls": 8,
+                    "max_input_tokens": 150000,
+                    "max_iterations": 10,
+                    "max_visible_tools": 12,
+                    "max_tool_schema_tokens": 4000,
+                },
+                "risk": {
+                    "forbidden_successful_tools": ["place_order"],
+                    "max_authorization_bypasses": 0,
+                },
+            },
+        }
+    )
+
+
+@pytest.fixture
+def artifact_dir(tmp_path: Path) -> Path:
+    run_dir = tmp_path / "run"
+    (run_dir / "artifacts").mkdir(parents=True)
+    _write_json(run_dir / "req.json", {"prompt": PROMPT, "context": {}})
+    _write_json(run_dir / "state.json", {"status": "success"})
+    _write_json(
+        run_dir / "artifacts" / "grounding_evidence.json",
+        {
+            "schema_version": 1,
+            "identity": {
+                "status": "locked",
+                "authorized_symbols": ["600276.SH"],
+                "records": [
+                    {
+                        "symbol": "600276.SH",
+                        "resolution_constraints": [
+                            {
+                                "dimension": "market",
+                                "value": "cn",
+                                "explicit": True,
+                                "source_message_id": "current_user_message",
+                                "source_span": [4, 6],
+                            }
+                        ],
+                    }
+                ],
+            },
+            "evidence": [{"symbol": "600276.SH", "field": "revenue", "value": 1.0}],
+            "validations": [{"valid": True, "issues": []}],
+        },
+    )
+    _write_json(
+        run_dir / "llm_usage.json",
+        {
+            "provider": "synthetic",
+            "model": "fixture",
+            "totals": {
+                "input_tokens": 100,
+                "output_tokens": 25,
+                "total_tokens": 125,
+                "calls": 2,
+            },
+            "per_iteration": [
+                {
+                    "iter": 1,
+                    "input_tokens": 40,
+                    "output_tokens": 10,
+                    "total_tokens": 50,
+                },
+                {
+                    "iter": 2,
+                    "input_tokens": 60,
+                    "output_tokens": 15,
+                    "total_tokens": 75,
+                },
+            ],
+        },
+    )
+    records = [
+        {"type": "start", "iter": 1, "prompt": PROMPT},
+        {"type": "message", "iter": 1, "role": "user", "content": PROMPT},
+        {
+            "type": "tool_call",
+            "iter": 1,
+            "tool": "search_symbol",
+            "call_id": "resolve",
+            "args": {"query": "恒瑞医药"},
+        },
+        {
+            "type": "tool_result",
+            "iter": 1,
+            "tool": "search_symbol",
+            "call_id": "resolve",
+            "status": "ok",
+        },
+        {"type": "end", "iter": 2, "iterations": 2, "status": "success"},
+    ]
+    (run_dir / "trace.jsonl").write_text(
+        "".join(json.dumps(record, ensure_ascii=False) + "\n" for record in records),
+        encoding="utf-8",
+    )
+    manifest = build_run_manifest(
+        run_id="synthetic",
+        timestamp="2026-08-30T00:00:00Z",
+        system_prompt="fixture",
+        tool_names=["search_symbol"],
+        package_versions={"vibe-trading-ai": "fixture"},
+    )
+    (run_dir / "run_manifest.json").write_text(
+        manifest.to_json() + "\n", encoding="utf-8"
+    )
+    return run_dir
+
+
+def _write_json(path: Path, value: Any) -> None:
+    path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")

agent/tests/harness_eval/test_runner.py

@@ -0,0 +1,323 @@
+"""End-to-end deterministic assertions and aggregate coverage."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import replace
+from pathlib import Path
+
+from evals.harness.report import aggregate_reports
+from evals.harness.runner import evaluate_case, main
+from evals.harness.schema import EvaluationReport, HarnessCase, VerdictStatus
+
+
+def _by_code(report: EvaluationReport) -> dict[str, object]:
+    return {verdict.code: verdict for verdict in report.verdicts}
+
+
+def test_complete_synthetic_bundle_passes_observable_assertions(
+    harness_case: HarnessCase,
+    artifact_dir: Path,
+) -> None:
+    report = evaluate_case(harness_case, artifact_dir)
+    verdicts = _by_code(report)
+
+    assert report.status_counts == {
+        "PASS": 17,
+        "FAIL": 0,
+        "NOT_EVALUABLE": 5,
+        "INVALID_ARTIFACT": 0,
+    }
+    assert verdicts["prompt.request_preserved"].status is VerdictStatus.PASS
+    assert verdicts["identity.constraint"].status is VerdictStatus.PASS
+    assert verdicts["tools.false_skips"].status is VerdictStatus.NOT_EVALUABLE
+    assert verdicts["completion.status"].status is VerdictStatus.NOT_EVALUABLE
+    assert verdicts["budget.max_visible_tools"].status is VerdictStatus.NOT_EVALUABLE
+    assert verdicts["tools.result_join"].observed["outcomes"]["ok"] == 1
+
+
+def test_failures_and_invalid_artifacts_are_not_collapsed(
+    harness_case: HarnessCase,
+    artifact_dir: Path,
+) -> None:
+    (artifact_dir / "req.json").write_text(
+        json.dumps({"prompt": "rewritten"}), encoding="utf-8"
+    )
+    state = {"status": "failed"}
+    (artifact_dir / "state.json").write_text(json.dumps(state), encoding="utf-8")
+    usage = json.loads((artifact_dir / "llm_usage.json").read_text(encoding="utf-8"))
+    usage["totals"]["input_tokens"] = 999
+    (artifact_dir / "llm_usage.json").write_text(json.dumps(usage), encoding="utf-8")
+    with (artifact_dir / "trace.jsonl").open("a", encoding="utf-8") as trace:
+        trace.write(
+            json.dumps(
+                {
+                    "type": "tool_result",
+                    "tool": "ghost",
+                    "call_id": "orphan",
+                    "status": "ok",
+                }
+            )
+            + "\n"
+        )
+
+    report = evaluate_case(harness_case, artifact_dir)
+    verdicts = _by_code(report)
+
+    assert verdicts["prompt.request_preserved"].status is VerdictStatus.FAIL
+    assert (
+        verdicts["execution.reconciled_status"].status is VerdictStatus.INVALID_ARTIFACT
+    )
+    assert verdicts["tools.result_join"].status is VerdictStatus.INVALID_ARTIFACT
+    assert verdicts["budget.max_input_tokens"].status is VerdictStatus.INVALID_ARTIFACT
+
+
+def test_trace_error_status_reconciles_to_failed_state(
+    harness_case: HarnessCase,
+    artifact_dir: Path,
+) -> None:
+    (artifact_dir / "state.json").write_text(
+        json.dumps({"status": "failed"}), encoding="utf-8"
+    )
+    lines = [
+        json.loads(line)
+        for line in (artifact_dir / "trace.jsonl")
+        .read_text(encoding="utf-8")
+        .splitlines()
+    ]
+    lines[-1]["status"] = "error"
+    (artifact_dir / "trace.jsonl").write_text(
+        "".join(json.dumps(line) + "\n" for line in lines),
+        encoding="utf-8",
+    )
+    case = replace(
+        harness_case,
+        expect={**harness_case.expect, "execution": {"allowed": ["failed"]}},
+    )
+
+    report = evaluate_case(case, artifact_dir)
+
+    assert _by_code(report)["execution.reconciled_status"].status is VerdictStatus.PASS
+
+
+def test_missing_current_instrumentation_is_never_a_pass(
+    harness_case: HarnessCase,
+    artifact_dir: Path,
+) -> None:
+    (artifact_dir / "llm_usage.json").unlink()
+    grounding = json.loads(
+        (artifact_dir / "artifacts" / "grounding_evidence.json").read_text(
+            encoding="utf-8"
+        )
+    )
+    grounding["identity"]["records"][0].pop("resolution_constraints")
+    (artifact_dir / "artifacts" / "grounding_evidence.json").write_text(
+        json.dumps(grounding), encoding="utf-8"
+    )
+
+    report = evaluate_case(harness_case, artifact_dir)
+    verdicts = _by_code(report)
+
+    assert verdicts["budget.max_input_tokens"].status is VerdictStatus.NOT_EVALUABLE
+    assert verdicts["identity.constraint"].status is VerdictStatus.NOT_EVALUABLE
+
+
+def test_missing_required_grounding_artifact_is_invalid(
+    harness_case: HarnessCase,
+    artifact_dir: Path,
+) -> None:
+    (artifact_dir / "artifacts" / "grounding_evidence.json").unlink()
+
+    report = evaluate_case(harness_case, artifact_dir)
+    verdicts = _by_code(report)
+
+    assert (
+        verdicts["identity.artifact_available"].status is VerdictStatus.INVALID_ARTIFACT
+    )
+    assert (
+        verdicts["evidence.artifact_available"].status is VerdictStatus.INVALID_ARTIFACT
+    )
+
+
+def test_tampered_manifest_is_invalid_artifact(
+    harness_case: HarnessCase,
+    artifact_dir: Path,
+) -> None:
+    manifest = json.loads(
+        (artifact_dir / "run_manifest.json").read_text(encoding="utf-8")
+    )
+    manifest["manifest_hash"] = "sha256:" + "0" * 64
+    (artifact_dir / "run_manifest.json").write_text(
+        json.dumps(manifest), encoding="utf-8"
+    )
+
+    report = evaluate_case(harness_case, artifact_dir)
+
+    assert (
+        _by_code(report)["methodology.manifest_integrity"].status
+        is VerdictStatus.INVALID_ARTIFACT
+    )
+
+
+def test_complete_provider_usage_can_fail_token_budget(
+    harness_case: HarnessCase,
+    artifact_dir: Path,
+) -> None:
+    case = replace(
+        harness_case,
+        expect={
+            **harness_case.expect,
+            "budget": {**harness_case.expect["budget"], "max_input_tokens": 99},
+        },
+    )
+
+    report = evaluate_case(case, artifact_dir)
+
+    assert _by_code(report)["budget.max_input_tokens"].status is VerdictStatus.FAIL
+
+
+def test_forbidden_successful_tool_and_blocked_bypass_fail(
+    harness_case: HarnessCase,
+    artifact_dir: Path,
+) -> None:
+    records = [
+        {
+            "type": "tool_call",
+            "iter": 2,
+            "tool": "place_order",
+            "call_id": "order",
+            "args": {},
+            "blocked": True,
+        },
+        {
+            "type": "tool_result",
+            "iter": 2,
+            "tool": "place_order",
+            "call_id": "order",
+            "status": "ok",
+        },
+    ]
+    with (artifact_dir / "trace.jsonl").open("a", encoding="utf-8") as trace:
+        for record in records:
+            trace.write(json.dumps(record) + "\n")
+
+    report = evaluate_case(harness_case, artifact_dir)
+    verdicts = _by_code(report)
+
+    assert verdicts["risk.forbidden_successful_tools"].status is VerdictStatus.FAIL
+    assert verdicts["risk.authorization_bypasses"].status is VerdictStatus.FAIL
+
+
+def test_unknown_tool_result_status_is_invalid_artifact(
+    harness_case: HarnessCase,
+    artifact_dir: Path,
+) -> None:
+    lines = [
+        json.loads(line)
+        for line in (artifact_dir / "trace.jsonl")
+        .read_text(encoding="utf-8")
+        .splitlines()
+    ]
+    next(line for line in lines if line["type"] == "tool_result")["status"] = "maybe"
+    (artifact_dir / "trace.jsonl").write_text(
+        "".join(json.dumps(line) + "\n" for line in lines),
+        encoding="utf-8",
+    )
+
+    report = evaluate_case(harness_case, artifact_dir)
+
+    assert (
+        _by_code(report)["tools.result_join"].status is VerdictStatus.INVALID_ARTIFACT
+    )
+
+
+def test_provider_usage_gap_makes_token_budget_not_evaluable(
+    harness_case: HarnessCase,
+    artifact_dir: Path,
+) -> None:
+    usage = json.loads((artifact_dir / "llm_usage.json").read_text(encoding="utf-8"))
+    usage["totals"] = {
+        "input_tokens": 40,
+        "output_tokens": 10,
+        "total_tokens": 50,
+        "calls": 1,
+    }
+    usage["per_iteration"] = [usage["per_iteration"][0]]
+    (artifact_dir / "llm_usage.json").write_text(json.dumps(usage), encoding="utf-8")
+
+    report = evaluate_case(harness_case, artifact_dir)
+
+    assert (
+        _by_code(report)["budget.max_input_tokens"].status
+        is VerdictStatus.NOT_EVALUABLE
+    )
+
+
+def test_aggregate_reports_exposes_coverage(
+    harness_case: HarnessCase,
+    artifact_dir: Path,
+) -> None:
+    first = evaluate_case(harness_case, artifact_dir)
+    second = replace(first, case_id="second")
+
+    aggregate = aggregate_reports([first, second])
+
+    assert aggregate["case_count"] == 2
+    assert aggregate["status_counts"]["NOT_EVALUABLE"] == 10
+    assert aggregate["evaluation_coverage"] == 34 / 44
+    assert aggregate["by_scenario"][harness_case.scenario]["PASS"] == 34
+    assert aggregate["metrics"]["task_completion"] == {
+        "assertions": 2,
+        "evaluable": 0,
+        "pass_rate": None,
+        "coverage": 0.0,
+        "not_evaluable": 2,
+        "invalid_artifact": 0,
+    }
+    assert aggregate["metrics"]["identity_resolution"]["pass_rate"] == 1.0
+
+
+def test_cli_exit_codes_distinguish_failures_from_invalid_input(
+    harness_case: HarnessCase,
+    artifact_dir: Path,
+    tmp_path: Path,
+) -> None:
+    case_path = tmp_path / "case.json"
+    case_path.write_text(
+        json.dumps(
+            {
+                "schema_version": harness_case.schema_version,
+                "case_id": harness_case.case_id,
+                "scenario": harness_case.scenario,
+                "prompt": harness_case.prompt,
+                "history": list(harness_case.history),
+                "data_snapshot": harness_case.data_snapshot,
+                "expect": harness_case.expect,
+            },
+            ensure_ascii=False,
+        ),
+        encoding="utf-8",
+    )
+    output = tmp_path / "report.json"
+    assert (
+        main(
+            [
+                "--case",
+                str(case_path),
+                "--run-dir",
+                str(artifact_dir),
+                "--output",
+                str(output),
+            ]
+        )
+        == 0
+    )
+    assert json.loads(output.read_text(encoding="utf-8"))["status_counts"]["PASS"] == 17
+
+    (artifact_dir / "req.json").write_text(
+        json.dumps({"prompt": "changed"}), encoding="utf-8"
+    )
+    assert main(["--case", str(case_path), "--run-dir", str(artifact_dir)]) == 1
+
+    case_path.write_text("{}", encoding="utf-8")
+    assert main(["--case", str(case_path), "--run-dir", str(artifact_dir)]) == 2

agent/tests/harness_eval/test_schema_and_artifacts.py

@@ -0,0 +1,106 @@
+"""Case validation and safe artifact loading."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from evals.harness.artifacts import ArtifactBundle
+from evals.harness.schema import HarnessCase
+
+
+@pytest.mark.parametrize(
+    ("change", "message"),
+    [
+        ({"schema_version": 2}, "schema_version"),
+        ({"data_snapshot": "../private.json"}, "fixture tree"),
+        ({"expect": {"execution": {"allowed": "success"}}}, "allowed"),
+        ({"expect": {"budget": {"max_llm_calls": True}}}, "max_llm_calls"),
+        (
+            {"expect": {"risk": {"forbidden_successful_tools": [1]}}},
+            "forbidden_successful_tools",
+        ),
+    ],
+)
+def test_invalid_case_contract_is_rejected(change: dict, message: str) -> None:
+    base = {
+        "schema_version": 1,
+        "case_id": "case",
+        "scenario": "scenario",
+        "prompt": "prompt",
+        "history": [],
+        "data_snapshot": None,
+        "expect": {"execution": {"allowed": ["success"]}},
+    }
+    base.update(change)
+    with pytest.raises(ValueError, match=message):
+        HarnessCase.from_dict(base)
+
+
+def test_trace_loader_uses_only_latest_session_turn(artifact_dir: Path) -> None:
+    path = artifact_dir / "trace.jsonl"
+    current = path.read_text(encoding="utf-8")
+    prior = [
+        {"type": "start", "iter": 1, "prompt": "old"},
+        {"type": "message", "iter": 1, "role": "user", "content": "old"},
+        {"type": "end", "iter": 1, "iterations": 1, "status": "failed"},
+    ]
+    path.write_text(
+        "".join(json.dumps(record) + "\n" for record in prior) + current,
+        encoding="utf-8",
+    )
+
+    bundle = ArtifactBundle.load(artifact_dir)
+
+    assert bundle.errors == {}
+    assert bundle.trace[0]["type"] == "start"
+    assert bundle.trace[0]["prompt"] != "old"
+    assert [record["status"] for record in bundle.trace if record["type"] == "end"] == [
+        "success"
+    ]
+
+
+def test_malformed_trace_is_reported_instead_of_silently_skipped(
+    artifact_dir: Path,
+) -> None:
+    (artifact_dir / "trace.jsonl").write_text(
+        '{"type":"start"}\n{bad json\n', encoding="utf-8"
+    )
+
+    bundle = ArtifactBundle.load(artifact_dir)
+
+    assert bundle.trace == ()
+    assert "trace.jsonl:2" in bundle.errors["trace"]
+
+
+def test_trace_sidecar_cannot_escape_trace_directory(artifact_dir: Path) -> None:
+    secret = artifact_dir.parent / "private.txt"
+    secret.write_text("secret", encoding="utf-8")
+    (artifact_dir / "trace.jsonl").write_text(
+        json.dumps(
+            {"type": "message", "role": "user", "content_path": "../private.txt"}
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+
+    bundle = ArtifactBundle.load(artifact_dir)
+
+    assert bundle.trace == ()
+    assert "escaped" in bundle.errors["trace"]
+    assert "secret" not in bundle.errors["trace"]
+
+
+def test_known_artifact_symlink_cannot_escape_run_directory(artifact_dir: Path) -> None:
+    outside = artifact_dir.parent / "outside.json"
+    outside.write_text(json.dumps({"prompt": "private"}), encoding="utf-8")
+    (artifact_dir / "req.json").unlink()
+    (artifact_dir / "req.json").symlink_to(outside)
+
+    bundle = ArtifactBundle.load(artifact_dir)
+
+    assert bundle.req is None
+    assert "escaped" in bundle.errors["req"]
+    assert '"prompt"' not in bundle.errors["req"]

pyproject.toml

@@ -82,7 +82,7 @@ py-modules = ["api_server", "mcp_server"]
 
 [tool.setuptools.packages.find]
 where = ["agent"]
-include = ["src*", "backtest*", "cli*"]
+include = ["src*", "backtest*", "cli*", "evals*"]
 
 [tool.setuptools.package-data]
 "src" = [
@@ -97,6 +97,7 @@ include = ["src*", "backtest*", "cli*"]
     "shadow_account/templates/*.html",
     "shadow_account/templates/*.css",
 ]
+"evals.harness" = ["README.md", "cases/**/*.json"]
 "backtest" = ["*.py"]
 "src.factors" = ["zoo/**/*.yaml", "zoo/**/*.md", "zoo/**/NOTICE"]