feat(governance): connect the model-risk primitives to the running system
The governance pieces existed as tested libraries that nothing called, which
is the same amount of governance as none: a Session.owner that is always None
carries no more information than a missing field.
Principal Threaded from require_auth through the session route into
SessionService, and persisted. It records HOW a request was
authorised and refuses to overstate it: a shared bearer key
and loopback trust both authorise without identifying, so
attributable is False and is recomputed on load rather than
read from storage. "Owner unknown" (None) and "authorised but
unattributable" stay distinct states.
ModelRecord Reachable through sdm_register/sdm_status. Supplying any
governance field makes intended_use and limitations
mandatory, because a registration missing either cannot be
reviewed. Four-eyes conflicts are reported, not blocked --
that call belongs to policy, not to a tool.
Run manifest Written beside each run's trace: prompt hash, tool registry,
package versions. Hashes only, never prompt text, and it
states that mid-run load_skill content is outside its
coverage rather than implying otherwise.
Chained ledger On by default, written alongside the classic audit.jsonl so
every existing reader is unaffected. The classic record lands
and is fsynced first; a chain failure is logged and swallowed,
because a detectable gap beats an undetectable half-executed
trade.
Multiple-testing control now feeds the alpha bench under a new key, leaving
every existing key untouched.
This commit is contained in:
Haozhe Wu committed
1 parent
11f97f9dfe
commit
a7d180fe61
18 files changed
+2129
-19
No files matched your search
@@ -564,6 +564,63 @@ class AgentLoop:
|
||||
"""
|
||||
self._cancel_event.set()
|
||||
|
||||
def _write_run_manifest(self, trace_dir: "Path", messages: List[Dict[str, Any]]) -> None:
|
||||
"""Record what methodology produced this run, beside its trace.
|
||||
|
||||
Answers "under what system prompt, which skills, and which tool set was
|
||||
that number produced" -- the question a reproducibility review asks and
|
||||
that nothing in this repo could previously answer. Written once per run
|
||||
as ``run_manifest.json`` next to ``trace.jsonl``.
|
||||
|
||||
The system-prompt hash transitively covers every skill injected at
|
||||
context-build time, because those skills ARE part of the prompt string.
|
||||
Skills pulled mid-run via ``load_skill`` are not, and appear in the
|
||||
trace instead; the manifest says so rather than implying coverage it
|
||||
does not have.
|
||||
|
||||
The prompt itself is never stored -- only its hash. The prompt can carry
|
||||
user memory and workspace content, and this file is a provenance record,
|
||||
not a second copy of the conversation.
|
||||
|
||||
Args:
|
||||
trace_dir: Directory holding this run's trace.
|
||||
messages: The fully built message list about to be sent.
|
||||
|
||||
Note:
|
||||
Never raises. A provenance record that can break a run is worse than
|
||||
a missing one; a failure is logged and the run continues.
|
||||
"""
|
||||
try:
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from src.governance.manifest import (
|
||||
build_run_manifest,
|
||||
collect_key_package_versions,
|
||||
)
|
||||
|
||||
system_prompt = next(
|
||||
(m.get("content", "") for m in messages if m.get("role") == "system"), ""
|
||||
)
|
||||
manifest = build_run_manifest(
|
||||
run_id=f"iter-{self._run_iteration + 1}",
|
||||
timestamp=datetime.now(timezone.utc).isoformat(),
|
||||
system_prompt=str(system_prompt),
|
||||
tool_names=list(self.registry.tool_names),
|
||||
package_versions=collect_key_package_versions(),
|
||||
extra={
|
||||
"skill_coverage": (
|
||||
"skills injected at context-build time are inside the "
|
||||
"hashed system prompt; skills loaded mid-run via "
|
||||
"load_skill appear in trace.jsonl, not here"
|
||||
),
|
||||
},
|
||||
)
|
||||
path = trace_dir / "run_manifest.json"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(manifest.to_json(indent=2), encoding="utf-8")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("run manifest not written (%s: %s)", type(exc).__name__, exc)
|
||||
|
||||
def run(self, user_message: str, history: Optional[List[Dict[str, Any]]] = None, session_id: str = "") -> Dict[str, Any]:
|
||||
"""Run the ReAct loop synchronously.
|
||||
|
||||
@@ -617,6 +674,7 @@ class AgentLoop:
|
||||
|
||||
trace_dir = SESSIONS_DIR / session_id if session_id else run_dir
|
||||
trace = TraceWriter(trace_dir)
|
||||
self._write_run_manifest(trace_dir, messages)
|
||||
if self._run_iteration == 0 and trace.path.exists():
|
||||
existing = TraceWriter.read(trace_dir)
|
||||
self._run_iteration = max(
|
||||
|
||||
@@ -17,6 +17,8 @@ from fastapi import HTTPException, Query, Request, Security, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
|
||||
from src.session.models import AuthMethod, Principal
|
||||
|
||||
from src.api._compat import host_attr as _host_attr
|
||||
from src.config.accessor import get_env_config
|
||||
|
||||
@@ -415,19 +417,40 @@ def _require_shutdown_authorization(
|
||||
)
|
||||
|
||||
|
||||
#: Subject recorded when the caller proved possession of the shared API key.
|
||||
#: It is a role, not a person: every holder of that one secret authenticates
|
||||
#: identically, so this string must never be presented as an identity.
|
||||
SHARED_KEY_SUBJECT = "shared-key-holder"
|
||||
|
||||
#: Subject recorded when no key is configured and a loopback client was trusted.
|
||||
LOOPBACK_SUBJECT = "loopback-operator"
|
||||
|
||||
|
||||
def _validate_api_auth(
|
||||
*,
|
||||
request: Request,
|
||||
cred: Optional[HTTPAuthorizationCredentials],
|
||||
query_api_key: Optional[str] = None,
|
||||
allow_query: bool = False,
|
||||
) -> None:
|
||||
) -> Principal:
|
||||
"""Validate configured auth, preserving loopback-only dev mode.
|
||||
|
||||
Key-first precedence: when an API key is configured every peer -- including
|
||||
loopback -- must present a valid credential (GHSA-7wgj). Only when no key is
|
||||
configured does the loopback dev-trust apply. Mirrors
|
||||
:func:`require_settings_write_auth`.
|
||||
|
||||
Returns:
|
||||
The :class:`~src.session.models.Principal` the request authenticated as.
|
||||
Both paths available today authorise without identifying, so the
|
||||
returned principal has ``attributable=False``; a caller that needs a
|
||||
named human must check that flag and refuse rather than read ``subject``
|
||||
as a person. Previously this function returned None; the return value is
|
||||
additive and every existing caller that ignores it is unaffected.
|
||||
|
||||
Raises:
|
||||
HTTPException: 401 on a bad or missing credential, 403 for a non-local
|
||||
client when no key is configured.
|
||||
"""
|
||||
if request.method.upper() not in _SAFE_BROWSER_METHODS:
|
||||
_reject_cross_site_browser_request(request)
|
||||
@@ -437,10 +460,10 @@ def _validate_api_auth(
|
||||
token = _auth_credential_from_header_or_query(cred, query_api_key, allow_query=allow_query)
|
||||
if not token or not hmac.compare_digest(token, api_key):
|
||||
raise HTTPException(status_code=401, detail="Invalid or missing API key")
|
||||
return
|
||||
return Principal(subject=SHARED_KEY_SUBJECT, auth_method=AuthMethod.SHARED_KEY)
|
||||
|
||||
if _is_local_client(request):
|
||||
return
|
||||
return Principal(subject=LOOPBACK_SUBJECT, auth_method=AuthMethod.LOOPBACK_TRUST)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="API_AUTH_KEY is required for non-local API access",
|
||||
@@ -514,9 +537,21 @@ def _shell_tools_enabled_for_request(request: Request) -> bool:
|
||||
async def require_auth(
|
||||
request: Request,
|
||||
cred: Optional[HTTPAuthorizationCredentials] = Security(_security),
|
||||
) -> None:
|
||||
"""Validate Bearer token for sensitive API endpoints."""
|
||||
_validate_api_auth(request=request, cred=cred)
|
||||
) -> Principal:
|
||||
"""Validate Bearer token for sensitive API endpoints.
|
||||
|
||||
Returns:
|
||||
The authenticated :class:`~src.session.models.Principal`. Routes that
|
||||
only need the check keep using ``dependencies=[Depends(require_auth)]``
|
||||
and discard it; routes that need to record *who* acted declare
|
||||
``principal: Principal = Depends(require_auth)`` instead. The return
|
||||
value is additive -- it was previously None and no existing caller
|
||||
inspected it.
|
||||
|
||||
Raises:
|
||||
HTTPException: Propagated from :func:`_validate_api_auth`.
|
||||
"""
|
||||
return _validate_api_auth(request=request, cred=cred)
|
||||
|
||||
|
||||
async def require_event_stream_auth(
|
||||
|
||||
@@ -332,13 +332,26 @@ def register_sessions_routes(app: FastAPI) -> None:
|
||||
# Session CRUD routes
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
@app.post("/sessions", response_model=SessionResponse, status_code=status.HTTP_201_CREATED, dependencies=[Depends(require_auth)])
|
||||
async def create_session(request: CreateSessionRequest):
|
||||
"""Create a chat session."""
|
||||
@app.post("/sessions", response_model=SessionResponse, status_code=status.HTTP_201_CREATED)
|
||||
async def create_session(
|
||||
request: CreateSessionRequest,
|
||||
principal=Depends(require_auth),
|
||||
):
|
||||
"""Create a chat session.
|
||||
|
||||
The authenticated principal is recorded as the session owner. Under the
|
||||
shared-key and loopback auth modes that principal is not attributable to
|
||||
a named human -- it carries ``attributable=False`` and must not be read
|
||||
as an identity. Recording it anyway is still worth doing: it captures
|
||||
HOW the session was authorised, which is the part that becomes an
|
||||
identity once an identity provider is wired in.
|
||||
"""
|
||||
svc = _host_get_session_service()
|
||||
if not svc:
|
||||
raise HTTPException(status_code=501, detail="Session runtime not enabled")
|
||||
session = svc.create_session(title=request.title, config=request.config)
|
||||
session = svc.create_session(
|
||||
title=request.title, config=request.config, owner=principal
|
||||
)
|
||||
return SessionResponse(
|
||||
session_id=session.session_id,
|
||||
title=session.title,
|
||||
|
||||
@@ -22,8 +22,15 @@ import time
|
||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||||
from typing import Any, Callable, Iterable
|
||||
|
||||
import numpy as np
|
||||
|
||||
from src.config.accessor import get_env_config
|
||||
from src.factors.factor_analysis_core import compute_ic_series
|
||||
from src.quantlib.multipletesting import (
|
||||
MIN_OBSERVATIONS,
|
||||
deflated_sharpe_ratio,
|
||||
expected_maximum_sharpe,
|
||||
)
|
||||
from src.factors.registry import (
|
||||
Registry,
|
||||
RegistryError,
|
||||
@@ -331,10 +338,60 @@ def run_bench(
|
||||
except Exception: # noqa: BLE001
|
||||
universe_meta = {"raw": str(universe_meta_raw)}
|
||||
|
||||
# Multiple-testing correction. `n_alphas_tested` has always been carried
|
||||
# here and has only ever been displayed; the best IR out of hundreds of
|
||||
# searched alphas is the maximum of hundreds of draws, and under a null
|
||||
# where none of them works that maximum is comfortably positive. The block
|
||||
# below prices that. Every key is NEW -- nothing existing changes value, so
|
||||
# no consumer of this dict is affected.
|
||||
#
|
||||
# Note on what is being deflated: `ir` here is mean(IC)/std(IC), not a
|
||||
# return Sharpe. The deflation family applies to any max-of-N t-like
|
||||
# statistic, but the resulting probability must be read as "this IC-IR
|
||||
# survives the search", never as a statement about realised returns.
|
||||
multiple_testing: dict[str, Any] = {"applicable": False}
|
||||
if len(rows) >= 2:
|
||||
ir_values = np.array([float(r["ir"]) for r in rows], dtype=float)
|
||||
best_row = rows_by_ir[0]
|
||||
best_ir = float(best_row["ir"])
|
||||
ir_dispersion = float(ir_values.std(ddof=1))
|
||||
observations = int(best_row.get("ic_count", 0))
|
||||
expected_max = expected_maximum_sharpe(len(rows), ir_dispersion)
|
||||
multiple_testing = {
|
||||
"applicable": True,
|
||||
"n_trials": len(rows),
|
||||
"best_id": best_row["id"],
|
||||
"best_ir": round(best_ir, 6),
|
||||
"ir_dispersion": round(ir_dispersion, 6),
|
||||
"expected_max_ir_under_null": round(expected_max, 6),
|
||||
"ir_haircut": round(best_ir - expected_max, 6),
|
||||
"ic_observations": observations,
|
||||
"statistic": "ic_information_ratio",
|
||||
}
|
||||
if observations >= MIN_OBSERVATIONS:
|
||||
deflated = deflated_sharpe_ratio(
|
||||
observed_sharpe=best_ir,
|
||||
n_trials=len(rows),
|
||||
n_observations=observations,
|
||||
trial_sharpe_std=ir_dispersion,
|
||||
)
|
||||
multiple_testing["deflated_probability"] = round(
|
||||
deflated.deflated_sharpe_ratio, 6
|
||||
)
|
||||
multiple_testing["survives_deflation"] = deflated.survives
|
||||
else:
|
||||
multiple_testing["deflated_probability"] = None
|
||||
multiple_testing["survives_deflation"] = None
|
||||
multiple_testing["note"] = (
|
||||
f"only {observations} IC observation(s); the deflated statistic "
|
||||
f"needs at least {MIN_OBSERVATIONS} to mean anything"
|
||||
)
|
||||
|
||||
entry.update(
|
||||
{
|
||||
"status": "ok",
|
||||
"n_alphas_tested": len(rows),
|
||||
"multiple_testing": multiple_testing,
|
||||
"n_skipped": len(skipped),
|
||||
"alive": counts["alive"],
|
||||
"reversed": counts["reversed"],
|
||||
|
||||
+116
-3
@@ -26,12 +26,43 @@ so sensitive values are ``[redacted]`` before they reach the trace, the live
|
||||
ledger, or the SSE bus. The opaque ``account_ref`` provenance carried by the
|
||||
authorizing mandate is intentionally NOT a record field here and is preserved
|
||||
elsewhere as the mandate→consent accountability chain.
|
||||
|
||||
**Durability + tamper-evidence (2026-08-06 governance hardening).** The
|
||||
dedicated ledger write is now fsynced on every append — closing a gap where
|
||||
the per-run diagnostic trace (``src.agent.trace.TraceWriter``) already
|
||||
fsynced every record but this compliance ledger did not, so a host crash
|
||||
could lose the last live-action record while losing nothing from the trace.
|
||||
This applies unconditionally; it changes no on-disk record shape and no
|
||||
function signature.
|
||||
|
||||
**Tamper-evident chain (on by default since 2026-08-06).**
|
||||
:func:`write_live_action` now writes BOTH ledgers on every call: the classic
|
||||
``audit.jsonl`` first, then a hash-chained, tamper-evident copy (via
|
||||
:mod:`src.governance.ledger`) to a DEDICATED second file,
|
||||
:func:`audit_chain_ledger_path` (``audit_chain.jsonl``).
|
||||
|
||||
Two files, not one, because the classic ledger's pre-existing plain records
|
||||
have no ``seq``/``prev_record_hash`` — retrofitting the chain format onto that
|
||||
file would make every legacy record look like chain corruption. Writing both
|
||||
means every existing reader of ``audit.jsonl`` is unaffected byte-for-byte
|
||||
while the chained copy makes editing or deleting a record detectable.
|
||||
|
||||
Order matters: the classic ledger is written and fsynced FIRST, so the
|
||||
compliance record is durable before the chained copy is attempted. If the
|
||||
chain append then fails, it is logged at ERROR and swallowed — the record is
|
||||
already on disk, so nothing is lost except tamper-evidence for that one entry,
|
||||
and that gap is itself detectable later by
|
||||
:func:`src.governance.ledger.verify_chain`. Crashing an in-flight order over a
|
||||
logging fault would trade a detectable gap for an undetectable half-executed
|
||||
trade, which is strictly worse. Pass ``chain=False`` to write only the classic
|
||||
ledger.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
@@ -43,6 +74,27 @@ from src.tools.redaction import redact_payload
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_fsync_warned = False
|
||||
|
||||
|
||||
def _warn_fsync_failure(exc: OSError, target: Path) -> None:
|
||||
"""Log the first ledger fsync failure for this process, then stay quiet.
|
||||
|
||||
Mirrors ``src.agent.trace.TraceWriter._warn_fsync_failure``: the write
|
||||
already landed in the OS page cache, so we degrade to flush-only rather
|
||||
than raising out of an audit call sitting downstream of a live order
|
||||
that has already happened.
|
||||
"""
|
||||
global _fsync_warned
|
||||
if _fsync_warned:
|
||||
return
|
||||
_fsync_warned = True
|
||||
logger.warning(
|
||||
"live-action ledger fsync failed on %s (%s); ledger durability degraded to flush-only",
|
||||
target,
|
||||
exc,
|
||||
)
|
||||
|
||||
#: SSE / surface event name for a live-action record (SPEC §5).
|
||||
_LIVE_ACTION_EVENT = "live.action"
|
||||
|
||||
@@ -66,6 +118,7 @@ class _TraceWriterLike(Protocol):
|
||||
EventCallback = Callable[[str, dict[str, Any]], Any]
|
||||
|
||||
_LEDGER_FILENAME = "audit.jsonl"
|
||||
_CHAIN_LEDGER_FILENAME = "audit_chain.jsonl"
|
||||
|
||||
#: Canonical live-action kinds (SPEC §5).
|
||||
LiveActionKind = Literal[
|
||||
@@ -103,6 +156,19 @@ def audit_ledger_path() -> Path:
|
||||
return live_root() / _LEDGER_FILENAME
|
||||
|
||||
|
||||
def audit_chain_ledger_path() -> Path:
|
||||
"""Return the path to the opt-in hash-chained live-action ledger.
|
||||
|
||||
Returns:
|
||||
``<runtime_root>/live/audit_chain.jsonl`` — a separate file from
|
||||
:func:`audit_ledger_path` (see module docstring for why). Not
|
||||
created here; :func:`write_live_action` with ``chain=True`` creates
|
||||
the directory on first append via
|
||||
:func:`src.governance.ledger.append_record`.
|
||||
"""
|
||||
return live_root() / _CHAIN_LEDGER_FILENAME
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LiveActionEvent:
|
||||
"""One immutable live-action audit record (SPEC.md Consent §5).
|
||||
@@ -184,6 +250,7 @@ def write_live_action(
|
||||
*,
|
||||
event_callback: EventCallback | None = None,
|
||||
trace_writer: _TraceWriterLike | None = None,
|
||||
chain: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Fan a redacted live-action record out to up to three sinks (SPEC §5).
|
||||
|
||||
@@ -195,7 +262,9 @@ def write_live_action(
|
||||
1. **Dedicated ledger** (always) — appended as single-line JSONL with a
|
||||
trailing newline to ``<runtime_root>/live/audit.jsonl``, opened in append
|
||||
mode so concurrent writers cannot truncate it (append-only). The directory
|
||||
tree is created ``0700`` on first write.
|
||||
tree is created ``0700`` on first write. The write is flushed and
|
||||
fsynced before this call returns (2026-08-06: previously flush-only,
|
||||
unlike the per-run trace writer — see module docstring).
|
||||
2. **Per-run trace** (when ``trace_writer`` is given) — written via
|
||||
``trace_writer.write({...})`` with ``type="live_action"`` so it sits
|
||||
alongside ``tool_call`` / ``tool_result`` in the run dir.
|
||||
@@ -212,19 +281,63 @@ def write_live_action(
|
||||
skipped when ``None``.
|
||||
trace_writer: Optional per-run trace sink exposing ``write(dict)``;
|
||||
skipped when ``None``.
|
||||
chain: When ``True``, sink 1 additionally writes a hash-chained,
|
||||
tamper-evident record (:mod:`src.governance.ledger`) to
|
||||
:func:`audit_chain_ledger_path` INSTEAD of the classic
|
||||
``audit.jsonl`` path, and the record returned/fanned out to
|
||||
sinks 2/3 carries the extra ``seq`` / ``prev_record_hash`` /
|
||||
``record_hash`` chain fields. Default ``False`` preserves the
|
||||
exact historical record shape and ledger file. Raises
|
||||
:class:`src.governance.ledger.LedgerCorruptionError` if that
|
||||
chain ledger's existing history is already broken.
|
||||
|
||||
Returns:
|
||||
The redacted record dict — identical to what was written to every sink —
|
||||
for the caller to embed or re-emit.
|
||||
for the caller to embed or re-emit. Carries chain fields too when
|
||||
``chain=True``.
|
||||
"""
|
||||
record = event.to_record()
|
||||
|
||||
# Sink 1: dedicated compliance ledger (always, append-only).
|
||||
# Sink 1: dedicated compliance ledger (always, append-only, fsynced).
|
||||
# The classic ledger is written unconditionally and FIRST. Any existing
|
||||
# tooling that reads audit.jsonl keeps working byte-for-byte, and the
|
||||
# compliance record exists on disk before the chained copy is attempted.
|
||||
path = audit_ledger_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
line = json.dumps(record, ensure_ascii=False)
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(line + "\n")
|
||||
handle.flush()
|
||||
try:
|
||||
os.fsync(handle.fileno())
|
||||
except OSError as exc:
|
||||
_warn_fsync_failure(exc, path)
|
||||
|
||||
if chain:
|
||||
from src.governance.ledger import append_record
|
||||
|
||||
try:
|
||||
# The chained copy carries three extra fields (seq /
|
||||
# prev_record_hash / record_hash). They belong to the chain file
|
||||
# only: the return value and the classic ledger line must stay
|
||||
# byte-identical to what callers already depend on, so the augmented
|
||||
# record is deliberately NOT rebound onto ``record``.
|
||||
append_record(audit_chain_ledger_path(), record, fsync=True)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# A chain-append failure must not take down an in-flight order. The
|
||||
# record is already durable in the classic ledger above, so nothing
|
||||
# is lost; what is lost is tamper-evidence for THIS entry, and that
|
||||
# gap is itself detectable later by ``verify_chain``. Crashing the
|
||||
# order path over a logging fault would trade a detectable gap for
|
||||
# an undetectable half-executed trade, which is the worse of the two.
|
||||
logger.error(
|
||||
"live-action chain ledger append failed (%s: %s); the record is "
|
||||
"still in %s, but the hash chain now has a gap that verify_chain "
|
||||
"will report",
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
path.name,
|
||||
)
|
||||
|
||||
# Sink 2: per-run trace (optional). type="live_action" sits alongside
|
||||
# tool_call / tool_result for this run.
|
||||
|
||||
@@ -13,6 +13,111 @@ def _utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
class AuthMethod(str, Enum):
|
||||
"""How a request proved it was allowed in.
|
||||
|
||||
The distinction that matters is not which mechanism was used but whether the
|
||||
mechanism can name a human. A shared bearer key and loopback dev-trust both
|
||||
authorise; neither identifies. See :attr:`Principal.attributable`.
|
||||
"""
|
||||
|
||||
#: One process-wide static bearer key. Authorises, cannot identify.
|
||||
SHARED_KEY = "shared_key"
|
||||
#: No key configured; the request came from loopback and was trusted.
|
||||
LOOPBACK_TRUST = "loopback_trust"
|
||||
#: A named human authenticated through an identity provider. Not reachable
|
||||
#: today -- listed so that ``attributable`` has a True case to grow into and
|
||||
#: so downstream code branches on the enum rather than on a string it
|
||||
#: invented.
|
||||
FEDERATED_IDENTITY = "federated_identity"
|
||||
|
||||
|
||||
#: Auth methods that can attribute an action to a named human. Everything else
|
||||
#: proves only that *somebody with the secret* acted.
|
||||
ATTRIBUTABLE_AUTH_METHODS: frozenset[AuthMethod] = frozenset({AuthMethod.FEDERATED_IDENTITY})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Principal:
|
||||
"""Who a session belongs to, and how much that claim is worth.
|
||||
|
||||
A ``Principal`` exists so that a number produced by this system can be
|
||||
traced to an actor. What it must never do is *manufacture* an identity the
|
||||
authentication layer could not establish.
|
||||
|
||||
Today the API authenticates with one process-wide static bearer key, or with
|
||||
loopback dev-trust when no key is set. Under either, every request from
|
||||
every person looks identical. :attr:`attributable` is therefore False, and
|
||||
it stays False until an identity provider is wired in. Any control that
|
||||
needs a named human -- four-eyes approval, a MiFID II recipient log, SEC
|
||||
17a-4 attribution -- must check ``attributable`` and refuse rather than
|
||||
quote ``subject`` as if it were a person.
|
||||
|
||||
Attributes:
|
||||
subject: Identifier for the actor. Under a shared secret this is a
|
||||
role label such as ``"shared-key-holder"``, not a person.
|
||||
auth_method: How the request was authorised.
|
||||
attributable: Whether ``subject`` names an actual human. Derived from
|
||||
``auth_method`` at construction, never set by a caller -- a
|
||||
caller-settable flag is a caller-settable lie.
|
||||
tenant: Optional tenant scope, for a future multi-tenant runtime root.
|
||||
display_name: Optional human-readable label for the UI.
|
||||
"""
|
||||
|
||||
subject: str
|
||||
auth_method: AuthMethod
|
||||
tenant: Optional[str] = None
|
||||
display_name: Optional[str] = None
|
||||
attributable: bool = field(init=False, default=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not str(self.subject).strip():
|
||||
raise ValueError("a principal must have a non-empty subject")
|
||||
object.__setattr__(
|
||||
self, "attributable", self.auth_method in ATTRIBUTABLE_AUTH_METHODS
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Serialize the principal to a JSON-safe dictionary.
|
||||
|
||||
Returns:
|
||||
A dictionary carrying ``attributable`` explicitly, so a consumer
|
||||
reading the serialized form cannot miss it.
|
||||
"""
|
||||
return {
|
||||
"subject": self.subject,
|
||||
"auth_method": self.auth_method.value,
|
||||
"tenant": self.tenant,
|
||||
"display_name": self.display_name,
|
||||
"attributable": self.attributable,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> Principal:
|
||||
"""Rebuild a principal from its serialized form.
|
||||
|
||||
``attributable`` in the input is ignored and recomputed from
|
||||
``auth_method``: a stored record whose flag was tampered with, or
|
||||
written by an older version, must not be able to assert attribution the
|
||||
auth method never supported.
|
||||
|
||||
Args:
|
||||
data: Dictionary produced by :meth:`to_dict`.
|
||||
|
||||
Returns:
|
||||
A Principal.
|
||||
|
||||
Raises:
|
||||
ValueError: If ``auth_method`` is missing or unknown.
|
||||
"""
|
||||
return cls(
|
||||
subject=data["subject"],
|
||||
auth_method=AuthMethod(data["auth_method"]),
|
||||
tenant=data.get("tenant"),
|
||||
display_name=data.get("display_name"),
|
||||
)
|
||||
|
||||
|
||||
class SessionStatus(str, Enum):
|
||||
"""Session lifecycle states."""
|
||||
|
||||
@@ -44,6 +149,10 @@ class Session:
|
||||
updated_at: Last update time in ISO format.
|
||||
last_attempt_id: ID of the most recent Attempt.
|
||||
config: Session-level configuration such as model overrides or strategy parameters.
|
||||
owner: Principal the session belongs to, or None for a session created
|
||||
before principals existed or by a path that has no request context.
|
||||
None means "unknown owner", which is different from an
|
||||
unattributable one -- do not collapse the two.
|
||||
"""
|
||||
|
||||
session_id: str = field(default_factory=lambda: uuid.uuid4().hex[:12])
|
||||
@@ -53,6 +162,7 @@ class Session:
|
||||
updated_at: str = field(default_factory=_utc_now_iso)
|
||||
last_attempt_id: Optional[str] = None
|
||||
config: Dict[str, Any] = field(default_factory=dict)
|
||||
owner: Optional[Principal] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Serialize the session to a dictionary.
|
||||
@@ -62,6 +172,7 @@ class Session:
|
||||
"""
|
||||
data = asdict(self)
|
||||
data["status"] = self.status.value
|
||||
data["owner"] = self.owner.to_dict() if self.owner else None
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
@@ -77,6 +188,11 @@ class Session:
|
||||
data = dict(data)
|
||||
if "status" in data:
|
||||
data["status"] = SessionStatus(data["status"])
|
||||
owner = data.get("owner")
|
||||
if isinstance(owner, dict):
|
||||
data["owner"] = Principal.from_dict(owner)
|
||||
elif owner is None:
|
||||
data.pop("owner", None)
|
||||
return cls(**data)
|
||||
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from src.session.models import (
|
||||
Attempt,
|
||||
AttemptStatus,
|
||||
Message,
|
||||
Principal,
|
||||
Session,
|
||||
)
|
||||
from src.session.search import get_shared_index
|
||||
@@ -114,17 +115,28 @@ class SessionService:
|
||||
with self._inflight_lock:
|
||||
self._inflight.discard(session_id)
|
||||
|
||||
def create_session(self, title: str = "", config: Optional[Dict[str, Any]] = None) -> Session:
|
||||
def create_session(
|
||||
self,
|
||||
title: str = "",
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
owner: Optional["Principal"] = None,
|
||||
) -> Session:
|
||||
"""Create a new session.
|
||||
|
||||
Args:
|
||||
title: Session title.
|
||||
config: Session configuration.
|
||||
owner: Principal the session belongs to, from the authenticated
|
||||
request. Optional because sessions are also created by the CLI
|
||||
and by internal paths that have no request context; those get
|
||||
``None``, which reads as "owner unknown" and is deliberately
|
||||
distinct from a principal that authenticated but cannot be
|
||||
attributed to a person (see ``Principal.attributable``).
|
||||
|
||||
Returns:
|
||||
The newly created Session.
|
||||
"""
|
||||
session = Session(title=title, config=config or {})
|
||||
session = Session(title=title, config=config or {}, owner=owner)
|
||||
self.store.create_session(session)
|
||||
self._search_index.index_session(session.session_id, title)
|
||||
self.event_bus.emit(session.session_id, "session.created", {"session_id": session.session_id, "title": title})
|
||||
|
||||
@@ -43,6 +43,40 @@ class BenchCategory(str, Enum):
|
||||
NOISE = "noise"
|
||||
|
||||
|
||||
class ModelTier(str, Enum):
|
||||
"""Model materiality/risk tier — governs the required validation rigor.
|
||||
|
||||
Mirrors the tiering used in bank model-risk-management policies (e.g.
|
||||
SR 11-7): higher tiers demand deeper independent validation before a
|
||||
model can be marked ``ValidationStatus.APPROVED``.
|
||||
"""
|
||||
|
||||
TIER_1_CRITICAL = "tier_1_critical"
|
||||
TIER_2_SIGNIFICANT = "tier_2_significant"
|
||||
TIER_3_LIMITED = "tier_3_limited"
|
||||
|
||||
|
||||
class ValidationStatus(str, Enum):
|
||||
"""Independent-validation lifecycle for a registered model.
|
||||
|
||||
Intended progression::
|
||||
|
||||
UNVALIDATED -> IN_VALIDATION -> VALIDATED -> APPROVED
|
||||
\\-> REJECTED
|
||||
|
||||
``APPROVED`` is only reachable from ``VALIDATED`` (or ``APPROVED``
|
||||
itself, for an idempotent re-approval) — see
|
||||
``validate_validation_status_transition``. An unvalidated model can
|
||||
never be marked approved.
|
||||
"""
|
||||
|
||||
UNVALIDATED = "unvalidated"
|
||||
IN_VALIDATION = "in_validation"
|
||||
VALIDATED = "validated"
|
||||
APPROVED = "approved"
|
||||
REJECTED = "rejected"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Artifact:
|
||||
"""Persisted research artifact record."""
|
||||
@@ -73,6 +107,29 @@ class Artifact:
|
||||
updated_at: str = ""
|
||||
disabled_at: str | None = None
|
||||
disabled_reason: str | None = None
|
||||
# Model governance (2026-08-06) — makes each artifact expressible as a
|
||||
# registered model under a firm's model-risk-management policy: who
|
||||
# built it, who owns it, who independently validated it, who signed
|
||||
# off, at what version, at what materiality tier, for what declared
|
||||
# use, with what known limitations. Every field here is optional at the
|
||||
# dataclass level so pre-existing bare Artifact construction (factors/
|
||||
# strategies registered before this governance layer existed, and the
|
||||
# SDM tools that still build bare artifacts) keeps working unchanged.
|
||||
# Use ``validate_model_registration`` to enforce the non-empty
|
||||
# intended_use/limitations rule when a caller wants full governance
|
||||
# rigor, and ``validate_validation_status_transition`` to guard the
|
||||
# unvalidated-can't-become-approved state machine.
|
||||
developer: str | None = None
|
||||
owner: str | None = None
|
||||
validator: str | None = None
|
||||
approver: str | None = None
|
||||
model_version: str | None = None
|
||||
artifact_version: str | None = None
|
||||
model_tier: ModelTier | None = None
|
||||
intended_use: str | None = None
|
||||
limitations: str | None = None
|
||||
validation_status: ValidationStatus = ValidationStatus.UNVALIDATED
|
||||
validation_date: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -118,3 +175,82 @@ class DecaySnapshot:
|
||||
consecutive_warnings: int = 0
|
||||
detail: str | None = None
|
||||
created_at: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model-governance validation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Pure functions — no I/O, no datetime.now(). All timestamps (validation_date
|
||||
# included) are supplied by the caller, same convention as created_at/
|
||||
# updated_at elsewhere in this module.
|
||||
|
||||
|
||||
def is_four_eyes_violation(artifact: Artifact) -> bool:
|
||||
"""Report whether *artifact* was both developed and approved by the same person.
|
||||
|
||||
The four-eyes (maker-checker) principle requires the developer and
|
||||
approver roles to be held by different people. This function only
|
||||
*detects* the condition — it does not raise or block, since whether to
|
||||
reject a same-person dev/approve pair is a firm-policy decision, not a
|
||||
library decision.
|
||||
|
||||
Args:
|
||||
artifact: The artifact/model record to inspect.
|
||||
|
||||
Returns:
|
||||
True when both ``developer`` and ``approver`` are set and refer to
|
||||
the same person (case-insensitive, whitespace-trimmed comparison).
|
||||
False when either role is unset (nothing to compare) or they differ.
|
||||
"""
|
||||
if not artifact.developer or not artifact.approver:
|
||||
return False
|
||||
return artifact.developer.strip().casefold() == artifact.approver.strip().casefold()
|
||||
|
||||
|
||||
def validate_validation_status_transition(
|
||||
current: ValidationStatus, new: ValidationStatus
|
||||
) -> None:
|
||||
"""Guard the model-validation state machine.
|
||||
|
||||
The only hard rule: a model cannot be marked ``APPROVED`` unless it is
|
||||
already ``VALIDATED`` (re-approving an already-``APPROVED`` model is a
|
||||
harmless no-op transition). Every other transition is left to caller
|
||||
discretion.
|
||||
|
||||
Args:
|
||||
current: The model's validation status before the change.
|
||||
new: The requested validation status.
|
||||
|
||||
Raises:
|
||||
ValueError: if *new* is ``APPROVED`` while *current* is anything
|
||||
other than ``VALIDATED`` or ``APPROVED``.
|
||||
"""
|
||||
if new is ValidationStatus.APPROVED and current not in (
|
||||
ValidationStatus.VALIDATED,
|
||||
ValidationStatus.APPROVED,
|
||||
):
|
||||
raise ValueError(
|
||||
"cannot mark a model APPROVED from validation_status="
|
||||
f"{current.value!r}; it must reach VALIDATED first"
|
||||
)
|
||||
|
||||
|
||||
def validate_model_registration(artifact: Artifact) -> None:
|
||||
"""Validate that *artifact* carries the minimum model-registration fields.
|
||||
|
||||
A model registration without a stated intended use or a stated
|
||||
limitation is, for governance purposes, not actually registered — so
|
||||
both fields are required and must be non-blank.
|
||||
|
||||
Args:
|
||||
artifact: The artifact/model record to validate.
|
||||
|
||||
Raises:
|
||||
ValueError: if ``intended_use`` or ``limitations`` is missing or
|
||||
consists only of whitespace.
|
||||
"""
|
||||
if not artifact.intended_use or not artifact.intended_use.strip():
|
||||
raise ValueError("intended_use is required and must be non-empty")
|
||||
if not artifact.limitations or not artifact.limitations.strip():
|
||||
raise ValueError("limitations is required and must be non-empty")
|
||||
@@ -25,6 +25,9 @@ from src.strategy_store.models import (
|
||||
BenchResult,
|
||||
DecaySignal,
|
||||
DecaySnapshot,
|
||||
ModelTier,
|
||||
ValidationStatus,
|
||||
validate_validation_status_transition,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -33,6 +36,30 @@ from src.strategy_store.models import (
|
||||
|
||||
_DEFAULT_DB_PATH = Path.home() / ".vibe-trading" / "strategy_store.db"
|
||||
|
||||
# Schema version introduced by the model-governance migration (adds the
|
||||
# developer/owner/validator/approver/model_version/artifact_version/
|
||||
# model_tier/intended_use/limitations/validation_status/validation_date
|
||||
# columns to `artifacts`). Bumping PRAGMA user_version documents intent;
|
||||
# the migration itself is idempotent via a column-existence check so it is
|
||||
# safe to run unconditionally on every open regardless of the stamped
|
||||
# version (covers DBs that were hand-edited or partially migrated).
|
||||
_SCHEMA_VERSION_GOVERNANCE = 2
|
||||
|
||||
# column_name -> DDL type/default fragment used by ALTER TABLE ADD COLUMN.
|
||||
_GOVERNANCE_COLUMNS: dict[str, str] = {
|
||||
"developer": "TEXT",
|
||||
"owner": "TEXT",
|
||||
"validator": "TEXT",
|
||||
"approver": "TEXT",
|
||||
"model_version": "TEXT",
|
||||
"artifact_version": "TEXT",
|
||||
"model_tier": "TEXT",
|
||||
"intended_use": "TEXT",
|
||||
"limitations": "TEXT",
|
||||
"validation_status": "TEXT DEFAULT 'unvalidated'",
|
||||
"validation_date": "TEXT",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -209,8 +236,27 @@ class SqliteStrategyStore:
|
||||
)
|
||||
if self._conn.execute("PRAGMA user_version").fetchone()[0] < 1:
|
||||
self._conn.execute("PRAGMA user_version=1")
|
||||
self._migrate_governance_columns()
|
||||
if self._conn.execute("PRAGMA user_version").fetchone()[0] < _SCHEMA_VERSION_GOVERNANCE:
|
||||
self._conn.execute(f"PRAGMA user_version={_SCHEMA_VERSION_GOVERNANCE}")
|
||||
self._conn.commit()
|
||||
|
||||
def _migrate_governance_columns(self) -> None:
|
||||
"""Add model-governance columns to ``artifacts`` if missing.
|
||||
|
||||
Idempotent: checks ``PRAGMA table_info`` for each column before
|
||||
issuing ``ALTER TABLE ADD COLUMN``, so it is safe to call on every
|
||||
connection open — including against a pre-governance database that
|
||||
predates this migration (the primary backward-compatibility path)
|
||||
and against re-running this exact method twice in the same process.
|
||||
"""
|
||||
existing_columns = {
|
||||
row["name"] for row in self._conn.execute("PRAGMA table_info(artifacts)")
|
||||
}
|
||||
for column, ddl in _GOVERNANCE_COLUMNS.items():
|
||||
if column not in existing_columns:
|
||||
self._conn.execute(f"ALTER TABLE artifacts ADD COLUMN {column} {ddl}")
|
||||
|
||||
# -- Write transaction --------------------------------------------------
|
||||
|
||||
@contextmanager
|
||||
@@ -253,6 +299,21 @@ class SqliteStrategyStore:
|
||||
updated_at=row["updated_at"],
|
||||
disabled_at=row["disabled_at"],
|
||||
disabled_reason=row["disabled_reason"],
|
||||
developer=row["developer"],
|
||||
owner=row["owner"],
|
||||
validator=row["validator"],
|
||||
approver=row["approver"],
|
||||
model_version=row["model_version"],
|
||||
artifact_version=row["artifact_version"],
|
||||
model_tier=ModelTier(row["model_tier"]) if row["model_tier"] else None,
|
||||
intended_use=row["intended_use"],
|
||||
limitations=row["limitations"],
|
||||
validation_status=(
|
||||
ValidationStatus(row["validation_status"])
|
||||
if row["validation_status"]
|
||||
else ValidationStatus.UNVALIDATED
|
||||
),
|
||||
validation_date=row["validation_date"],
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -303,6 +364,13 @@ class SqliteStrategyStore:
|
||||
@_synchronized
|
||||
def register_artifact(self, artifact: Artifact) -> str:
|
||||
"""Register a new artifact. Returns the artifact_id."""
|
||||
# A brand-new record has no prior validation history, so its implicit
|
||||
# "current" state is UNVALIDATED — registering directly with
|
||||
# validation_status=APPROVED would skip the VALIDATED step.
|
||||
validate_validation_status_transition(
|
||||
ValidationStatus.UNVALIDATED, artifact.validation_status
|
||||
)
|
||||
|
||||
now = _now_iso()
|
||||
artifact_id = artifact.id or _new_artifact_id()
|
||||
created_at = artifact.created_at or now
|
||||
@@ -316,8 +384,12 @@ class SqliteStrategyStore:
|
||||
theme, columns_required, decay_horizon, signal_definition,
|
||||
entry_rules, exit_rules, position_sizing, universe,
|
||||
signal_engine_path, run_dir, hypothesis_id, status,
|
||||
created_at, updated_at, disabled_at, disabled_reason
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
created_at, updated_at, disabled_at, disabled_reason,
|
||||
developer, owner, validator, approver, model_version,
|
||||
artifact_version, model_tier, intended_use, limitations,
|
||||
validation_status, validation_date
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
artifact_id,
|
||||
@@ -342,6 +414,17 @@ class SqliteStrategyStore:
|
||||
now,
|
||||
artifact.disabled_at,
|
||||
artifact.disabled_reason,
|
||||
artifact.developer,
|
||||
artifact.owner,
|
||||
artifact.validator,
|
||||
artifact.approver,
|
||||
artifact.model_version,
|
||||
artifact.artifact_version,
|
||||
artifact.model_tier.value if artifact.model_tier else None,
|
||||
artifact.intended_use,
|
||||
artifact.limitations,
|
||||
artifact.validation_status.value,
|
||||
artifact.validation_date,
|
||||
),
|
||||
)
|
||||
except sqlite3.IntegrityError as exc:
|
||||
@@ -446,6 +529,12 @@ class SqliteStrategyStore:
|
||||
if existing is None:
|
||||
return None
|
||||
|
||||
# Structural enforcement of the validation state machine: an
|
||||
# unvalidated model can never be updated straight into APPROVED.
|
||||
validate_validation_status_transition(
|
||||
existing.validation_status, artifact.validation_status
|
||||
)
|
||||
|
||||
now = _now_iso()
|
||||
with self._write_transaction():
|
||||
self._conn.execute(
|
||||
@@ -457,7 +546,11 @@ class SqliteStrategyStore:
|
||||
entry_rules = ?, exit_rules = ?, position_sizing = ?,
|
||||
universe = ?, signal_engine_path = ?, run_dir = ?,
|
||||
hypothesis_id = ?, status = ?, updated_at = ?,
|
||||
disabled_at = ?, disabled_reason = ?
|
||||
disabled_at = ?, disabled_reason = ?,
|
||||
developer = ?, owner = ?, validator = ?, approver = ?,
|
||||
model_version = ?, artifact_version = ?, model_tier = ?,
|
||||
intended_use = ?, limitations = ?, validation_status = ?,
|
||||
validation_date = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(
|
||||
@@ -481,6 +574,17 @@ class SqliteStrategyStore:
|
||||
now,
|
||||
artifact.disabled_at,
|
||||
artifact.disabled_reason,
|
||||
artifact.developer,
|
||||
artifact.owner,
|
||||
artifact.validator,
|
||||
artifact.approver,
|
||||
artifact.model_version,
|
||||
artifact.artifact_version,
|
||||
artifact.model_tier.value if artifact.model_tier else None,
|
||||
artifact.intended_use,
|
||||
artifact.limitations,
|
||||
artifact.validation_status.value,
|
||||
artifact.validation_date,
|
||||
artifact.id,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -28,6 +28,8 @@ from src.strategy_store.models import (
|
||||
BenchResult,
|
||||
DecaySnapshot,
|
||||
DecaySignal,
|
||||
ValidationStatus,
|
||||
validate_validation_status_transition,
|
||||
)
|
||||
|
||||
|
||||
@@ -168,6 +170,12 @@ class InMemoryStrategyStore:
|
||||
f"artifact '{artifact.name}' already exists in "
|
||||
f"universe '{artifact.universe}'"
|
||||
)
|
||||
# A brand-new record has no prior validation history, so its implicit
|
||||
# "current" state is UNVALIDATED — registering directly with
|
||||
# validation_status=APPROVED would skip the VALIDATED step.
|
||||
validate_validation_status_transition(
|
||||
ValidationStatus.UNVALIDATED, artifact.validation_status
|
||||
)
|
||||
now = _now_iso()
|
||||
artifact_id = artifact.id or _new_artifact_id()
|
||||
stored = replace(
|
||||
@@ -251,9 +259,16 @@ class InMemoryStrategyStore:
|
||||
|
||||
Returns the updated artifact, or ``None`` if the ID is not found.
|
||||
"""
|
||||
if artifact.id not in self._artifacts:
|
||||
existing = self._artifacts.get(artifact.id)
|
||||
if existing is None:
|
||||
return None
|
||||
|
||||
# Structural enforcement of the validation state machine: an
|
||||
# unvalidated model can never be updated straight into APPROVED.
|
||||
validate_validation_status_transition(
|
||||
existing.validation_status, artifact.validation_status
|
||||
)
|
||||
|
||||
now = _now_iso()
|
||||
stored = replace(artifact, updated_at=now)
|
||||
self._artifacts[artifact.id] = stored
|
||||
|
||||
@@ -11,6 +11,8 @@ from src.strategy_store.models import (
|
||||
Artifact,
|
||||
ArtifactStatus,
|
||||
ArtifactType,
|
||||
ModelTier,
|
||||
ValidationStatus,
|
||||
)
|
||||
from src.strategy_store._shared import get_store as _get_store
|
||||
|
||||
@@ -83,10 +85,84 @@ class SdmRegisterTool(BaseTool):
|
||||
"type": "string",
|
||||
"description": "Linked hypothesis ID",
|
||||
},
|
||||
# --- model governance -------------------------------------------
|
||||
# Without these an artifact is an UNREGISTERED MODEL under any
|
||||
# firm's own model-risk policy: nobody owns it, nobody validated
|
||||
# it, and nothing records what it may and may not be used for.
|
||||
"developer": {
|
||||
"type": "string",
|
||||
"description": "Who built this model. Kept separate from approver on purpose.",
|
||||
},
|
||||
"owner": {
|
||||
"type": "string",
|
||||
"description": "Who is accountable for it in production.",
|
||||
},
|
||||
"validator": {
|
||||
"type": "string",
|
||||
"description": "Who independently validated it.",
|
||||
},
|
||||
"approver": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Who approved it for use. Same person as developer is a "
|
||||
"four-eyes violation -- allowed here, but reported by "
|
||||
"sdm_status so it can be seen."
|
||||
),
|
||||
},
|
||||
"model_version": {"type": "string", "description": "Version of the model itself."},
|
||||
"artifact_version": {
|
||||
"type": "string",
|
||||
"description": "Version of the artifact this model is bound to.",
|
||||
},
|
||||
"model_tier": {
|
||||
"type": "string",
|
||||
"enum": ["tier_1_critical", "tier_2_significant", "tier_3_limited"],
|
||||
"description": "Materiality tier; decides how much validation is required.",
|
||||
},
|
||||
"intended_use": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"What this model may be used for. Required whenever any "
|
||||
"governance field is supplied."
|
||||
),
|
||||
},
|
||||
"limitations": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Where it stops being valid. Required whenever any "
|
||||
"governance field is supplied -- a model registration with "
|
||||
"no stated limitations is not a registration."
|
||||
),
|
||||
},
|
||||
"validation_status": {
|
||||
"type": "string",
|
||||
"enum": ["unvalidated", "in_validation", "validated", "approved", "rejected"],
|
||||
"description": (
|
||||
"Lifecycle state. Cannot jump straight to 'approved' from "
|
||||
"'unvalidated'; the store refuses it."
|
||||
),
|
||||
},
|
||||
"validation_date": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"ISO date the validation was performed. Never auto-filled -- "
|
||||
"stamping a date on a model nobody validated would fabricate "
|
||||
"the evidence the field exists to carry."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["artifact_type", "name", "universe"],
|
||||
}
|
||||
|
||||
#: Governance fields. Supplying any one of them means this registration is
|
||||
#: claiming to be a model record, which makes intended_use and limitations
|
||||
#: mandatory -- a claim without those two cannot be reviewed.
|
||||
_GOVERNANCE_FIELDS = (
|
||||
"developer", "owner", "validator", "approver", "model_version",
|
||||
"artifact_version", "model_tier", "intended_use", "limitations",
|
||||
"validation_status", "validation_date",
|
||||
)
|
||||
|
||||
def execute(self, **kwargs: Any) -> str:
|
||||
"""Build an Artifact from kwargs, register it, and return the record."""
|
||||
try:
|
||||
@@ -114,6 +190,7 @@ class SdmRegisterTool(BaseTool):
|
||||
signal_engine_path=kwargs.get("signal_engine_path"),
|
||||
hypothesis_id=kwargs.get("hypothesis_id"),
|
||||
status=ArtifactStatus.CREATED,
|
||||
**self._governance_kwargs(kwargs),
|
||||
)
|
||||
|
||||
artifact_id = store.register_artifact(artifact)
|
||||
@@ -121,3 +198,46 @@ class SdmRegisterTool(BaseTool):
|
||||
return _ok({"artifact": asdict(stored)}) # type: ignore[arg-type]
|
||||
except Exception as exc:
|
||||
return _error(exc)
|
||||
|
||||
def _governance_kwargs(self, kwargs: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Extract the model-governance fields, enforcing the pair that matters.
|
||||
|
||||
Supplying any governance field means this registration claims to be a
|
||||
model record. Two of them are then mandatory: ``intended_use`` says what
|
||||
the model may be used for and ``limitations`` says where it stops being
|
||||
valid. A registration missing either cannot be reviewed by anyone, which
|
||||
makes it indistinguishable from no registration at all.
|
||||
|
||||
Args:
|
||||
kwargs: Raw tool arguments.
|
||||
|
||||
Returns:
|
||||
Keyword arguments to pass to :class:`Artifact`, empty when no
|
||||
governance field was supplied.
|
||||
|
||||
Raises:
|
||||
ValueError: If governance fields were supplied without both
|
||||
``intended_use`` and ``limitations``, or if an enum value is
|
||||
unknown.
|
||||
"""
|
||||
supplied = {
|
||||
f: kwargs[f]
|
||||
for f in self._GOVERNANCE_FIELDS
|
||||
if kwargs.get(f) not in (None, "")
|
||||
}
|
||||
if not supplied:
|
||||
return {}
|
||||
|
||||
missing = [f for f in ("intended_use", "limitations") if f not in supplied]
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"model governance fields were supplied ({sorted(supplied)}), so "
|
||||
f"{missing} are required too. A model record without a stated "
|
||||
"intended use and stated limitations cannot be reviewed."
|
||||
)
|
||||
|
||||
if "model_tier" in supplied:
|
||||
supplied["model_tier"] = ModelTier(supplied["model_tier"])
|
||||
if "validation_status" in supplied:
|
||||
supplied["validation_status"] = ValidationStatus(supplied["validation_status"])
|
||||
return supplied
|
||||
@@ -4,6 +4,8 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import asdict
|
||||
|
||||
from src.strategy_store.models import is_four_eyes_violation
|
||||
from typing import Any
|
||||
|
||||
from src.agent.tools import BaseTool
|
||||
@@ -40,6 +42,31 @@ def _artifact_to_dict(artifact: Any) -> dict[str, Any]:
|
||||
if hasattr(d["category"], "value")
|
||||
else d["category"]
|
||||
)
|
||||
for enum_field in ("model_tier", "validation_status"):
|
||||
if d.get(enum_field) is not None and hasattr(d[enum_field], "value"):
|
||||
d[enum_field] = d[enum_field].value
|
||||
|
||||
# Model-governance readout. Reported on every artifact, including the ones
|
||||
# carrying no governance data at all -- "this model is unregistered" is the
|
||||
# finding, and omitting the block when it is empty would hide it.
|
||||
governance = {
|
||||
"registered": any(
|
||||
d.get(f) for f in ("developer", "owner", "validator", "approver",
|
||||
"intended_use", "limitations")
|
||||
),
|
||||
"four_eyes_violation": is_four_eyes_violation(artifact),
|
||||
}
|
||||
if not governance["registered"]:
|
||||
governance["note"] = (
|
||||
"no owner, validator, approver, intended use or limitations recorded: "
|
||||
"this is an unregistered model under any model-risk policy"
|
||||
)
|
||||
elif governance["four_eyes_violation"]:
|
||||
governance["note"] = (
|
||||
f"developer and approver are the same person ({d.get('developer')!r}); "
|
||||
"the approval carries no independent review"
|
||||
)
|
||||
d["governance"] = governance
|
||||
return d
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Tests for the multiple-testing block wired into run_bench.
|
||||
|
||||
`n_alphas_tested` has been carried in the bench result since it was written and
|
||||
has only ever been displayed. These tests pin the correction that now uses it,
|
||||
and pin that adding it changed nothing else about the result.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
from src.factors.bench_runner import run_bench
|
||||
from src.quantlib.multipletesting import MIN_OBSERVATIONS, expected_maximum_sharpe
|
||||
|
||||
|
||||
def _make_panel(n_symbols: int = 10, n_days: int = 120):
|
||||
rng = np.random.default_rng(42)
|
||||
dates = pd.date_range("2020-01-01", periods=n_days, freq="B")
|
||||
symbols = [f"SYM{i:03d}" for i in range(n_symbols)]
|
||||
close = pd.DataFrame(
|
||||
np.cumsum(rng.standard_normal((n_days, n_symbols)), axis=0) + 100,
|
||||
index=dates,
|
||||
columns=symbols,
|
||||
)
|
||||
return {"close": close, "open": close, "high": close, "low": close, "volume": close}
|
||||
|
||||
|
||||
def _setup(monkeypatch, n_alphas=8, n_days=120):
|
||||
panel = _make_panel(n_days=n_days)
|
||||
return_df = panel["close"].pct_change().shift(-1).iloc[:-1]
|
||||
rng = np.random.default_rng(7)
|
||||
|
||||
reg = MagicMock()
|
||||
reg.list.return_value = [f"alpha_{i}" for i in range(n_alphas)]
|
||||
reg.get.return_value = MagicMock(meta={"theme": ["test"], "formula_latex": ""})
|
||||
reg.compute.side_effect = lambda aid, p: pd.DataFrame(
|
||||
rng.standard_normal(p["close"].shape), index=p["close"].index, columns=p["close"].columns
|
||||
)
|
||||
|
||||
monkeypatch.setattr("src.factors.bench_runner._load_universe_panel", lambda u, p: panel)
|
||||
monkeypatch.setattr("src.factors.bench_runner._compute_forward_returns", lambda p: return_df)
|
||||
monkeypatch.setattr("src.factors.bench_runner.get_default_registry", lambda: reg)
|
||||
return reg
|
||||
|
||||
|
||||
def test_bench_reports_a_multiple_testing_block(monkeypatch):
|
||||
reg = _setup(monkeypatch, n_alphas=8)
|
||||
result = run_bench("test_zoo", "mock", "2020-2021", registry=reg)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
block = result["multiple_testing"]
|
||||
assert block["applicable"] is True
|
||||
assert block["n_trials"] == result["n_alphas_tested"]
|
||||
assert block["statistic"] == "ic_information_ratio"
|
||||
|
||||
|
||||
def test_the_haircut_is_exactly_the_expected_maximum_under_the_null(monkeypatch):
|
||||
reg = _setup(monkeypatch, n_alphas=10)
|
||||
result = run_bench("test_zoo", "mock", "2020-2021", registry=reg)
|
||||
block = result["multiple_testing"]
|
||||
|
||||
irs = np.array([float(r["ir"]) for r in result["rows"]])
|
||||
expected = expected_maximum_sharpe(len(irs), float(irs.std(ddof=1)))
|
||||
|
||||
assert block["expected_max_ir_under_null"] == pytest.approx(expected, abs=1e-6)
|
||||
assert block["ir_haircut"] == pytest.approx(block["best_ir"] - expected, abs=1e-6)
|
||||
|
||||
|
||||
def test_the_best_alpha_named_by_the_block_is_the_best_alpha_in_the_rows(monkeypatch):
|
||||
reg = _setup(monkeypatch, n_alphas=12)
|
||||
result = run_bench("test_zoo", "mock", "2020-2021", registry=reg)
|
||||
block = result["multiple_testing"]
|
||||
|
||||
best_by_ir = max(result["rows"], key=lambda r: r["ir"])
|
||||
assert block["best_id"] == best_by_ir["id"]
|
||||
assert block["best_ir"] == pytest.approx(best_by_ir["ir"])
|
||||
|
||||
|
||||
def test_pure_noise_alphas_do_not_survive_deflation(monkeypatch):
|
||||
# Every alpha here is random noise against the returns, so the best IR of
|
||||
# the batch is the maximum of N draws and nothing more. The correction is
|
||||
# supposed to say so.
|
||||
reg = _setup(monkeypatch, n_alphas=16)
|
||||
result = run_bench("test_zoo", "mock", "2020-2021", registry=reg)
|
||||
block = result["multiple_testing"]
|
||||
|
||||
assert block["deflated_probability"] is not None
|
||||
assert block["survives_deflation"] is False
|
||||
|
||||
|
||||
def test_a_single_alpha_has_no_search_to_correct_for(monkeypatch):
|
||||
reg = _setup(monkeypatch, n_alphas=1)
|
||||
result = run_bench("test_zoo", "mock", "2020-2021", registry=reg)
|
||||
assert result["multiple_testing"]["applicable"] is False
|
||||
|
||||
|
||||
def test_a_short_ic_sample_reports_no_deflated_statistic_rather_than_a_bad_one(monkeypatch):
|
||||
# Too few IC observations for the moments behind the statistic to mean
|
||||
# anything: the block must say so instead of returning a number.
|
||||
reg = _setup(monkeypatch, n_alphas=6, n_days=MIN_OBSERVATIONS + 2)
|
||||
result = run_bench("test_zoo", "mock", "2020-2021", registry=reg)
|
||||
block = result["multiple_testing"]
|
||||
|
||||
if block["applicable"] and block["ic_observations"] < MIN_OBSERVATIONS:
|
||||
assert block["deflated_probability"] is None
|
||||
assert block["survives_deflation"] is None
|
||||
assert "note" in block
|
||||
|
||||
|
||||
def test_wiring_the_block_in_did_not_change_any_existing_key(monkeypatch):
|
||||
reg = _setup(monkeypatch, n_alphas=5)
|
||||
result = run_bench("test_zoo", "mock", "2020-2021", registry=reg)
|
||||
|
||||
# The keys the bench has always returned are all still present, and
|
||||
# `n_alphas_tested` still means what it always meant.
|
||||
for key in (
|
||||
"status", "n_alphas_tested", "n_skipped", "alive", "reversed", "dead",
|
||||
"by_theme", "top5_by_ir", "dead_examples", "rows", "skipped", "meta",
|
||||
"wall_seconds",
|
||||
):
|
||||
assert key in result, f"existing key {key} disappeared"
|
||||
assert result["n_alphas_tested"] == len(result["rows"])
|
||||
@@ -0,0 +1,218 @@
|
||||
"""Tests for the session Principal and the auth surface that produces it.
|
||||
|
||||
The point of these tests is not that a Principal exists. It is that the
|
||||
Principal cannot claim an attribution the authentication layer did not earn --
|
||||
which is the only thing that makes it better than no principal at all.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from src.api.security import LOOPBACK_SUBJECT, SHARED_KEY_SUBJECT
|
||||
from src.session.models import (
|
||||
ATTRIBUTABLE_AUTH_METHODS,
|
||||
AuthMethod,
|
||||
Principal,
|
||||
Session,
|
||||
SessionStatus,
|
||||
)
|
||||
|
||||
|
||||
# --- the honesty invariant ---
|
||||
|
||||
|
||||
def test_a_shared_key_principal_is_not_attributable():
|
||||
# Everyone holding the one static bearer key authenticates identically, so
|
||||
# no action taken under it can be attributed to a named human.
|
||||
principal = Principal(subject=SHARED_KEY_SUBJECT, auth_method=AuthMethod.SHARED_KEY)
|
||||
assert principal.attributable is False
|
||||
|
||||
|
||||
def test_a_loopback_principal_is_not_attributable():
|
||||
principal = Principal(subject=LOOPBACK_SUBJECT, auth_method=AuthMethod.LOOPBACK_TRUST)
|
||||
assert principal.attributable is False
|
||||
|
||||
|
||||
def test_only_federated_identity_is_attributable():
|
||||
assert ATTRIBUTABLE_AUTH_METHODS == frozenset({AuthMethod.FEDERATED_IDENTITY})
|
||||
named = Principal(subject="alice@example.com", auth_method=AuthMethod.FEDERATED_IDENTITY)
|
||||
assert named.attributable is True
|
||||
|
||||
|
||||
def test_attributable_cannot_be_set_by_the_caller():
|
||||
# A caller-settable flag is a caller-settable lie.
|
||||
with pytest.raises(TypeError):
|
||||
Principal( # type: ignore[call-arg]
|
||||
subject="alice",
|
||||
auth_method=AuthMethod.SHARED_KEY,
|
||||
attributable=True,
|
||||
)
|
||||
|
||||
|
||||
def test_attributable_cannot_be_mutated_after_construction():
|
||||
principal = Principal(subject="x", auth_method=AuthMethod.SHARED_KEY)
|
||||
with pytest.raises(Exception):
|
||||
principal.attributable = True # type: ignore[misc]
|
||||
|
||||
|
||||
def test_a_tampered_stored_flag_is_recomputed_not_trusted():
|
||||
# A record written by an older version, or edited by hand, must not be able
|
||||
# to assert attribution its auth method never supported.
|
||||
forged = {
|
||||
"subject": "alice",
|
||||
"auth_method": AuthMethod.SHARED_KEY.value,
|
||||
"attributable": True,
|
||||
"tenant": None,
|
||||
"display_name": None,
|
||||
}
|
||||
assert Principal.from_dict(forged).attributable is False
|
||||
|
||||
|
||||
def test_an_empty_subject_is_rejected():
|
||||
for bad in ("", " "):
|
||||
with pytest.raises(ValueError, match="non-empty subject"):
|
||||
Principal(subject=bad, auth_method=AuthMethod.SHARED_KEY)
|
||||
|
||||
|
||||
def test_unknown_auth_method_rejected():
|
||||
with pytest.raises(ValueError):
|
||||
Principal.from_dict({"subject": "a", "auth_method": "telepathy"})
|
||||
|
||||
|
||||
# --- serialization round trip ---
|
||||
|
||||
|
||||
def test_principal_round_trips_through_dict():
|
||||
original = Principal(
|
||||
subject="alice@example.com",
|
||||
auth_method=AuthMethod.FEDERATED_IDENTITY,
|
||||
tenant="desk-a",
|
||||
display_name="Alice",
|
||||
)
|
||||
restored = Principal.from_dict(original.to_dict())
|
||||
assert restored == original
|
||||
assert restored.attributable is True
|
||||
|
||||
|
||||
def test_serialized_form_carries_attributable_explicitly():
|
||||
# A consumer reading the JSON must not have to know the enum table to learn
|
||||
# whether the identity is real.
|
||||
data = Principal(subject="x", auth_method=AuthMethod.SHARED_KEY).to_dict()
|
||||
assert data["attributable"] is False
|
||||
assert data["auth_method"] == "shared_key"
|
||||
|
||||
|
||||
# --- Session integration, backward compatible ---
|
||||
|
||||
|
||||
def test_a_session_without_an_owner_still_constructs():
|
||||
session = Session(title="legacy")
|
||||
assert session.owner is None
|
||||
|
||||
|
||||
def test_an_old_serialized_session_with_no_owner_key_still_loads():
|
||||
old = {
|
||||
"session_id": "abc123",
|
||||
"title": "before principals existed",
|
||||
"status": "active",
|
||||
"created_at": "2026-01-01T00:00:00+00:00",
|
||||
"updated_at": "2026-01-01T00:00:00+00:00",
|
||||
"last_attempt_id": None,
|
||||
"config": {},
|
||||
}
|
||||
session = Session.from_dict(old)
|
||||
assert session.owner is None
|
||||
assert session.status is SessionStatus.ACTIVE
|
||||
assert session.session_id == "abc123"
|
||||
|
||||
|
||||
def test_a_session_with_an_owner_round_trips():
|
||||
principal = Principal(subject=SHARED_KEY_SUBJECT, auth_method=AuthMethod.SHARED_KEY)
|
||||
session = Session(title="owned", owner=principal)
|
||||
restored = Session.from_dict(session.to_dict())
|
||||
|
||||
assert restored.owner == principal
|
||||
assert restored.owner is not None and restored.owner.attributable is False
|
||||
|
||||
|
||||
def test_session_to_dict_is_json_serializable():
|
||||
import json
|
||||
|
||||
session = Session(
|
||||
title="t",
|
||||
owner=Principal(subject="x", auth_method=AuthMethod.LOOPBACK_TRUST),
|
||||
)
|
||||
text = json.dumps(session.to_dict())
|
||||
assert '"attributable": false' in text
|
||||
|
||||
|
||||
def test_unknown_owner_and_unattributable_owner_are_different_states():
|
||||
# None means "we do not know who"; an unattributable principal means "we
|
||||
# know the request was authorised but cannot name a person". Collapsing the
|
||||
# two would lose the distinction a governance review needs.
|
||||
unknown = Session(title="a")
|
||||
unattributable = Session(
|
||||
title="b", owner=Principal(subject=LOOPBACK_SUBJECT, auth_method=AuthMethod.LOOPBACK_TRUST)
|
||||
)
|
||||
assert unknown.owner is None
|
||||
assert unattributable.owner is not None
|
||||
assert unattributable.owner.attributable is False
|
||||
|
||||
|
||||
# --- the auth surface returns one ---
|
||||
|
||||
|
||||
def test_validate_api_auth_returns_a_shared_key_principal(monkeypatch):
|
||||
from src.api import security
|
||||
|
||||
monkeypatch.setattr(security, "_configured_api_key", lambda: "secret-key")
|
||||
|
||||
class _Cred:
|
||||
credentials = "secret-key"
|
||||
|
||||
class _Req:
|
||||
method = "GET"
|
||||
headers: dict[str, str] = {}
|
||||
|
||||
class client: # noqa: N801
|
||||
host = "127.0.0.1"
|
||||
|
||||
principal = security._validate_api_auth(request=_Req(), cred=_Cred())
|
||||
assert principal.auth_method is AuthMethod.SHARED_KEY
|
||||
assert principal.subject == SHARED_KEY_SUBJECT
|
||||
assert principal.attributable is False
|
||||
|
||||
|
||||
def test_validate_api_auth_returns_a_loopback_principal_when_no_key(monkeypatch):
|
||||
from src.api import security
|
||||
|
||||
monkeypatch.setattr(security, "_configured_api_key", lambda: "")
|
||||
monkeypatch.setattr(security, "_is_local_client", lambda request: True)
|
||||
|
||||
class _Req:
|
||||
method = "GET"
|
||||
headers: dict[str, str] = {}
|
||||
|
||||
principal = security._validate_api_auth(request=_Req(), cred=None)
|
||||
assert principal.auth_method is AuthMethod.LOOPBACK_TRUST
|
||||
assert principal.attributable is False
|
||||
|
||||
|
||||
def test_a_bad_credential_still_raises_and_returns_no_principal(monkeypatch):
|
||||
from fastapi import HTTPException
|
||||
|
||||
from src.api import security
|
||||
|
||||
monkeypatch.setattr(security, "_configured_api_key", lambda: "secret-key")
|
||||
|
||||
class _Cred:
|
||||
credentials = "wrong"
|
||||
|
||||
class _Req:
|
||||
method = "GET"
|
||||
headers: dict[str, str] = {}
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
security._validate_api_auth(request=_Req(), cred=_Cred())
|
||||
assert excinfo.value.status_code == 401
|
||||
@@ -0,0 +1,130 @@
|
||||
"""The Principal must actually reach a stored Session.
|
||||
|
||||
`Session.owner` and `_validate_api_auth` returning a Principal were both added
|
||||
without anything connecting them, so every session's owner was None. A field
|
||||
that is always None carries the same information as no field, which is why the
|
||||
wiring needs its own tests rather than relying on the unit tests of each half.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from src.session.events import EventBus
|
||||
from src.session.models import AuthMethod, Principal, Session
|
||||
from src.session.service import SessionService
|
||||
from src.session.store import SessionStore
|
||||
|
||||
|
||||
def _service(root: pathlib.Path) -> SessionService:
|
||||
"""Build a SessionService against a throwaway runtime root."""
|
||||
return SessionService(
|
||||
store=SessionStore(root / "sessions"),
|
||||
event_bus=EventBus(),
|
||||
runs_dir=root / "runs",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def runtime_root(monkeypatch):
|
||||
root = pathlib.Path(tempfile.mkdtemp())
|
||||
monkeypatch.setenv("VIBE_TRADING_HOME", str(root))
|
||||
return root
|
||||
|
||||
|
||||
# --- the service accepts and stores an owner ---
|
||||
|
||||
|
||||
def test_create_session_records_the_owner(runtime_root):
|
||||
service = _service(runtime_root)
|
||||
principal = Principal(subject="alice@example.com", auth_method=AuthMethod.FEDERATED_IDENTITY)
|
||||
|
||||
session = service.create_session(title="owned", owner=principal)
|
||||
|
||||
assert session.owner == principal
|
||||
assert session.owner.attributable is True
|
||||
|
||||
|
||||
def test_create_session_without_an_owner_still_works(runtime_root):
|
||||
# The CLI and internal paths have no request context; None means "unknown".
|
||||
service = _service(runtime_root)
|
||||
assert service.create_session(title="cli").owner is None
|
||||
|
||||
|
||||
def test_the_owner_survives_a_round_trip_through_the_store(runtime_root):
|
||||
service = _service(runtime_root)
|
||||
principal = Principal(
|
||||
subject="shared-key-holder", auth_method=AuthMethod.SHARED_KEY, tenant="desk-a"
|
||||
)
|
||||
created = service.create_session(title="persisted", owner=principal)
|
||||
|
||||
# A fresh store reading from disk, not the in-memory object.
|
||||
reloaded = SessionStore(runtime_root / "sessions").get_session(created.session_id)
|
||||
|
||||
assert reloaded is not None
|
||||
assert reloaded.owner == principal
|
||||
assert reloaded.owner.tenant == "desk-a"
|
||||
assert reloaded.owner.attributable is False
|
||||
|
||||
|
||||
def test_a_session_stored_before_owners_existed_still_loads(runtime_root):
|
||||
# Backward compatibility: an on-disk record with no owner key at all.
|
||||
store = SessionStore(runtime_root / "sessions")
|
||||
legacy = Session(title="legacy")
|
||||
store.create_session(legacy)
|
||||
|
||||
path = next((runtime_root / "sessions").rglob("session.json"))
|
||||
import json
|
||||
|
||||
data = json.loads(path.read_text())
|
||||
data.pop("owner", None)
|
||||
path.write_text(json.dumps(data))
|
||||
|
||||
reloaded = SessionStore(runtime_root / "sessions").get_session(legacy.session_id)
|
||||
assert reloaded is not None
|
||||
assert reloaded.owner is None
|
||||
|
||||
|
||||
# --- the API route threads it through ---
|
||||
|
||||
|
||||
def test_creating_a_session_over_http_records_how_it_was_authorised(runtime_root):
|
||||
import importlib
|
||||
|
||||
import api_server
|
||||
|
||||
importlib.reload(api_server)
|
||||
client = TestClient(api_server.app)
|
||||
|
||||
response = client.post("/sessions", json={"title": "http owned", "config": {}})
|
||||
assert response.status_code == 201
|
||||
|
||||
service = api_server._get_session_service()
|
||||
session = service.get_session(response.json()["session_id"])
|
||||
|
||||
assert session.owner is not None
|
||||
assert session.owner.auth_method is AuthMethod.LOOPBACK_TRUST
|
||||
# The honesty invariant survives the whole path: loopback trust authorises
|
||||
# but does not identify, so nothing downstream may treat this as a person.
|
||||
assert session.owner.attributable is False
|
||||
|
||||
|
||||
def test_require_auth_hands_back_a_principal(runtime_root):
|
||||
import asyncio
|
||||
|
||||
from src.api.security import require_auth
|
||||
|
||||
class _Req:
|
||||
method = "GET"
|
||||
headers: dict[str, str] = {}
|
||||
|
||||
class client: # noqa: N801
|
||||
host = "127.0.0.1"
|
||||
|
||||
principal = asyncio.run(require_auth(_Req(), None))
|
||||
assert isinstance(principal, Principal)
|
||||
assert principal.attributable is False
|
||||
@@ -0,0 +1,132 @@
|
||||
"""The run manifest must actually be produced by a run.
|
||||
|
||||
`build_run_manifest` existed with no caller, so nothing in the system could
|
||||
answer "what methodology produced that number". These tests pin the wiring, and
|
||||
pin the two properties that make the record safe to keep: it stores no prompt
|
||||
text, and it can never break a run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
||||
from src.agent.loop import AgentLoop
|
||||
|
||||
|
||||
class _Registry:
|
||||
tool_names = ["get_market_data", "backtest", "quantlib_call"]
|
||||
|
||||
|
||||
def _loop() -> AgentLoop:
|
||||
"""An AgentLoop shell built without running __init__ (we only need the helper)."""
|
||||
loop = AgentLoop.__new__(AgentLoop)
|
||||
loop.registry = _Registry()
|
||||
loop._run_iteration = 0
|
||||
return loop
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def trace_dir():
|
||||
return pathlib.Path(tempfile.mkdtemp()) / "run"
|
||||
|
||||
|
||||
_MESSAGES = [
|
||||
{"role": "system", "content": "You are a finance agent. <skill>tushare body</skill>"},
|
||||
{"role": "user", "content": "what is nvda worth"},
|
||||
]
|
||||
|
||||
|
||||
def test_a_run_writes_a_manifest_next_to_its_trace(trace_dir):
|
||||
_loop()._write_run_manifest(trace_dir, _MESSAGES)
|
||||
|
||||
path = trace_dir / "run_manifest.json"
|
||||
assert path.exists()
|
||||
data = json.loads(path.read_text())
|
||||
assert data["system_prompt_hash"]
|
||||
assert data["manifest_hash"]
|
||||
|
||||
|
||||
def test_the_manifest_records_the_tool_registry(trace_dir):
|
||||
_loop()._write_run_manifest(trace_dir, _MESSAGES)
|
||||
data = json.loads((trace_dir / "run_manifest.json").read_text())
|
||||
|
||||
assert set(data["tools"]["tool_names"]) == set(_Registry.tool_names)
|
||||
assert data["tools"]["tools_hash"]
|
||||
|
||||
|
||||
def test_the_prompt_text_itself_is_never_stored(trace_dir):
|
||||
# The prompt can carry user memory and workspace content. This file is a
|
||||
# provenance record, not a second copy of the conversation.
|
||||
_loop()._write_run_manifest(trace_dir, _MESSAGES)
|
||||
raw = (trace_dir / "run_manifest.json").read_text()
|
||||
|
||||
assert "You are a finance agent" not in raw
|
||||
assert "tushare body" not in raw
|
||||
assert "what is nvda worth" not in raw
|
||||
|
||||
|
||||
def test_a_changed_system_prompt_changes_the_hash(trace_dir):
|
||||
_loop()._write_run_manifest(trace_dir, _MESSAGES)
|
||||
first = json.loads((trace_dir / "run_manifest.json").read_text())
|
||||
|
||||
changed = [dict(_MESSAGES[0], content="You are a finance agent. <skill>EDITED</skill>"), _MESSAGES[1]]
|
||||
_loop()._write_run_manifest(trace_dir, changed)
|
||||
second = json.loads((trace_dir / "run_manifest.json").read_text())
|
||||
|
||||
assert first["system_prompt_hash"] != second["system_prompt_hash"]
|
||||
assert first["manifest_hash"] != second["manifest_hash"]
|
||||
|
||||
|
||||
def test_an_identical_run_reproduces_the_same_manifest_hash(trace_dir):
|
||||
# manifest_hash deliberately excludes run_id and timestamp, so the same
|
||||
# methodology on a different day hashes identically.
|
||||
a = pathlib.Path(tempfile.mkdtemp()) / "a"
|
||||
b = pathlib.Path(tempfile.mkdtemp()) / "b"
|
||||
_loop()._write_run_manifest(a, _MESSAGES)
|
||||
_loop()._write_run_manifest(b, _MESSAGES)
|
||||
|
||||
assert (
|
||||
json.loads((a / "run_manifest.json").read_text())["manifest_hash"]
|
||||
== json.loads((b / "run_manifest.json").read_text())["manifest_hash"]
|
||||
)
|
||||
|
||||
|
||||
def test_a_changed_tool_registry_changes_the_hash(trace_dir):
|
||||
_loop()._write_run_manifest(trace_dir, _MESSAGES)
|
||||
before = json.loads((trace_dir / "run_manifest.json").read_text())["manifest_hash"]
|
||||
|
||||
loop = _loop()
|
||||
loop.registry = type("R", (), {"tool_names": ["get_market_data"]})()
|
||||
loop._write_run_manifest(trace_dir, _MESSAGES)
|
||||
after = json.loads((trace_dir / "run_manifest.json").read_text())["manifest_hash"]
|
||||
|
||||
assert before != after
|
||||
|
||||
|
||||
def test_the_skill_coverage_limit_is_stated_not_implied(trace_dir):
|
||||
# Mid-run load_skill content is NOT in the hashed prompt. Saying so beats
|
||||
# implying a coverage the record does not have.
|
||||
_loop()._write_run_manifest(trace_dir, _MESSAGES)
|
||||
data = json.loads((trace_dir / "run_manifest.json").read_text())
|
||||
assert "load_skill" in data["extra"]["skill_coverage"]
|
||||
|
||||
|
||||
def test_a_manifest_failure_never_breaks_the_run(trace_dir, monkeypatch):
|
||||
# A provenance record that can kill a run is worse than a missing one.
|
||||
import src.governance.manifest as manifest_mod
|
||||
|
||||
def boom(**kwargs):
|
||||
raise RuntimeError("disk on fire")
|
||||
|
||||
monkeypatch.setattr(manifest_mod, "build_run_manifest", boom)
|
||||
_loop()._write_run_manifest(trace_dir, _MESSAGES) # must not raise
|
||||
assert not (trace_dir / "run_manifest.json").exists()
|
||||
|
||||
|
||||
def test_a_run_with_no_system_message_still_writes_a_manifest(trace_dir):
|
||||
_loop()._write_run_manifest(trace_dir, [{"role": "user", "content": "hi"}])
|
||||
assert (trace_dir / "run_manifest.json").exists()
|
||||
@@ -0,0 +1,189 @@
|
||||
"""The model-governance fields must be reachable through the SDM tools.
|
||||
|
||||
`ModelRecord` existed as a tested store schema while the three agent-facing SDM
|
||||
tools exposed none of it, so an agent registering a factor could not record an
|
||||
owner, a validator or a limitation even though the columns were there. A
|
||||
governance field nothing can write to is the same as no governance field.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tools(monkeypatch):
|
||||
"""Fresh store per test, so registrations cannot leak between them."""
|
||||
db = pathlib.Path(tempfile.mkdtemp()) / "store.db"
|
||||
monkeypatch.setenv("VIBE_TRADING_STRATEGY_STORE_DB_PATH", str(db))
|
||||
import src.strategy_store._shared as shared
|
||||
|
||||
monkeypatch.setattr(shared, "_STORE", None, raising=False)
|
||||
from src.tools.sdm_register_tool import SdmRegisterTool
|
||||
from src.tools.sdm_status_tool import SdmStatusTool
|
||||
|
||||
return SdmRegisterTool(), SdmStatusTool()
|
||||
|
||||
|
||||
def _register(reg, **kwargs):
|
||||
base = {"artifact_type": "factor", "universe": "csi300"}
|
||||
base.update(kwargs)
|
||||
return json.loads(reg.execute(**base))
|
||||
|
||||
|
||||
def _detail(status, artifact_id):
|
||||
return json.loads(status.execute(action="detail", artifact_id=artifact_id))["artifact"]
|
||||
|
||||
|
||||
# --- an artifact with no governance data says so ---
|
||||
|
||||
|
||||
def test_an_artifact_without_governance_is_reported_as_unregistered(tools):
|
||||
reg, status = tools
|
||||
result = _register(reg, name="plain")
|
||||
assert result["status"] == "ok"
|
||||
|
||||
governance = _detail(status, result["artifact"]["id"])["governance"]
|
||||
assert governance["registered"] is False
|
||||
assert "unregistered model" in governance["note"]
|
||||
|
||||
|
||||
def test_the_governance_block_is_present_even_when_empty(tools):
|
||||
# Omitting the block when there is nothing in it would hide the finding.
|
||||
reg, status = tools
|
||||
result = _register(reg, name="plain2")
|
||||
assert "governance" in _detail(status, result["artifact"]["id"])
|
||||
|
||||
|
||||
# --- intended_use and limitations are mandatory once any field is claimed ---
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"partial",
|
||||
[
|
||||
{"developer": "alice"},
|
||||
{"owner": "bob"},
|
||||
{"model_tier": "tier_1_critical"},
|
||||
{"intended_use": "x"}, # limitations missing
|
||||
{"limitations": "y"}, # intended_use missing
|
||||
{"validator": "carol", "approver": "dave"},
|
||||
],
|
||||
)
|
||||
def test_partial_governance_is_refused(tools, partial):
|
||||
reg, _ = tools
|
||||
result = _register(reg, name=f"partial_{abs(hash(str(partial)))}", **partial)
|
||||
assert result["status"] == "error"
|
||||
assert "intended_use" in result["error"] or "limitations" in result["error"]
|
||||
|
||||
|
||||
def test_a_complete_registration_is_accepted_and_stored(tools):
|
||||
reg, status = tools
|
||||
result = _register(
|
||||
reg,
|
||||
name="complete",
|
||||
developer="alice",
|
||||
owner="bob",
|
||||
validator="carol",
|
||||
approver="dave",
|
||||
model_tier="tier_2_significant",
|
||||
intended_use="CSI300 cross-sectional stock selection",
|
||||
limitations="not valid below 50m average daily turnover",
|
||||
)
|
||||
assert result["status"] == "ok"
|
||||
|
||||
artifact = _detail(status, result["artifact"]["id"])
|
||||
assert artifact["model_tier"] == "tier_2_significant"
|
||||
assert artifact["validation_status"] == "unvalidated"
|
||||
assert artifact["limitations"].startswith("not valid below")
|
||||
assert artifact["governance"]["registered"] is True
|
||||
|
||||
|
||||
# --- four eyes ---
|
||||
|
||||
|
||||
def test_developer_approving_their_own_model_is_flagged(tools):
|
||||
reg, status = tools
|
||||
result = _register(
|
||||
reg, name="selfapproved", developer="alice", approver="alice",
|
||||
intended_use="x", limitations="y",
|
||||
)
|
||||
governance = _detail(status, result["artifact"]["id"])["governance"]
|
||||
assert governance["four_eyes_violation"] is True
|
||||
assert "same person" in governance["note"]
|
||||
|
||||
|
||||
def test_two_different_people_are_not_flagged(tools):
|
||||
reg, status = tools
|
||||
result = _register(
|
||||
reg, name="twoperson", developer="alice", approver="bob",
|
||||
intended_use="x", limitations="y",
|
||||
)
|
||||
assert _detail(status, result["artifact"]["id"])["governance"]["four_eyes_violation"] is False
|
||||
|
||||
|
||||
def test_a_four_eyes_violation_is_reported_not_refused(tools):
|
||||
# Whether to block self-approval is firm policy, not this tool's call. It
|
||||
# must be visible, and it must not be silently prevented either.
|
||||
reg, _ = tools
|
||||
result = _register(
|
||||
reg, name="allowed", developer="alice", approver="alice",
|
||||
intended_use="x", limitations="y",
|
||||
)
|
||||
assert result["status"] == "ok"
|
||||
|
||||
|
||||
# --- state machine ---
|
||||
|
||||
|
||||
def test_registering_straight_into_approved_is_refused(tools):
|
||||
reg, _ = tools
|
||||
result = _register(
|
||||
reg, name="jump", developer="a", approver="b",
|
||||
intended_use="x", limitations="y", validation_status="approved",
|
||||
)
|
||||
assert result["status"] == "error"
|
||||
assert "approved" in result["error"].lower()
|
||||
|
||||
|
||||
def test_registering_as_in_validation_is_allowed(tools):
|
||||
reg, status = tools
|
||||
result = _register(
|
||||
reg, name="inval", developer="a", approver="b",
|
||||
intended_use="x", limitations="y", validation_status="in_validation",
|
||||
)
|
||||
assert result["status"] == "ok"
|
||||
assert _detail(status, result["artifact"]["id"])["validation_status"] == "in_validation"
|
||||
|
||||
|
||||
def test_an_unknown_tier_is_refused(tools):
|
||||
reg, _ = tools
|
||||
result = _register(
|
||||
reg, name="badtier", model_tier="tier_9_imaginary",
|
||||
intended_use="x", limitations="y",
|
||||
)
|
||||
assert result["status"] == "error"
|
||||
|
||||
|
||||
# --- the schema itself must advertise the fields ---
|
||||
|
||||
|
||||
def test_every_governance_field_is_in_the_tool_schema(tools):
|
||||
reg, _ = tools
|
||||
properties = reg.parameters["properties"]
|
||||
for field in reg._GOVERNANCE_FIELDS:
|
||||
assert field in properties, f"{field} is not reachable through the tool schema"
|
||||
|
||||
|
||||
def test_validation_date_is_never_auto_filled(tools):
|
||||
# Stamping a validation date on a model nobody validated fabricates exactly
|
||||
# the evidence the field exists to carry.
|
||||
reg, status = tools
|
||||
result = _register(
|
||||
reg, name="nodate", developer="a", approver="b",
|
||||
intended_use="x", limitations="y",
|
||||
)
|
||||
assert _detail(status, result["artifact"]["id"])["validation_date"] is None
|
||||
@@ -3,6 +3,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from src.strategy_store.models import (
|
||||
@@ -13,6 +15,11 @@ from src.strategy_store.models import (
|
||||
BenchCategory,
|
||||
DecaySnapshot,
|
||||
DecaySignal,
|
||||
ModelTier,
|
||||
ValidationStatus,
|
||||
is_four_eyes_violation,
|
||||
validate_model_registration,
|
||||
validate_validation_status_transition,
|
||||
)
|
||||
from src.strategy_store.store import InMemoryStrategyStore
|
||||
from src.strategy_store.decay import DecayEvaluator, DecayThresholds
|
||||
@@ -886,3 +893,505 @@ class TestSqliteStore:
|
||||
|
||||
assert self.store.get_bench_history(aid) == []
|
||||
assert self.store.get_decay_history(aid) == []
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# TestModelGovernance — pure-logic ModelRecord helpers (models.py)
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestModelGovernance:
|
||||
"""Tests for the model-governance fields and validation helpers."""
|
||||
|
||||
def test_artifact_governance_defaults(self):
|
||||
"""Bare Artifact construction (pre-governance call sites) is unaffected."""
|
||||
art = _make_artifact()
|
||||
assert art.developer is None
|
||||
assert art.owner is None
|
||||
assert art.validator is None
|
||||
assert art.approver is None
|
||||
assert art.model_version is None
|
||||
assert art.artifact_version is None
|
||||
assert art.model_tier is None
|
||||
assert art.intended_use is None
|
||||
assert art.limitations is None
|
||||
assert art.validation_status == ValidationStatus.UNVALIDATED
|
||||
assert art.validation_date is None
|
||||
|
||||
def test_model_tier_enum(self):
|
||||
expected = {"tier_1_critical", "tier_2_significant", "tier_3_limited"}
|
||||
assert {t.value for t in ModelTier} == expected
|
||||
|
||||
def test_validation_status_enum(self):
|
||||
expected = {"unvalidated", "in_validation", "validated", "approved", "rejected"}
|
||||
assert {s.value for s in ValidationStatus} == expected
|
||||
|
||||
# -- four-eyes principle -------------------------------------------------
|
||||
|
||||
def test_four_eyes_violation_same_person(self):
|
||||
art = _make_artifact()
|
||||
art = Artifact(**{**art.__dict__, "developer": "Alice", "approver": "Alice"})
|
||||
assert is_four_eyes_violation(art) is True
|
||||
|
||||
def test_four_eyes_violation_case_and_whitespace_insensitive(self):
|
||||
art = _make_artifact()
|
||||
art = Artifact(
|
||||
**{**art.__dict__, "developer": " Alice ", "approver": "alice"}
|
||||
)
|
||||
assert is_four_eyes_violation(art) is True
|
||||
|
||||
def test_four_eyes_no_violation_different_people(self):
|
||||
art = _make_artifact()
|
||||
art = Artifact(
|
||||
**{**art.__dict__, "developer": "Alice", "approver": "Bob"}
|
||||
)
|
||||
assert is_four_eyes_violation(art) is False
|
||||
|
||||
def test_four_eyes_not_flagged_when_roles_unset(self):
|
||||
"""With developer or approver unset there is nothing to compare — not a violation."""
|
||||
art = _make_artifact()
|
||||
assert is_four_eyes_violation(art) is False
|
||||
|
||||
only_developer = Artifact(**{**art.__dict__, "developer": "Alice"})
|
||||
assert is_four_eyes_violation(only_developer) is False
|
||||
|
||||
def test_four_eyes_does_not_raise(self):
|
||||
"""Four-eyes detection is advisory only — it must never raise."""
|
||||
art = _make_artifact()
|
||||
art = Artifact(**{**art.__dict__, "developer": "Alice", "approver": "Alice"})
|
||||
# Calling it must not raise even though it flags a violation.
|
||||
result = is_four_eyes_violation(art)
|
||||
assert result is True
|
||||
|
||||
# -- intended_use / limitations required ---------------------------------
|
||||
|
||||
def test_validate_model_registration_requires_intended_use(self):
|
||||
art = _make_artifact()
|
||||
art = Artifact(**{**art.__dict__, "limitations": "US large-cap only"})
|
||||
with pytest.raises(ValueError, match="intended_use"):
|
||||
validate_model_registration(art)
|
||||
|
||||
def test_validate_model_registration_requires_limitations(self):
|
||||
art = _make_artifact()
|
||||
art = Artifact(**{**art.__dict__, "intended_use": "Daily rebalance signal"})
|
||||
with pytest.raises(ValueError, match="limitations"):
|
||||
validate_model_registration(art)
|
||||
|
||||
def test_validate_model_registration_rejects_blank_strings(self):
|
||||
"""Whitespace-only intended_use/limitations count as empty."""
|
||||
art = _make_artifact()
|
||||
art = Artifact(
|
||||
**{**art.__dict__, "intended_use": " ", "limitations": " "}
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
validate_model_registration(art)
|
||||
|
||||
def test_validate_model_registration_passes_when_both_set(self):
|
||||
art = _make_artifact()
|
||||
art = Artifact(
|
||||
**{
|
||||
**art.__dict__,
|
||||
"intended_use": "Daily rebalance signal for CSI300 longs",
|
||||
"limitations": "Not validated out-of-sample post-2024",
|
||||
}
|
||||
)
|
||||
validate_model_registration(art) # must not raise
|
||||
|
||||
# -- validation state machine --------------------------------------------
|
||||
|
||||
def test_unvalidated_to_approved_blocked(self):
|
||||
with pytest.raises(ValueError, match="VALIDATED"):
|
||||
validate_validation_status_transition(
|
||||
ValidationStatus.UNVALIDATED, ValidationStatus.APPROVED
|
||||
)
|
||||
|
||||
def test_in_validation_to_approved_blocked(self):
|
||||
with pytest.raises(ValueError):
|
||||
validate_validation_status_transition(
|
||||
ValidationStatus.IN_VALIDATION, ValidationStatus.APPROVED
|
||||
)
|
||||
|
||||
def test_rejected_to_approved_blocked(self):
|
||||
with pytest.raises(ValueError):
|
||||
validate_validation_status_transition(
|
||||
ValidationStatus.REJECTED, ValidationStatus.APPROVED
|
||||
)
|
||||
|
||||
def test_validated_to_approved_allowed(self):
|
||||
validate_validation_status_transition(
|
||||
ValidationStatus.VALIDATED, ValidationStatus.APPROVED
|
||||
) # must not raise
|
||||
|
||||
def test_approved_to_approved_idempotent(self):
|
||||
validate_validation_status_transition(
|
||||
ValidationStatus.APPROVED, ValidationStatus.APPROVED
|
||||
) # re-approval is a no-op, must not raise
|
||||
|
||||
def test_non_approved_transitions_unrestricted(self):
|
||||
"""The guard only restricts reaching APPROVED; other moves are free."""
|
||||
validate_validation_status_transition(
|
||||
ValidationStatus.UNVALIDATED, ValidationStatus.IN_VALIDATION
|
||||
)
|
||||
validate_validation_status_transition(
|
||||
ValidationStatus.IN_VALIDATION, ValidationStatus.VALIDATED
|
||||
)
|
||||
validate_validation_status_transition(
|
||||
ValidationStatus.VALIDATED, ValidationStatus.REJECTED
|
||||
)
|
||||
validate_validation_status_transition(
|
||||
ValidationStatus.APPROVED, ValidationStatus.UNVALIDATED
|
||||
) # e.g. re-opening for a new review cycle
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# TestModelGovernanceStore — store-level enforcement + persistence
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestModelGovernanceStore:
|
||||
"""Governance-field persistence and enforcement across both store backends."""
|
||||
|
||||
def _governed_artifact(self, **overrides) -> Artifact:
|
||||
base = _make_artifact(name=overrides.pop("name", "governed_factor"))
|
||||
fields = {
|
||||
**base.__dict__,
|
||||
"developer": "Alice",
|
||||
"owner": "Bob",
|
||||
"validator": "Carol",
|
||||
"approver": "Dave",
|
||||
"model_version": "1.0.0",
|
||||
"artifact_version": "art-v3",
|
||||
"model_tier": ModelTier.TIER_2_SIGNIFICANT,
|
||||
"intended_use": "Daily rebalance signal for CSI300 longs",
|
||||
"limitations": "Not validated out-of-sample post-2024",
|
||||
"validation_status": ValidationStatus.VALIDATED,
|
||||
"validation_date": "2026-08-01",
|
||||
}
|
||||
fields.update(overrides)
|
||||
return Artifact(**fields)
|
||||
|
||||
@pytest.mark.parametrize("backend", ["sqlite", "memory"])
|
||||
def test_governance_fields_round_trip(self, backend, tmp_path):
|
||||
store = (
|
||||
_sqlite_store(tmp_path)
|
||||
if backend == "sqlite"
|
||||
else InMemoryStrategyStore()
|
||||
)
|
||||
art = self._governed_artifact()
|
||||
aid = store.register_artifact(art)
|
||||
fetched = store.get_artifact(aid)
|
||||
|
||||
assert fetched is not None
|
||||
assert fetched.developer == "Alice"
|
||||
assert fetched.owner == "Bob"
|
||||
assert fetched.validator == "Carol"
|
||||
assert fetched.approver == "Dave"
|
||||
assert fetched.model_version == "1.0.0"
|
||||
assert fetched.artifact_version == "art-v3"
|
||||
assert fetched.model_tier == ModelTier.TIER_2_SIGNIFICANT
|
||||
assert fetched.intended_use == "Daily rebalance signal for CSI300 longs"
|
||||
assert fetched.limitations == "Not validated out-of-sample post-2024"
|
||||
assert fetched.validation_status == ValidationStatus.VALIDATED
|
||||
assert fetched.validation_date == "2026-08-01"
|
||||
|
||||
@pytest.mark.parametrize("backend", ["sqlite", "memory"])
|
||||
def test_register_directly_as_approved_blocked(self, backend, tmp_path):
|
||||
"""A model can't be born APPROVED — it has no prior VALIDATED state."""
|
||||
store = (
|
||||
_sqlite_store(tmp_path)
|
||||
if backend == "sqlite"
|
||||
else InMemoryStrategyStore()
|
||||
)
|
||||
art = self._governed_artifact(validation_status=ValidationStatus.APPROVED)
|
||||
with pytest.raises(ValueError, match="VALIDATED"):
|
||||
store.register_artifact(art)
|
||||
|
||||
@pytest.mark.parametrize("backend", ["sqlite", "memory"])
|
||||
def test_update_to_approved_requires_prior_validated(self, backend, tmp_path):
|
||||
store = (
|
||||
_sqlite_store(tmp_path)
|
||||
if backend == "sqlite"
|
||||
else InMemoryStrategyStore()
|
||||
)
|
||||
art = self._governed_artifact(validation_status=ValidationStatus.UNVALIDATED)
|
||||
aid = store.register_artifact(art)
|
||||
fetched = store.get_artifact(aid)
|
||||
assert fetched is not None
|
||||
|
||||
# Still UNVALIDATED — jumping to APPROVED must be blocked.
|
||||
illegal = Artifact(**{**fetched.__dict__, "validation_status": ValidationStatus.APPROVED})
|
||||
with pytest.raises(ValueError, match="VALIDATED"):
|
||||
store.update_artifact(illegal)
|
||||
|
||||
# Going through VALIDATED first works, and APPROVED after that works too.
|
||||
validated = Artifact(**{**fetched.__dict__, "validation_status": ValidationStatus.VALIDATED})
|
||||
result = store.update_artifact(validated)
|
||||
assert result is not None
|
||||
assert result.validation_status == ValidationStatus.VALIDATED
|
||||
|
||||
approved = Artifact(**{**result.__dict__, "validation_status": ValidationStatus.APPROVED})
|
||||
result2 = store.update_artifact(approved)
|
||||
assert result2 is not None
|
||||
assert result2.validation_status == ValidationStatus.APPROVED
|
||||
|
||||
def test_four_eyes_detectable_after_round_trip(self, tmp_path):
|
||||
"""A same-person dev/approve record is registerable (not force-rejected)
|
||||
but is_four_eyes_violation must flag it after reading it back."""
|
||||
store = _sqlite_store(tmp_path)
|
||||
art = self._governed_artifact(
|
||||
name="four_eyes_factor", developer="Alice", approver="Alice"
|
||||
)
|
||||
aid = store.register_artifact(art)
|
||||
fetched = store.get_artifact(aid)
|
||||
assert fetched is not None
|
||||
assert is_four_eyes_violation(fetched) is True
|
||||
|
||||
|
||||
def _sqlite_store(tmp_path):
|
||||
from src.strategy_store.sqlite_store import SqliteStrategyStore
|
||||
|
||||
return SqliteStrategyStore(db_path=tmp_path / f"gov_{uuid.uuid4().hex[:8]}.db")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# TestSchemaMigration — backward-compatible SQLite migration
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
_LEGACY_SCHEMA_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS artifacts (
|
||||
id TEXT PRIMARY KEY,
|
||||
type TEXT NOT NULL CHECK(type IN ('factor', 'strategy')),
|
||||
name TEXT NOT NULL,
|
||||
source_paper TEXT,
|
||||
source_url TEXT,
|
||||
formula_latex TEXT,
|
||||
theme TEXT,
|
||||
columns_required TEXT,
|
||||
decay_horizon INTEGER DEFAULT 20,
|
||||
signal_definition TEXT,
|
||||
entry_rules TEXT,
|
||||
exit_rules TEXT,
|
||||
position_sizing TEXT,
|
||||
universe TEXT NOT NULL,
|
||||
signal_engine_path TEXT,
|
||||
run_dir TEXT,
|
||||
hypothesis_id TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'created'
|
||||
CHECK(status IN (
|
||||
'created','benching','active',
|
||||
'monitoring','decayed','disabled')),
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
disabled_at TEXT,
|
||||
disabled_reason TEXT,
|
||||
UNIQUE(name, universe)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bench_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
artifact_id TEXT NOT NULL
|
||||
REFERENCES artifacts(id) ON DELETE CASCADE,
|
||||
bench_type TEXT NOT NULL
|
||||
CHECK(bench_type IN ('initial','periodic','manual')),
|
||||
ic_mean REAL,
|
||||
ic_std REAL,
|
||||
ir REAL,
|
||||
ic_positive_ratio REAL,
|
||||
t_stat REAL,
|
||||
sharpe REAL,
|
||||
annual_return REAL,
|
||||
max_drawdown REAL,
|
||||
calmar REAL,
|
||||
category TEXT
|
||||
CHECK(category IN (
|
||||
'alive','reversed','dead',
|
||||
'confirmed_alive','noise')),
|
||||
train_start TEXT,
|
||||
train_end TEXT,
|
||||
test_start TEXT,
|
||||
test_end TEXT,
|
||||
run_dir TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS decay_snapshots (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
artifact_id TEXT NOT NULL
|
||||
REFERENCES artifacts(id) ON DELETE CASCADE,
|
||||
rolling_ic_mean REAL,
|
||||
rolling_ir REAL,
|
||||
baseline_ic_mean REAL,
|
||||
ic_ratio REAL,
|
||||
decay_signal TEXT
|
||||
CHECK(decay_signal IN (
|
||||
'healthy','warning','decayed','critical')),
|
||||
consecutive_warnings INTEGER DEFAULT 0,
|
||||
detail TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_artifacts_status
|
||||
ON artifacts(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_artifacts_type
|
||||
ON artifacts(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_bench_artifact
|
||||
ON bench_history(artifact_id, created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_decay_artifact
|
||||
ON decay_snapshots(artifact_id, created_at);
|
||||
"""
|
||||
|
||||
|
||||
def _build_legacy_db(db_path) -> None:
|
||||
"""Create a pre-governance-schema database with one legacy artifact row.
|
||||
|
||||
Mirrors exactly the ``_init_db`` DDL as it existed before the
|
||||
model-governance migration (no developer/owner/validator/approver/
|
||||
model_version/artifact_version/model_tier/intended_use/limitations/
|
||||
validation_status/validation_date columns).
|
||||
"""
|
||||
import sqlite3
|
||||
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.executescript(_LEGACY_SCHEMA_SQL)
|
||||
conn.execute("PRAGMA user_version=1")
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO artifacts (
|
||||
id, type, name, source_paper, source_url, formula_latex,
|
||||
theme, columns_required, decay_horizon, signal_definition,
|
||||
entry_rules, exit_rules, position_sizing, universe,
|
||||
signal_engine_path, run_dir, hypothesis_id, status,
|
||||
created_at, updated_at, disabled_at, disabled_reason
|
||||
) VALUES (
|
||||
'art_legacy001', 'factor', 'legacy_momentum', NULL, NULL, NULL,
|
||||
'[]', '[]', 20, NULL,
|
||||
NULL, NULL, NULL, 'CSI300',
|
||||
NULL, NULL, NULL, 'active',
|
||||
'2025-01-01T00:00:00+00:00', '2025-01-01T00:00:00+00:00', NULL, NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
class TestSchemaMigration:
|
||||
"""Backward-compatible migration of a pre-governance SQLite database."""
|
||||
|
||||
def test_old_db_opens_and_reads_via_new_code(self, tmp_path):
|
||||
"""The most important test: a DB written before governance fields
|
||||
existed must still be readable (and writable) by the new store."""
|
||||
from src.strategy_store.sqlite_store import SqliteStrategyStore
|
||||
|
||||
db_path = tmp_path / "legacy.db"
|
||||
_build_legacy_db(db_path)
|
||||
|
||||
store = SqliteStrategyStore(db_path=db_path)
|
||||
fetched = store.get_artifact("art_legacy001")
|
||||
|
||||
assert fetched is not None
|
||||
assert fetched.name == "legacy_momentum"
|
||||
assert fetched.universe == "CSI300"
|
||||
assert fetched.status == ArtifactStatus.ACTIVE
|
||||
# New governance columns default sanely for pre-existing rows.
|
||||
assert fetched.developer is None
|
||||
assert fetched.owner is None
|
||||
assert fetched.validator is None
|
||||
assert fetched.approver is None
|
||||
assert fetched.model_version is None
|
||||
assert fetched.artifact_version is None
|
||||
assert fetched.model_tier is None
|
||||
assert fetched.intended_use is None
|
||||
assert fetched.limitations is None
|
||||
assert fetched.validation_status == ValidationStatus.UNVALIDATED
|
||||
assert fetched.validation_date is None
|
||||
|
||||
def test_old_db_still_writable_after_migration(self, tmp_path):
|
||||
"""New writes (including governance fields) work on a migrated DB."""
|
||||
from src.strategy_store.sqlite_store import SqliteStrategyStore
|
||||
|
||||
db_path = tmp_path / "legacy_write.db"
|
||||
_build_legacy_db(db_path)
|
||||
|
||||
store = SqliteStrategyStore(db_path=db_path)
|
||||
aid = store.register_artifact(
|
||||
Artifact(
|
||||
id="",
|
||||
type=ArtifactType.FACTOR,
|
||||
name="post_migration_factor",
|
||||
universe="CSI300",
|
||||
developer="Alice",
|
||||
intended_use="Signal for post-migration coverage",
|
||||
limitations="Untested pre-2020",
|
||||
)
|
||||
)
|
||||
fetched = store.get_artifact(aid)
|
||||
assert fetched is not None
|
||||
assert fetched.developer == "Alice"
|
||||
assert fetched.intended_use == "Signal for post-migration coverage"
|
||||
|
||||
# The pre-existing legacy row is still intact and readable too.
|
||||
legacy = store.get_artifact("art_legacy001")
|
||||
assert legacy is not None
|
||||
assert legacy.name == "legacy_momentum"
|
||||
|
||||
def test_old_db_update_status_still_works(self, tmp_path):
|
||||
"""update_status (unrelated to governance) keeps working on a migrated row."""
|
||||
from src.strategy_store.sqlite_store import SqliteStrategyStore
|
||||
|
||||
db_path = tmp_path / "legacy_status.db"
|
||||
_build_legacy_db(db_path)
|
||||
|
||||
store = SqliteStrategyStore(db_path=db_path)
|
||||
updated = store.update_status(
|
||||
"art_legacy001", ArtifactStatus.DISABLED, reason="migrated-test"
|
||||
)
|
||||
assert updated is not None
|
||||
assert updated.status == ArtifactStatus.DISABLED
|
||||
assert updated.disabled_reason == "migrated-test"
|
||||
# Governance fields remain untouched/default.
|
||||
assert updated.validation_status == ValidationStatus.UNVALIDATED
|
||||
|
||||
def test_migration_is_idempotent(self, tmp_path):
|
||||
"""Running the column-migration twice does not error or duplicate columns."""
|
||||
from src.strategy_store.sqlite_store import SqliteStrategyStore
|
||||
|
||||
db_path = tmp_path / "idempotent.db"
|
||||
_build_legacy_db(db_path)
|
||||
|
||||
store = SqliteStrategyStore(db_path=db_path)
|
||||
# First open already ran the migration once (in __init__ -> _init_db).
|
||||
# Run it again explicitly, twice more, to prove idempotency.
|
||||
store._migrate_governance_columns()
|
||||
store._migrate_governance_columns()
|
||||
|
||||
columns = [
|
||||
row["name"]
|
||||
for row in store._conn.execute("PRAGMA table_info(artifacts)")
|
||||
]
|
||||
# No duplicate column names.
|
||||
assert len(columns) == len(set(columns))
|
||||
assert columns.count("validation_status") == 1
|
||||
assert columns.count("developer") == 1
|
||||
|
||||
# Store still functions normally after repeated migration calls.
|
||||
fetched = store.get_artifact("art_legacy001")
|
||||
assert fetched is not None
|
||||
|
||||
def test_migration_idempotent_across_fresh_reopen(self, tmp_path):
|
||||
"""Re-opening an already-migrated DB with a fresh store instance is safe."""
|
||||
from src.strategy_store.sqlite_store import SqliteStrategyStore
|
||||
|
||||
db_path = tmp_path / "reopen.db"
|
||||
_build_legacy_db(db_path)
|
||||
|
||||
store1 = SqliteStrategyStore(db_path=db_path)
|
||||
aid = store1.register_artifact(_make_artifact(name="reopen_factor"))
|
||||
|
||||
# Re-open with a brand-new store instance — _init_db (and the
|
||||
# migration inside it) runs again against an already-migrated file.
|
||||
store2 = SqliteStrategyStore(db_path=db_path)
|
||||
fetched = store2.get_artifact(aid)
|
||||
assert fetched is not None
|
||||
assert fetched.name == "reopen_factor"
|
||||
Reference in new issue
Block a user