From 82340585628ef41595ad033df6c50e6323981796 Mon Sep 17 00:00:00 2001 From: shadowinlife Date: Wed, 8 Jul 2026 19:36:24 +0800 Subject: [PATCH] refactor: centralize environment variable management with Pydantic schema + CI gate - Add agent/src/config/env_schema.py: Pydantic models for ~50 env vars across 6 sub-models (LLMConfig, DataConfig, APIConfig, SwarmConfig, AgentTuningConfig, PathConfig) - Add agent/src/config/accessor.py: thread-safe lazy singleton get_env_config() + unified _parse_bool() + reset_env_config() for test isolation - Fix TIMEOUT_SECONDS CLI display default (2400 -> 120, matching runtime) - Fix MAX_RETRIES CLI display default ('(not set)' -> 2, matching runtime) - Fix VIBE_TRADING_API_KEY server-side alias (security.py now resolves it via get_env_config().api.api_auth_key) - Fix VIBE_TRADING_CHANNELS_AUTO_START=on silently ignored (unified _parse_bool accepts 'on') - Migrate 207 scattered os.getenv/os.environ.get calls across 50+ files to centralized accessor - Convert 12 module-level import-time fixations to lazy accessor pattern (loop.py, swarm_tool.py, _backend.py, worker.py, security.py) - Add reset_env_config() calls after settings_routes.py env mutations (13 mutation points) - Replace print('[WARN]...') with logger.warning() in tushare/okx/yfinance loaders - Add tools/ci_env_var_gate.py: AST-based CI gate preventing new os.getenv calls outside src/config/ - Add tools/test_ci_env_var_gate.py: 18 test cases for the CI gate - Update tools/ci_grep_gates.sh to call the new env var gate - Add agent/tests/test_env_schema.py: 81 test cases covering defaults, type coercion, singleton, thread safety, bool parser - Update agent/.env.example with schema cross-references - Update agent/tests/conftest.py with reset_env_config autouse fixture Closes HKUDS/Vibe-Trading#438 Co-Authored-By: OpenCode AI-Model: alibaba-cn/glm-5.2 Signed-off-by: shadowinlife Co-Authored-By: opencode Co-Authored-By: Claude Code AI-Contributed/Feature: 1383/1383 AI-Contributed/UT: 735/735 --- agent/.env.example | 9 +- agent/api_server.py | 4 +- agent/backtest/loaders/alphavantage_loader.py | 5 +- agent/backtest/loaders/base.py | 12 +- agent/backtest/loaders/ccxt_loader.py | 4 +- agent/backtest/loaders/finnhub_loader.py | 9 +- agent/backtest/loaders/fmp_loader.py | 5 +- agent/backtest/loaders/futu.py | 15 +- agent/backtest/loaders/okx.py | 11 +- agent/backtest/loaders/sec_edgar_client.py | 5 +- agent/backtest/loaders/tiingo_loader.py | 5 +- agent/backtest/loaders/tushare.py | 30 +- .../backtest/loaders/tushare_fundamentals.py | 5 +- agent/backtest/loaders/yfinance_loader.py | 9 +- agent/cli/_legacy.py | 27 +- agent/cli/commands/chat.py | 8 +- agent/cli/main.py | 4 +- agent/mcp_server.py | 4 +- agent/scripts/bench_performance.py | 4 +- agent/scripts/w4a_run_benches.py | 5 +- agent/src/agent/loop.py | 132 +++-- agent/src/api/scheduled_routes.py | 7 +- agent/src/api/security.py | 45 +- agent/src/api/settings_routes.py | 5 + agent/src/api/state.py | 3 +- agent/src/config/accessor.py | 132 +++++ agent/src/config/env_schema.py | 327 ++++++++++++ agent/src/factors/_backend.py | 53 +- agent/src/factors/bench_runner.py | 3 +- agent/src/goal/store.py | 3 +- agent/src/hypotheses/registry.py | 4 +- agent/src/live/order_guard.py | 4 +- agent/src/preflight.py | 15 +- agent/src/providers/chat.py | 6 +- agent/src/providers/content_filter.py | 11 +- agent/src/providers/llm.py | 48 +- agent/src/providers/openai_codex.py | 4 +- agent/src/scheduled_research/executor.py | 6 +- agent/src/swarm/grounding.py | 14 +- agent/src/swarm/runtime.py | 11 +- agent/src/swarm/store.py | 7 +- agent/src/swarm/worker.py | 14 +- agent/src/tools/alpha_bench_tool.py | 3 +- agent/src/tools/fred_macro_tool.py | 5 +- agent/src/tools/iwencai_tool.py | 5 +- agent/src/tools/path_utils.py | 8 +- agent/src/tools/swarm_tool.py | 21 +- agent/src/tools/web_search_tool.py | 9 +- agent/tests/conftest.py | 11 + agent/tests/test_env_schema.py | 471 ++++++++++++++++++ agent/tests/test_futu_loader.py | 20 +- tools/ci_env_var_gate.py | 280 +++++++++++ tools/ci_grep_gates.sh | 18 + tools/test_ci_env_var_gate.py | 233 +++++++++ 54 files changed, 1907 insertions(+), 211 deletions(-) create mode 100644 agent/src/config/accessor.py create mode 100644 agent/src/config/env_schema.py create mode 100644 agent/tests/test_env_schema.py create mode 100644 tools/ci_env_var_gate.py create mode 100644 tools/test_ci_env_var_gate.py diff --git a/agent/.env.example b/agent/.env.example index d3d795d0..01f1173c 100644 --- a/agent/.env.example +++ b/agent/.env.example @@ -1,5 +1,6 @@ +# NOTE: All defaults are defined in agent/src/config/env_schema.py (EnvConfig). # ============================================================================ -# LLM Provider — uncomment ONE provider block below +# LLM Provider (LLMConfig) — uncomment ONE provider block below # ============================================================================ # --- OpenRouter (recommended, multi-model gateway) --- @@ -97,7 +98,7 @@ MAX_RETRIES=2 # LANGCHAIN_REASONING_EFFORT=medium # ============================================================================ -# Data Sources +# Data Sources (DataConfig) # ============================================================================ # A-shares: Tushare Pro token (https://tushare.pro) TUSHARE_TOKEN=your-tushare-token @@ -129,7 +130,7 @@ TUSHARE_TOKEN=your-tushare-token # VIBE_TRADING_DATA_CACHE=1 # ============================================================================ -# API Server (optional) +# API Server (APIConfig, optional) # ============================================================================ # Bearer token for API auth. # Empty dev mode is loopback-only. Set this before exposing port 8899 beyond localhost. @@ -153,7 +154,7 @@ TUSHARE_TOKEN=your-tushare-token # VIBE_TRADING_ALLOWED_RUN_ROOTS= # ============================================================================ -# Agent Tuning (optional, defaults are sensible) +# Agent Tuning (AgentTuningConfig, optional, defaults are sensible) # ============================================================================ # SWARM_WORKER_TIMEOUT=300 # SWARM_WORKER_MAX_ITER=50 diff --git a/agent/api_server.py b/agent/api_server.py index 181ab466..40b9eb59 100644 --- a/agent/api_server.py +++ b/agent/api_server.py @@ -157,7 +157,9 @@ async def _run_startup_preflight() -> None: run_preflight(console) _start_scheduled_research_executor() - if os.getenv("VIBE_TRADING_CHANNELS_AUTO_START", "").strip().lower() in {"1", "true", "yes"}: + from src.config.accessor import get_env_config + + if get_env_config().agent_tuning.vibe_trading_channels_auto_start: await _start_channel_runtime() diff --git a/agent/backtest/loaders/alphavantage_loader.py b/agent/backtest/loaders/alphavantage_loader.py index 0098b4b2..2eaacac6 100644 --- a/agent/backtest/loaders/alphavantage_loader.py +++ b/agent/backtest/loaders/alphavantage_loader.py @@ -22,7 +22,6 @@ never aborts the batch. from __future__ import annotations import logging -import os from typing import Dict, List, Optional import pandas as pd @@ -63,7 +62,9 @@ def _resolve_api_key() -> str: Returns: The trimmed key, or ``""`` when the env var is absent or a placeholder. """ - key = os.getenv(_API_KEY_ENV, "").strip() + from src.config.accessor import get_env_config + + key = get_env_config().data.alphavantage_api_key.strip() return "" if key in _API_KEY_PLACEHOLDERS else key diff --git a/agent/backtest/loaders/base.py b/agent/backtest/loaders/base.py index df7822e3..002ac417 100644 --- a/agent/backtest/loaders/base.py +++ b/agent/backtest/loaders/base.py @@ -231,14 +231,18 @@ _LOADER_CACHE_VERSION = 2 def loader_cache_enabled() -> bool: """Return whether the local market-data cache is explicitly enabled.""" - return os.getenv(LOADER_CACHE_ENV, "").strip().lower() in _LOADER_CACHE_TRUE_VALUES + from src.config.accessor import get_env_config + + return get_env_config().data.vibe_trading_data_cache def loader_cache_root() -> Path: """Return the root directory for opt-in loader cache files.""" - raw = os.getenv(LOADER_CACHE_ROOT_ENV) - if raw and raw.strip(): - return Path(raw).expanduser() + from src.config.accessor import get_env_config + + root = get_env_config().data.vibe_trading_data_cache_root + if root and root.strip(): + return Path(root).expanduser() return Path.home() / ".vibe-trading" / "cache" / "loaders" diff --git a/agent/backtest/loaders/ccxt_loader.py b/agent/backtest/loaders/ccxt_loader.py index b727b04b..d3cc1fb6 100644 --- a/agent/backtest/loaders/ccxt_loader.py +++ b/agent/backtest/loaders/ccxt_loader.py @@ -84,7 +84,9 @@ class DataLoader: def _get_exchange(self): """Create exchange instance.""" import ccxt - exchange_id = os.getenv("CCXT_EXCHANGE", "binance").lower() + from src.config.accessor import get_env_config + + exchange_id = get_env_config().data.ccxt_exchange.lower() exchange_cls = getattr(ccxt, exchange_id, None) if exchange_cls is None: logger.warning("Unknown CCXT exchange %s, falling back to binance", exchange_id) diff --git a/agent/backtest/loaders/finnhub_loader.py b/agent/backtest/loaders/finnhub_loader.py index b08c7874..e05882e2 100644 --- a/agent/backtest/loaders/finnhub_loader.py +++ b/agent/backtest/loaders/finnhub_loader.py @@ -22,7 +22,6 @@ the user-home loader cache via :func:`cached_loader_fetch`. from __future__ import annotations import logging -import os from typing import Any, Dict, List, Optional import pandas as pd @@ -161,7 +160,9 @@ class DataLoader: def is_available(self) -> bool: """Return whether a Finnhub API key is present in the environment.""" - return bool(os.getenv(_API_KEY_ENV)) + from src.config.accessor import get_env_config + + return bool(get_env_config().data.finnhub_api_key) def fetch( self, @@ -196,7 +197,9 @@ class DataLoader: del fields validate_date_range(start_date, end_date) - api_key = os.getenv(_API_KEY_ENV) + from src.config.accessor import get_env_config + + api_key = get_env_config().data.finnhub_api_key if not api_key: logger.warning("finnhub fetch skipped: %s not set", _API_KEY_ENV) return {} diff --git a/agent/backtest/loaders/fmp_loader.py b/agent/backtest/loaders/fmp_loader.py index fcacabd9..25412415 100644 --- a/agent/backtest/loaders/fmp_loader.py +++ b/agent/backtest/loaders/fmp_loader.py @@ -19,7 +19,6 @@ Auth: set ``FMP_API_KEY`` in the environment. Covers US equities only. from __future__ import annotations import logging -import os from typing import Any, Dict, List, Optional import pandas as pd @@ -44,7 +43,9 @@ _OHLCV_FIELDS = ("open", "high", "low", "close", "volume") def _api_key() -> str: """Return the FMP API key from the environment, stripped (``""`` if unset).""" - return os.getenv(_API_KEY_ENV, "").strip() + from src.config.accessor import get_env_config + + return get_env_config().data.fmp_api_key.strip() def _min_interval() -> float: diff --git a/agent/backtest/loaders/futu.py b/agent/backtest/loaders/futu.py index 02d1576f..107d7adc 100644 --- a/agent/backtest/loaders/futu.py +++ b/agent/backtest/loaders/futu.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import os from typing import Dict, List, Optional import pandas as pd @@ -94,12 +93,18 @@ class FutuLoader: requires_auth = True def __init__(self) -> None: - self._host = os.environ.get("FUTU_HOST", "127.0.0.1") - self._port = int(os.environ.get("FUTU_PORT", "11111")) + from src.config.accessor import get_env_config + + cfg = get_env_config().data + self._host = cfg.futu_host + self._port = cfg.futu_port def is_available(self) -> bool: - """Return True if env vars are set and FutuOpenD is reachable.""" - if not os.environ.get("FUTU_HOST") or not os.environ.get("FUTU_PORT"): + """Return True if FutuOpenD is reachable.""" + from src.config.accessor import get_env_config + + cfg = get_env_config().data + if not cfg.futu_host or not cfg.futu_port: return False try: import futu # noqa: PLC0415 diff --git a/agent/backtest/loaders/okx.py b/agent/backtest/loaders/okx.py index c3a9fbd8..09b09db8 100644 --- a/agent/backtest/loaders/okx.py +++ b/agent/backtest/loaders/okx.py @@ -5,12 +5,15 @@ Supports 1m/5m/15m/30m/1H/4H/1D. Up to 300 bars per request; paginates with ``after`` for longer history. """ +import logging import time from typing import Dict, List, Optional import pandas as pd import requests +logger = logging.getLogger(__name__) + from backtest.loaders.base import ( cached_loader_fetch, check_budget, @@ -70,11 +73,11 @@ class DataLoader: validate_date_range(start_date, end_date) if fields: - print(f"[WARN] OKX ignores extra fields: {fields}") + logger.warning("OKX ignores extra fields: %s", fields) valid_intervals = {"1m", "5m", "15m", "30m", "1H", "4H", "1D"} if interval not in valid_intervals: - print(f"[WARN] unsupported OKX interval {interval}, using 1D") + logger.warning("unsupported OKX interval %s, using 1D", interval) interval = "1D" codes = [c.replace("/", "-").upper() for c in codes] @@ -101,7 +104,7 @@ class DataLoader: if df is not None and not df.empty: result[symbol] = df except Exception as exc: - print(f"[WARN] failed to fetch {symbol}: {exc}") + logger.warning("failed to fetch %s: %s", symbol, exc) return result def _fetch_candles( @@ -161,7 +164,7 @@ class DataLoader: after = str(oldest_ts) if not all_rows: - print(f"[WARN] OKX empty response: {inst_id}") + logger.warning("OKX empty response: %s", inst_id) return None columns = ["ts", "open", "high", "low", "close", "vol", "volCcy", "volCcyQuote", "confirm"] diff --git a/agent/backtest/loaders/sec_edgar_client.py b/agent/backtest/loaders/sec_edgar_client.py index b0af9fe0..c232dafb 100644 --- a/agent/backtest/loaders/sec_edgar_client.py +++ b/agent/backtest/loaders/sec_edgar_client.py @@ -25,7 +25,6 @@ shape. The ticker->CIK table is fetched once per process and memoized. from __future__ import annotations import logging -import os import threading from typing import Any, Dict, Optional @@ -64,7 +63,9 @@ def _min_interval() -> float: def _user_agent() -> str: """Return the compliant contact UA, honoring the ``VIBE_TRADING_SEC_UA`` override.""" - override = os.getenv(_UA_ENV) + from src.config.accessor import get_env_config + + override = get_env_config().data.vibe_trading_sec_ua if override and override.strip(): return override.strip() return _DEFAULT_SEC_UA diff --git a/agent/backtest/loaders/tiingo_loader.py b/agent/backtest/loaders/tiingo_loader.py index 83c9ee7c..8c784fda 100644 --- a/agent/backtest/loaders/tiingo_loader.py +++ b/agent/backtest/loaders/tiingo_loader.py @@ -16,7 +16,6 @@ and session reuse — Tiingo rate-limits by client and rejects unspaced bursts. from __future__ import annotations import logging -import os from typing import Dict, List, Optional import pandas as pd @@ -51,7 +50,9 @@ def _resolve_key() -> str: The trimmed ``TIINGO_API_KEY`` value, or ``""`` when absent or a known placeholder. """ - key = os.getenv(_AUTH_ENV, "").strip() + from src.config.accessor import get_env_config + + key = get_env_config().data.tiingo_api_key.strip() return "" if key.lower() in _KEY_PLACEHOLDERS else key diff --git a/agent/backtest/loaders/tushare.py b/agent/backtest/loaders/tushare.py index 4307e6d0..abee0728 100644 --- a/agent/backtest/loaders/tushare.py +++ b/agent/backtest/loaders/tushare.py @@ -4,7 +4,7 @@ Supports ``interval``: 1D (default) / 1m / 5m / 15m / 30m / 1H. Minute data uses ``pro.stk_mins()`` (Tushare points >= 2000). """ -import os +import logging from typing import Dict, List, Optional import pandas as pd @@ -13,6 +13,8 @@ from backtest.loaders._symbol_utils import _is_etf_listed from backtest.loaders.base import cached_loader_fetch, validate_date_range from backtest.loaders.registry import register +logger = logging.getLogger(__name__) + TUSHARE_TOKEN_PLACEHOLDERS = {"", "your-tushare-token"} @@ -55,13 +57,17 @@ class DataLoader: def is_available(self) -> bool: """Available when TUSHARE_TOKEN is set.""" - return os.getenv("TUSHARE_TOKEN", "").strip() not in TUSHARE_TOKEN_PLACEHOLDERS + from src.config.accessor import get_env_config + + return get_env_config().data.tushare_token.strip() not in TUSHARE_TOKEN_PLACEHOLDERS def __init__(self) -> None: """Initialize Tushare pro API.""" import tushare as ts - token = os.getenv("TUSHARE_TOKEN", "") + from src.config.accessor import get_env_config + + token = get_env_config().data.tushare_token self.api = ts.pro_api(token) def fetch( @@ -108,7 +114,7 @@ class DataLoader: ) return merged.get(code) except Exception as exc: - print(f"[WARN] failed to fetch {code}: {exc}") + logger.warning("failed to fetch %s: %s", code, exc) return None df = cached_loader_fetch( @@ -133,7 +139,7 @@ class DataLoader: ) -> Optional[pd.DataFrame]: """Fetch and normalize one daily OHLCV frame, routing by symbol type.""" if _is_us_equity(code) or _is_crypto(code): - print(f"[WARN] tushare does not support {code} (US/crypto); skipping") + logger.warning("tushare does not support %s (US/crypto); skipping", code) return None if _is_etf_listed(code): @@ -150,7 +156,7 @@ class DataLoader: df = self.api.daily(ts_code=code, start_date=start_date, end_date=end_date) if df is None or df.empty: - print(f"[WARN] tushare returned empty for {code} via {endpoint_name}") + logger.warning("tushare returned empty for %s via %s", code, endpoint_name) return None df = df.sort_values("trade_date") df["trade_date"] = pd.to_datetime(df["trade_date"]) @@ -215,7 +221,7 @@ class DataLoader: if f in basic.columns: result[code][f] = basic[f] except Exception as exc: - print(f"[WARN] daily_basic for {code} failed: {exc}") + logger.warning("daily_basic for %s failed: %s", code, exc) return result @@ -240,7 +246,7 @@ class DataLoader: freq_map = {"1m": "1min", "5m": "5min", "15m": "15min", "30m": "30min", "1H": "60min"} freq = freq_map.get(interval) if not freq: - print(f"[ERROR] unsupported Tushare interval: {interval}") + logger.error("unsupported Tushare interval: %s", interval) return {} sd = start_date.replace("-", "") @@ -250,7 +256,7 @@ class DataLoader: for code in codes: if _is_etf_listed(code): # tushare has no fund_mins endpoint; ETF intraday is unavailable - print(f"[WARN] tushare does not support intraday data for {code} (ETF); skipping") + logger.warning("tushare does not support intraday data for %s (ETF); skipping", code) continue if _is_index(code) or _is_hk_equity(code) or _is_us_equity(code) or _is_crypto(code): sym_type = ( @@ -259,12 +265,12 @@ class DataLoader: else "US" if _is_us_equity(code) else "crypto" ) - print(f"[WARN] tushare does not support intraday data for {code} ({sym_type}); skipping") + logger.warning("tushare does not support intraday data for %s (%s); skipping", code, sym_type) continue try: df = self.api.stk_mins(ts_code=code, freq=freq, start_date=sd, end_date=ed) if df is None or df.empty: - print(f"[WARN] empty Tushare minute data: {code} (points >= 2000 required)") + logger.warning("empty Tushare minute data: %s (points >= 2000 required)", code) continue df = df.sort_values("trade_time") df["trade_date"] = pd.to_datetime(df["trade_time"]) @@ -278,5 +284,5 @@ class DataLoader: ) result[code] = ohlcv except Exception as exc: - print(f"[WARN] failed to fetch minute data {code}: {exc}") + logger.warning("failed to fetch minute data %s: %s", code, exc) return result diff --git a/agent/backtest/loaders/tushare_fundamentals.py b/agent/backtest/loaders/tushare_fundamentals.py index 75c111c5..2e18d4d9 100644 --- a/agent/backtest/loaders/tushare_fundamentals.py +++ b/agent/backtest/loaders/tushare_fundamentals.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os from dataclasses import dataclass from typing import Any, Iterable @@ -115,7 +114,9 @@ class TushareFundamentalProvider: if api is None: import tushare as ts - token = os.getenv("TUSHARE_TOKEN", "").strip() + from src.config.accessor import get_env_config + + token = get_env_config().data.tushare_token.strip() if token in TUSHARE_TOKEN_PLACEHOLDERS: token = "" api = ts.pro_api(token) diff --git a/agent/backtest/loaders/yfinance_loader.py b/agent/backtest/loaders/yfinance_loader.py index 80dd6f60..f2e560d1 100644 --- a/agent/backtest/loaders/yfinance_loader.py +++ b/agent/backtest/loaders/yfinance_loader.py @@ -2,12 +2,15 @@ from __future__ import annotations +import logging from collections import defaultdict from typing import Dict, List, Optional, Union import pandas as pd import yfinance as yf +logger = logging.getLogger(__name__) + from backtest.loaders.base import ( loader_cache_get, loader_cache_put, @@ -284,7 +287,7 @@ class DataLoader: try: bulk_data = _download_history(pending, start_date, yf_end_date, yf_interval) except Exception as exc: - print(f"[WARN] yfinance bulk download failed for {pending}: {exc}") + logger.warning("yfinance bulk download failed for %s: %s", pending, exc) bulk_data = pd.DataFrame() for symbol in pending: @@ -295,7 +298,7 @@ class DataLoader: normalized = _normalize_frame(symbol_frame, requested_interval) if normalized.empty: - print(f"[WARN] yfinance returned no usable data for {symbol}") + logger.warning("yfinance returned no usable data for %s", symbol) continue loader_cache_put( @@ -310,7 +313,7 @@ class DataLoader: for original_code in symbol_groups[symbol]: results[original_code] = normalized.copy() except Exception as exc: - print(f"[WARN] Failed to fetch data for {symbol}: {exc}") + logger.warning("Failed to fetch data for %s: %s", symbol, exc) continue return results diff --git a/agent/cli/_legacy.py b/agent/cli/_legacy.py index d1df9e08..718d3b87 100644 --- a/agent/cli/_legacy.py +++ b/agent/cli/_legacy.py @@ -50,6 +50,7 @@ from rich.table import Table from rich.text import Text from cli.theme import get_console +from src.config.accessor import get_env_config, reset_env_config console = get_console() AGENT_DIR = Path(__file__).resolve().parents[1] @@ -151,8 +152,9 @@ def _build_status_parts(stats: _SessionStats) -> list[str]: Returns: List of status text segments. """ - provider = os.getenv("LANGCHAIN_PROVIDER", "") - model = os.getenv("LANGCHAIN_MODEL_NAME", "") + _cfg = get_env_config() + provider = _cfg.llm.langchain_provider + model = _cfg.llm.langchain_model_name model_short = model.split("/")[-1] if "/" in model else model label = f"{provider}/{model_short}" if provider else model_short or "unknown" @@ -1536,8 +1538,9 @@ def _build_welcome_panel(term_width: Optional[int] = None) -> Panel: term_width = term_width or _terminal_width() compact = term_width < 64 widths = _welcome_widths(term_width) - provider = os.getenv("LANGCHAIN_PROVIDER", "(not set)") - model = os.getenv("LANGCHAIN_MODEL_NAME", "(not set)") + _cfg = get_env_config() + provider = _cfg.llm.langchain_provider or "(not set)" + model = _cfg.llm.langchain_model_name or "(not set)" key_env = _provider_key_env(provider) key_value = os.getenv(key_env or "") credential_ready = provider in {"ollama", "openai-codex"} or bool(key_value) @@ -1724,8 +1727,9 @@ def _show_settings() -> None: term_width = _terminal_width() compact = term_width < 104 value_limit = max(18, min(56, term_width - 28)) - provider = os.getenv("LANGCHAIN_PROVIDER", "(not set)") - model = os.getenv("LANGCHAIN_MODEL_NAME", "(not set)") + _cfg = get_env_config() + provider = _cfg.llm.langchain_provider or "(not set)" + model = _cfg.llm.langchain_model_name or "(not set)" provider_key_env = _provider_key_env(provider) provider_base_env = _provider_base_env(provider) provider_key = os.getenv(provider_key_env or "") @@ -1741,9 +1745,9 @@ def _show_settings() -> None: runtime_table = Table.grid(expand=True) runtime_table.add_column(width=13, style="dim") runtime_table.add_column(ratio=1) - runtime_table.add_row("Temperature", os.getenv("LANGCHAIN_TEMPERATURE", "0.0")) - runtime_table.add_row("Timeout", os.getenv("TIMEOUT_SECONDS", "2400") + "s") - runtime_table.add_row("Retries", os.getenv("MAX_RETRIES", "(not set)")) + runtime_table.add_row("Temperature", str(_cfg.llm.langchain_temperature)) + runtime_table.add_row("Timeout", str(_cfg.llm.timeout_seconds) + "s") + runtime_table.add_row("Retries", str(_cfg.llm.max_retries)) credential_table = Table.grid(expand=True) credential_table.add_column(width=21, style="dim") @@ -1758,7 +1762,7 @@ def _show_settings() -> None: else: credential_table.add_row("Provider key", "(unknown provider)") credential_ready = False - credential_table.add_row("TUSHARE_TOKEN", "***" if os.getenv("TUSHARE_TOKEN") else "(optional)") + credential_table.add_row("TUSHARE_TOKEN", "***" if _cfg.data.tushare_token else "(optional)") panels = [ Panel(provider_table, title=f"Provider {_state_badge(provider if provider != '(not set)' else None)}", border_style="cyan", padding=(0, 1)), @@ -2831,7 +2835,8 @@ def _live_api_base() -> str: def _api_auth_headers() -> Dict[str, str]: """Return Bearer auth headers for CLI-to-API control calls.""" - key = (os.environ.get("VIBE_TRADING_API_KEY") or os.environ.get("API_AUTH_KEY") or "").strip() + reset_env_config() # ensure fresh read of auth credentials + key = get_env_config().api.api_auth_key.strip() return {"Authorization": f"Bearer {key}"} if key else {} diff --git a/agent/cli/commands/chat.py b/agent/cli/commands/chat.py index 35c319f5..a319696d 100644 --- a/agent/cli/commands/chat.py +++ b/agent/cli/commands/chat.py @@ -11,7 +11,6 @@ fallback workflows. from __future__ import annotations -import os from typing import Any, Optional from rich.console import Console @@ -50,8 +49,11 @@ def cmd_model(ctx: Any = None, *args: str) -> int: # noqa: ARG001 — ctx unuse _show_settings() except Exception as exc: # noqa: BLE001 — legacy may be absent on partial install - provider = os.getenv("LANGCHAIN_PROVIDER", "(not set)") - model = os.getenv("LANGCHAIN_MODEL_NAME", "(not set)") + from src.config.accessor import get_env_config + + cfg = get_env_config() + provider = cfg.llm.langchain_provider or "(not set)" + model = cfg.llm.langchain_model_name or "(not set)" console.print(Text(f"Provider: {provider}", style="bold")) console.print(Text(f"Model: {model}", style="bold")) console.print(Text(f"(legacy _show_settings unavailable: {exc})", style="dim")) diff --git a/agent/cli/main.py b/agent/cli/main.py index b92c0c45..313c179e 100644 --- a/agent/cli/main.py +++ b/agent/cli/main.py @@ -120,7 +120,9 @@ _SESSION_STORE_CACHE: Any = None def _probe_model_name() -> str: """Return the configured LLM model id, or a placeholder.""" - name = os.environ.get("LANGCHAIN_MODEL_NAME") or os.environ.get("OPENAI_MODEL") + from src.config.accessor import get_env_config + + name = get_env_config().llm.langchain_model_name or os.environ.get("OPENAI_MODEL") if name: return name env_path = _first_existing_env_path() diff --git a/agent/mcp_server.py b/agent/mcp_server.py index 067f7c3d..7befdb1d 100644 --- a/agent/mcp_server.py +++ b/agent/mcp_server.py @@ -80,7 +80,9 @@ _include_shell_tools = True def _env_shell_tools_enabled() -> bool: """Return whether shell tools were explicitly enabled for network MCP.""" - return os.getenv("VIBE_TRADING_ENABLE_SHELL_TOOLS", "").strip().lower() in {"1", "true", "yes", "on"} + from src.config.accessor import get_env_config + + return get_env_config().api.vibe_trading_enable_shell_tools def _get_skills_loader(): diff --git a/agent/scripts/bench_performance.py b/agent/scripts/bench_performance.py index be8ea085..4faafd4f 100644 --- a/agent/scripts/bench_performance.py +++ b/agent/scripts/bench_performance.py @@ -124,7 +124,9 @@ if __name__ == "__main__": from src.factors._backend import HAS_BOTTLENECK print(f"Bottleneck available: {HAS_BOTTLENECK}") - print(f"VIBE_TRADING_DISABLE_BOTTLENECK: {os.environ.get('VIBE_TRADING_DISABLE_BOTTLENECK', '0')}") + from src.config.accessor import get_env_config + + print(f"VIBE_TRADING_DISABLE_BOTTLENECK: {get_env_config().agent_tuning.vibe_trading_disable_bottleneck}") print() bench_operators() diff --git a/agent/scripts/w4a_run_benches.py b/agent/scripts/w4a_run_benches.py index f13273da..4f637c12 100644 --- a/agent/scripts/w4a_run_benches.py +++ b/agent/scripts/w4a_run_benches.py @@ -16,7 +16,6 @@ from __future__ import annotations import json import logging -import os import sys import time from datetime import datetime, timezone @@ -145,7 +144,9 @@ def main() -> int: "zoos": {}, } - tushare_ok = bool(os.getenv("TUSHARE_TOKEN", "").strip()) + from src.config.accessor import get_env_config + + tushare_ok = bool(get_env_config().data.tushare_token) if not tushare_ok: logger.warning("TUSHARE_TOKEN unset; csi300 benches will be skipped") diff --git a/agent/src/agent/loop.py b/agent/src/agent/loop.py index f0ba2cec..8eef047d 100644 --- a/agent/src/agent/loop.py +++ b/agent/src/agent/loop.py @@ -17,8 +17,8 @@ import concurrent.futures import copy import json import logging -import os import queue +import sys import threading import time as _time from datetime import datetime, timezone @@ -43,37 +43,79 @@ from src.providers.content_filter import ( MAX_CONSECUTIVE_CONTENT_FILTER_SKIPS, compute_content_filter_warnings, ) +from src.config.accessor import get_env_config from src.tools.background_tools import get_background_manager from src.tools.redaction import redact_payload RUNS_DIR = Path(__file__).resolve().parents[2] / "runs" SESSIONS_DIR = Path(__file__).resolve().parents[2] / "sessions" -TOKEN_THRESHOLD = int(os.getenv("TOKEN_THRESHOLD", "40000")) KEEP_RECENT = 3 TOOL_RESULT_LIMIT = 10_000 -HEARTBEAT_INTERVAL_S = float(os.getenv("VT_HEARTBEAT_INTERVAL_S", "3.0")) -REASONING_DELTA_MIN_INTERVAL_S = float(os.getenv("VT_REASONING_DELTA_MIN_INTERVAL_S", "1.0")) -STREAM_RETRY_DELAY_S = float(os.getenv("VT_STREAM_RETRY_DELAY_S", "1.0")) -TOOL_TIMEOUT_SECONDS = float(os.getenv("VIBE_TRADING_TOOL_TIMEOUT_SECONDS", "1800")) -GOAL_MAX_CONTINUATIONS = int(os.getenv("VIBE_TRADING_GOAL_MAX_CONTINUATIONS", "3")) LLM_USAGE_ARTIFACT = "llm_usage.json" -# Layer 1: microcompact only prunes old tool results once the transcript is -# under real memory pressure. Below this it would clear tool history on every -# iteration even on short runs, discarding results the model may still need to -# reference (metrics, file contents, search hits). -MICROCOMPACT_THRESHOLD = int(TOKEN_THRESHOLD * 0.5) - -# Layer 2: Context collapse thresholds -COLLAPSE_THRESHOLD = int(TOKEN_THRESHOLD * 0.7) COLLAPSE_PRESERVE_RECENT = 6 COLLAPSE_TEXT_MIN = 2400 COLLAPSE_HEAD = 900 COLLAPSE_TAIL = 500 -# Layer 3: Token-budget tail protection TAIL_TOKEN_BUDGET = 20_000 + +def _override(name: str): + """Return a monkeypatched module-level override if present.""" + mod = sys.modules.get(__name__) + if mod is not None and name in mod.__dict__: + return mod.__dict__[name] + return None + + +def _token_threshold() -> int: + ov = _override("TOKEN_THRESHOLD") + if ov is not None: + return ov + from src.config.accessor import get_env_config + return get_env_config().agent_tuning.token_threshold + + +def _heartbeat_interval_s() -> float: + ov = _override("HEARTBEAT_INTERVAL_S") + if ov is not None: + return ov + from src.config.accessor import get_env_config + return get_env_config().agent_tuning.vt_heartbeat_interval_s + + +def _reasoning_delta_min_interval_s() -> float: + ov = _override("REASONING_DELTA_MIN_INTERVAL_S") + if ov is not None: + return ov + from src.config.accessor import get_env_config + return get_env_config().agent_tuning.vt_reasoning_delta_min_interval_s + + +def _stream_retry_delay_s() -> float: + ov = _override("STREAM_RETRY_DELAY_S") + if ov is not None: + return ov + from src.config.accessor import get_env_config + return get_env_config().agent_tuning.vt_stream_retry_delay_s + + +def _tool_timeout_seconds() -> float: + ov = _override("TOOL_TIMEOUT_SECONDS") + if ov is not None: + return ov + from src.config.accessor import get_env_config + return get_env_config().agent_tuning.vibe_trading_tool_timeout_seconds + + +def _goal_max_continuations() -> int: + ov = _override("GOAL_MAX_CONTINUATIONS") + if ov is not None: + return ov + from src.config.accessor import get_env_config + return get_env_config().agent_tuning.vibe_trading_goal_max_continuations + logger = logging.getLogger(__name__) @@ -111,8 +153,10 @@ def _normalize_llm_usage(usage: Any) -> dict[str, int] | None: def _new_llm_usage_summary(llm: Any) -> dict[str, Any]: """Create the run-scoped provider usage accumulator.""" - provider = os.getenv("LANGCHAIN_PROVIDER", "openai").strip() or "openai" - model = getattr(llm, "model_name", None) or os.getenv("LANGCHAIN_MODEL_NAME", "").strip() + from src.config.accessor import get_env_config + cfg = get_env_config() + provider = cfg.llm.langchain_provider.strip() or "openai" + model = getattr(llm, "model_name", None) or cfg.llm.langchain_model_name.strip() return { "provider": provider, "model": model, @@ -615,18 +659,19 @@ class AgentLoop: # memory pressure, so short, low-pressure runs keep their full # tool history available for the model to reference instead of # having every result past the most recent few cleared. - if tokens > MICROCOMPACT_THRESHOLD: + if tokens > int(_token_threshold() * 0.5): _microcompact(messages) tokens = estimate_tokens(messages) # Layer 2: context collapse (fold long text, zero API cost) - if tokens > COLLAPSE_THRESHOLD: + if tokens > int(_token_threshold() * 0.7): _context_collapse(messages) tokens = estimate_tokens(messages) # Layer 3: auto_compact (token threshold exceeded) - if tokens > TOKEN_THRESHOLD: - logger.info(f"Auto compact triggered: {tokens} tokens > {TOKEN_THRESHOLD}") + _tok_threshold = _token_threshold() + if tokens > _tok_threshold: + logger.info(f"Auto compact triggered: {tokens} tokens > {_tok_threshold}") self._auto_compact(messages, run_dir, trace, iteration=current_iter) logger.info(f"ReAct iteration {iteration}/{self.max_iterations}") @@ -668,7 +713,7 @@ class AgentLoop: now = _time.monotonic() if ( last_reasoning_emit is not None - and now - last_reasoning_emit < REASONING_DELTA_MIN_INTERVAL_S + and now - last_reasoning_emit < _reasoning_delta_min_interval_s() ): return last_reasoning_emit = now @@ -716,7 +761,7 @@ class AgentLoop: thinking_chunks.clear() reasoning_chars = 0 last_reasoning_emit = None - _time.sleep(STREAM_RETRY_DELAY_S) + _time.sleep(_stream_retry_delay_s()) response = self.llm.stream_chat( messages, tools=tool_defs, @@ -815,14 +860,15 @@ class AgentLoop: { "type": "empty_model_response", "iter": current_iter, - "provider": os.getenv("LANGCHAIN_PROVIDER", "openai"), - "model": getattr(self.llm, "model_name", None) or os.getenv("LANGCHAIN_MODEL_NAME", ""), + "provider": get_env_config().llm.langchain_provider, + "model": getattr(self.llm, "model_name", None) or get_env_config().llm.langchain_model_name, } ) break should_continue_goal = False continuation_snapshot = None - if active_goal_id and session_id and GOAL_MAX_CONTINUATIONS > 0: + _max_cont = _goal_max_continuations() + if active_goal_id and session_id and _max_cont > 0: try: if goal_store is None: from src.goal import GoalStore @@ -842,7 +888,7 @@ class AgentLoop: goal_last_progress is not None and current_progress <= goal_last_progress ) - if goal_continuations >= GOAL_MAX_CONTINUATIONS or ( + if goal_continuations >= _max_cont or ( no_new_progress and goal_continuations > 0 ): trace.write( @@ -964,8 +1010,9 @@ class AgentLoop: state_store.mark_success(run_dir) final_status = "success" elif empty_model_response_iter is not None: - provider = os.getenv("LANGCHAIN_PROVIDER", "openai").strip().lower() or "openai" - model = getattr(self.llm, "model_name", None) or os.getenv("LANGCHAIN_MODEL_NAME", "").strip() or "(unset)" + _cfg = get_env_config() + provider = _cfg.llm.langchain_provider.strip().lower() or "openai" + model = getattr(self.llm, "model_name", None) or _cfg.llm.langchain_model_name.strip() or "(unset)" final_reason = ( "empty_model_response: " f"provider={provider} model={model} iteration {empty_model_response_iter} " @@ -1209,7 +1256,7 @@ class AgentLoop: Installs a thread-local progress emitter so the tool may call ``emit_progress()`` without taking a callback parameter, and runs a - background heartbeat timer that ticks every ``HEARTBEAT_INTERVAL_S`` + background heartbeat timer that ticks every ``_heartbeat_interval_s()`` seconds. Both event streams are forwarded through ``self._emit`` and therefore land in the same SSE bus and CLI dashboard as normal tool events. @@ -1237,7 +1284,8 @@ class AgentLoop: self._emit("tool_heartbeat", payload) t0 = _time.perf_counter() - timeout = TOOL_TIMEOUT_SECONDS if TOOL_TIMEOUT_SECONDS > 0 else None + _tool_timeout = _tool_timeout_seconds() + timeout = _tool_timeout if _tool_timeout > 0 else None timeout_label = _format_timeout(timeout) if timeout is not None else "" def _elapsed_ms() -> int: @@ -1256,7 +1304,7 @@ class AgentLoop: """ return HeartbeatTimer( tool_name=tool_name, - interval=HEARTBEAT_INTERVAL_S, + interval=_heartbeat_interval_s(), emit=_on_heartbeat, ) @@ -1539,3 +1587,21 @@ class AgentLoop: def _update_memory(self, tool_name: str) -> None: """Update workspace memory counters after tool execution.""" self.memory.increment(tool_name) + + +_LEGACY_LAZY = { + "TOKEN_THRESHOLD": _token_threshold, + "MICROCOMPACT_THRESHOLD": lambda: int(_token_threshold() * 0.5), + "COLLAPSE_THRESHOLD": lambda: int(_token_threshold() * 0.7), + "HEARTBEAT_INTERVAL_S": _heartbeat_interval_s, + "REASONING_DELTA_MIN_INTERVAL_S": _reasoning_delta_min_interval_s, + "STREAM_RETRY_DELAY_S": _stream_retry_delay_s, + "TOOL_TIMEOUT_SECONDS": _tool_timeout_seconds, + "GOAL_MAX_CONTINUATIONS": _goal_max_continuations, +} + + +def __getattr__(name: str): + if name in _LEGACY_LAZY: + return _LEGACY_LAZY[name]() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/agent/src/api/scheduled_routes.py b/agent/src/api/scheduled_routes.py index 6ba2907d..55b04f3b 100644 --- a/agent/src/api/scheduled_routes.py +++ b/agent/src/api/scheduled_routes.py @@ -15,6 +15,8 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional from fastapi import Depends, FastAPI, HTTPException, Query, status from pydantic import BaseModel, Field +from src.config.accessor import get_env_config + logger = logging.getLogger(__name__) @@ -36,10 +38,7 @@ _scheduled_research_executor: Any = None def _scheduled_research_scheduler_enabled() -> bool: """Return whether scheduled research execution is enabled.""" - return ( - os.getenv(_SCHEDULED_RESEARCH_SCHEDULER_ENV, "").strip().lower() - in _SCHEDULED_RESEARCH_TRUE_VALUES - ) + return get_env_config().agent_tuning.vibe_trading_enable_scheduler def _get_scheduled_research_store(): diff --git a/agent/src/api/security.py b/agent/src/api/security.py index 9d20e486..97507630 100644 --- a/agent/src/api/security.py +++ b/agent/src/api/security.py @@ -14,6 +14,7 @@ from fastapi.responses import JSONResponse from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from src.api._compat import host_attr as _host_attr +from src.config.accessor import _parse_bool, get_env_config # ============================================================================ @@ -68,7 +69,9 @@ def _parse_extra_loopback_hosts(raw: Optional[str]) -> set[str]: return {host.strip().lower().rstrip(".") for host in raw.split(",") if host.strip()} -_EXTRA_LOOPBACK_HOSTS = _parse_extra_loopback_hosts(os.getenv("API_ALLOWED_HOSTS")) +def _get_extra_loopback_hosts() -> set[str]: + from src.config.accessor import get_env_config + return _parse_extra_loopback_hosts(get_env_config().api.api_allowed_hosts or None) def _host_without_port(host: str) -> str: @@ -89,7 +92,7 @@ def _host_without_port(host: str) -> str: def _is_allowed_loopback_host(host: str) -> bool: """Return whether *host* is allowed for loopback-trusted API requests.""" normalized = _host_without_port(host) - return normalized in _DEFAULT_LOOPBACK_HOSTS or normalized in _EXTRA_LOOPBACK_HOSTS + return normalized in _DEFAULT_LOOPBACK_HOSTS or normalized in _get_extra_loopback_hosts() def _is_loopback_bind_host(host: str) -> bool: @@ -100,7 +103,9 @@ def _is_loopback_bind_host(host: str) -> bool: return host == "localhost" -_CORS_ORIGINS = _parse_cors_origins(os.getenv("CORS_ORIGINS")) +def _get_cors_origins() -> List[str]: + from src.config.accessor import get_env_config + return _parse_cors_origins(get_env_config().api.cors_origins or None) # ============================================================================ @@ -123,12 +128,25 @@ async def _reject_untrusted_loopback_host(request: Request, call_next): # ============================================================================ _security = HTTPBearer(auto_error=False) -_API_KEY = os.getenv("API_AUTH_KEY") + + +def _get_api_key() -> str: + from src.config.accessor import get_env_config + return get_env_config().api.api_auth_key def _configured_api_key() -> str: - """Return the current API auth key, if configured.""" - return os.getenv("API_AUTH_KEY") or _host_attr("_API_KEY", _API_KEY) or "" + """Return the current API auth key, if configured. + + Resolves the ``VIBE_TRADING_API_KEY`` alias server-side via + :func:`get_env_config` so both ``API_AUTH_KEY`` and the legacy + alias produce the same key. + """ + return ( + get_env_config().api.api_auth_key + or _host_attr("_API_KEY", _get_api_key()) + or "" + ) def _auth_credential_from_header_or_query( @@ -263,7 +281,7 @@ def _is_local_client(request: Request) -> bool: def _env_flag_enabled(name: str) -> bool: """Return whether a boolean environment flag is enabled.""" - return os.getenv(name, "").strip().lower() in {"1", "true", "yes", "on"} + return _parse_bool(os.getenv(name)) def _default_gateway_ips() -> set[ipaddress.IPv4Address]: @@ -360,3 +378,16 @@ async def require_settings_write_auth( status_code=status.HTTP_403_FORBIDDEN, detail="Settings writes require API_AUTH_KEY or a local loopback client", ) + + +_LEGACY_LAZY_NAMES = { + "_API_KEY": _get_api_key, + "_CORS_ORIGINS": _get_cors_origins, + "_EXTRA_LOOPBACK_HOSTS": _get_extra_loopback_hosts, +} + + +def __getattr__(name: str): + if name in _LEGACY_LAZY_NAMES: + return _LEGACY_LAZY_NAMES[name]() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/agent/src/api/settings_routes.py b/agent/src/api/settings_routes.py index ae8d9ed0..21fdb98a 100644 --- a/agent/src/api/settings_routes.py +++ b/agent/src/api/settings_routes.py @@ -15,6 +15,8 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional from fastapi import Depends, FastAPI, HTTPException, status from pydantic import BaseModel, Field +from src.config.accessor import reset_env_config + # Agent root (agent/) — resolved from this file's location (agent/src/api/). _AGENT_DIR = Path(__file__).resolve().parent.parent.parent @@ -272,6 +274,8 @@ def _sync_runtime_env(provider: LLMProviderOption, updates: Dict[str, str]) -> N os.environ.pop("OPENAI_API_BASE", None) os.environ.pop("OPENAI_BASE_URL", None) + reset_env_config() + # --------------------------------------------------------------------------- # Registration @@ -424,6 +428,7 @@ def register_settings_routes( os.environ["TUSHARE_TOKEN"] = token else: os.environ.pop("TUSHARE_TOKEN", None) + reset_env_config() return _build_data_source_settings_response( host_ref._read_env_values(host_ref.ENV_PATH) diff --git a/agent/src/api/state.py b/agent/src/api/state.py index 92d18d8c..c4350d8b 100644 --- a/agent/src/api/state.py +++ b/agent/src/api/state.py @@ -8,6 +8,7 @@ from fastapi import HTTPException from src.api._compat import host_attr as _host_attr, set_host_attr as _set_host_attr from src.api.helpers import RUNS_DIR, SESSIONS_DIR +from src.config.accessor import get_env_config # ============================================================================ @@ -38,7 +39,7 @@ def _get_session_service(): elif _session_service is not None: return _session_service - if os.getenv("ENABLE_SESSION_RUNTIME", "true").lower() != "true": + if not get_env_config().api.enable_session_runtime: return None import asyncio diff --git a/agent/src/config/accessor.py b/agent/src/config/accessor.py new file mode 100644 index 00000000..32f6f5ed --- /dev/null +++ b/agent/src/config/accessor.py @@ -0,0 +1,132 @@ +"""Lazy singleton accessor for EnvConfig. + +Use :func:`get_env_config` to get the cached config instance. The first call +creates an :class:`~src.config.env_schema.EnvConfig` (which reads +``os.environ``) and caches it; subsequent calls return the same object. + +Call :func:`reset_env_config` after modifying ``os.environ`` at runtime +(e.g. in the Settings API) so the next :func:`get_env_config` call picks up +the new values. + +Thread-safety is guaranteed by a module-level :class:`threading.Lock` — +swarm workers and the agent loop both run in threads and may call +:func:`get_env_config` concurrently. +""" + +from __future__ import annotations + +import os +import threading +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from src.config.env_schema import EnvConfig + +__all__ = ["get_env_config", "reset_env_config", "_parse_bool", "get_env_or"] + +# --------------------------------------------------------------------------- +# Module-level singleton state +# --------------------------------------------------------------------------- + +_instance: EnvConfig | None = None +_lock = threading.Lock() + +# --------------------------------------------------------------------------- +# Truthy / falsy string sets for the unified boolean parser +# --------------------------------------------------------------------------- + +_TRUTHY: frozenset[str] = frozenset({"1", "true", "yes", "on"}) + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def get_env_config() -> EnvConfig: + """Return the cached :class:`EnvConfig` singleton, creating it on first access. + + Thread-safe: acquires a lock before checking / creating the instance so + concurrent callers (swarm workers, agent loop) never see a half-built + config or create duplicate instances. + + Returns: + The shared :class:`EnvConfig` instance. + """ + global _instance # noqa: PLW0603 + if _instance is not None: + return _instance + with _lock: + # Double-checked locking: another thread may have created the + # instance while we were waiting for the lock. + if _instance is not None: + return _instance + # Import here to avoid a circular import at module-load time and + # to keep the heavy Pydantic model out of the import path for + # callers that only need ``_parse_bool`` or ``get_env_or``. + from src.config.env_schema import EnvConfig as _EnvConfig + + _instance = _EnvConfig() + return _instance + + +def reset_env_config() -> None: + """Clear the cached :class:`EnvConfig` singleton. + + Acquires the same lock used by :func:`get_env_config` so the reset is + atomic with respect to concurrent readers. After this call the next + :func:`get_env_config` invocation will create a fresh instance that + reflects any ``os.environ`` mutations made since the previous access. + + Typical caller: ``settings_routes.py`` after writing new values to + ``agent/.env`` and patching ``os.environ``. + """ + global _instance # noqa: PLW0603 + with _lock: + _instance = None + + +def _parse_bool(value: str | None) -> bool: + """Parse a string (or ``None``) into a boolean. + + Truthy values (case-insensitive): ``"1"``, ``"true"``, ``"yes"``, ``"on"``. + Everything else — including ``None``, ``""``, ``"0"``, ``"false"``, + ``"no"``, ``"off"`` — returns ``False``. + + This is the single source of truth for boolean env-var parsing across the + codebase, replacing 6+ duplicated inline patterns. + + Args: + value: The raw string value (typically from ``os.getenv``) or ``None``. + + Returns: + ``True`` when *value* is a recognised truthy string; ``False`` otherwise. + """ + if value is None: + return False + return value.strip().lower() in _TRUTHY + + +def get_env_or(primary: str, fallback: str, default: str = "") -> str: + """Read an env var with a backward-compatible alias fallback. + + Several env vars were renamed during the centralization effort (e.g. + ``VIBE_TRADING_API_KEY`` → ``API_AUTH_KEY``). This helper reads the + *primary* (new) name first, falls back to the *fallback* (old) name, + and finally returns *default* when neither is set. + + Args: + primary: The preferred (new) environment variable name. + fallback: The legacy alias to try when *primary* is unset or empty. + default: Value returned when neither variable is set. + + Returns: + The first non-empty value found, or *default*. + """ + value = os.getenv(primary) + if value: + return value + value = os.getenv(fallback) + if value: + return value + return default diff --git a/agent/src/config/env_schema.py b/agent/src/config/env_schema.py new file mode 100644 index 00000000..e4d677e1 --- /dev/null +++ b/agent/src/config/env_schema.py @@ -0,0 +1,327 @@ +"""Single source of truth for all Vibe-Trading environment variable defaults. + +This module defines Pydantic models for every environment variable consumed by +the Vibe-Trading agent, grouped by functional category. Each field carries the +correct type, default value, and env-var alias so that ``EnvConfig()`` with no +arguments reads from ``os.environ`` and applies the documented defaults. + +The schema replaces the ~207 scattered ``os.getenv`` calls across 66 files with +a validated, typed, centrally-managed configuration layer. + +Usage:: + + from src.config.env_schema import EnvConfig + + cfg = EnvConfig() # reads os.environ, applies defaults + cfg.llm.timeout_seconds # 120 + cfg.data.tushare_token # "" +""" + +from __future__ import annotations + +import os +from typing import Annotated, Any + +from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, model_validator + +__all__ = [ + "EnvConfig", + "LLMConfig", + "DataConfig", + "APIConfig", + "SwarmConfig", + "AgentTuningConfig", + "PathConfig", +] + + +# --------------------------------------------------------------------------- +# Boolean coercion helper +# --------------------------------------------------------------------------- + + +def _parse_env_bool(v: Any) -> Any: + """Coerce environment string values to ``bool``. + + Accepts ``"1"``, ``"true"``, ``"yes"``, ``"on"`` (case-insensitive) as + ``True`` and ``"0"``, ``"false"``, ``"no"``, ``"off"``, ``""`` as ``False``. + Non-string values pass through for Pydantic's built-in coercion. + """ + if isinstance(v, str): + low = v.strip().lower() + if low in {"1", "true", "yes", "on"}: + return True + if low in {"0", "false", "no", "off", ""}: + return False + return v + + +EnvBool = Annotated[bool, BeforeValidator(_parse_env_bool)] + + +# --------------------------------------------------------------------------- +# Base class +# --------------------------------------------------------------------------- + + +class _EnvBase(BaseModel): + """Base for environment-variable-backed config sub-models. + + Each sub-model reads missing fields from ``os.environ`` using the field + alias (the UPPER_SNAKE_CASE env-var name). Explicit constructor arguments + always take precedence over environment values. + """ + + model_config = ConfigDict(populate_by_name=True, extra="ignore") + + @model_validator(mode="before") + @classmethod + def _load_from_env(cls, data: Any) -> Any: + """Populate missing fields from ``os.environ`` using field aliases. + + Invalid numeric env values are silently dropped so Pydantic applies + the field default instead of raising a ``ValidationError``. + """ + if not isinstance(data, dict): + return data + result = dict(data) + for field_name, field_info in cls.model_fields.items(): + alias = field_info.alias + # Skip fields already provided (by Python name or alias). + if field_name in result or (alias and alias in result): + continue + # Read from environment using the UPPER_SNAKE_CASE alias. + if alias and alias in os.environ: + env_val = os.environ[alias] + # Safe coercion for numeric fields: drop unparseable values + # so Pydantic falls back to the field default. + annotation = field_info.annotation + if annotation is int and isinstance(env_val, str): + try: + env_val = int(env_val) + except ValueError: + continue + elif annotation is float and isinstance(env_val, str): + try: + env_val = float(env_val) + except ValueError: + continue + result[alias] = env_val + return result + + +# --------------------------------------------------------------------------- +# LLM Provider +# --------------------------------------------------------------------------- + + +class LLMConfig(_EnvBase): + """LLM provider and generation parameters. + + Sources: ``src/providers/llm.py``, ``src/providers/chat.py``, + ``src/providers/openai_codex.py``, ``src/preflight.py``. + """ + + langchain_provider: str = Field(alias="LANGCHAIN_PROVIDER", default="openai") + langchain_model_name: str = Field(alias="LANGCHAIN_MODEL_NAME", default="") + langchain_temperature: float = Field(alias="LANGCHAIN_TEMPERATURE", default=0.0) + timeout_seconds: int = Field(alias="TIMEOUT_SECONDS", default=120) + max_retries: int = Field(alias="MAX_RETRIES", default=2) + langchain_reasoning_effort: str = Field(alias="LANGCHAIN_REASONING_EFFORT", default="") + vibe_trading_deepseek_adapter: str = Field(alias="VIBE_TRADING_DEEPSEEK_ADAPTER", default="auto") + moonshot_user_agent: str = Field(alias="MOONSHOT_USER_AGENT", default="") + openai_codex_base_url: str = Field( + alias="OPENAI_CODEX_BASE_URL", + default="https://chatgpt.com/backend-api/codex/responses", + ) + + +# --------------------------------------------------------------------------- +# Data Sources +# --------------------------------------------------------------------------- + + +class DataConfig(_EnvBase): + """Market-data source credentials and tuning. + + Sources: ``backtest/loaders/*.py``, ``src/tools/web_search_tool.py``, + ``src/tools/iwencai_tool.py``, ``src/tools/fred_macro_tool.py``. + """ + + tushare_token: str = Field(alias="TUSHARE_TOKEN", default="") + ccxt_exchange: str = Field(alias="CCXT_EXCHANGE", default="binance") + ccxt_timeout_ms: int = Field(alias="CCXT_TIMEOUT_MS", default=15000) + ccxt_fetch_budget_s: float = Field(alias="CCXT_FETCH_BUDGET_S", default=60.0) + futu_host: str = Field(alias="FUTU_HOST", default="127.0.0.1") + futu_port: int = Field(alias="FUTU_PORT", default=11111) + finnhub_api_key: str = Field(alias="FINNHUB_API_KEY", default="") + alphavantage_api_key: str = Field(alias="ALPHAVANTAGE_API_KEY", default="") + tiingo_api_key: str = Field(alias="TIINGO_API_KEY", default="") + fmp_api_key: str = Field(alias="FMP_API_KEY", default="") + fred_api_key: str = Field(alias="FRED_API_KEY", default="") + vibe_trading_iwencai_key: str = Field(alias="VIBE_TRADING_IWENCAI_KEY", default="") + vibe_trading_sec_ua: str = Field(alias="VIBE_TRADING_SEC_UA", default="") + vibe_trading_data_cache: EnvBool = Field(alias="VIBE_TRADING_DATA_CACHE", default=False) + vibe_trading_data_cache_root: str = Field(alias="VIBE_TRADING_DATA_CACHE_ROOT", default="") + aliyun_iqs_api_key: str = Field(alias="ALIYUN_IQS_API_KEY", default="") + + +# --------------------------------------------------------------------------- +# API Server +# --------------------------------------------------------------------------- + + +class APIConfig(_EnvBase): + """API server authentication, CORS, and security toggles. + + Sources: ``src/api/security.py``, ``src/api/state.py``, + ``mcp_server.py``, ``src/tools/path_utils.py``. + """ + + api_auth_key: str = Field(alias="API_AUTH_KEY", default="") + vibe_trading_api_key: str = Field(alias="VIBE_TRADING_API_KEY", default="") + cors_origins: str = Field(alias="CORS_ORIGINS", default="") + api_allowed_hosts: str = Field(alias="API_ALLOWED_HOSTS", default="") + enable_session_runtime: EnvBool = Field(alias="ENABLE_SESSION_RUNTIME", default=True) + vibe_trading_trust_docker_loopback: EnvBool = Field( + alias="VIBE_TRADING_TRUST_DOCKER_LOOPBACK", default=False, + ) + vibe_trading_enable_shell_tools: EnvBool = Field( + alias="VIBE_TRADING_ENABLE_SHELL_TOOLS", default=False, + ) + vibe_trading_allowed_file_roots: str = Field( + alias="VIBE_TRADING_ALLOWED_FILE_ROOTS", default="", + ) + vibe_trading_allowed_write_roots: str = Field( + alias="VIBE_TRADING_ALLOWED_WRITE_ROOTS", default="", + ) + vibe_trading_allowed_run_roots: str = Field( + alias="VIBE_TRADING_ALLOWED_RUN_ROOTS", default="", + ) + + +# --------------------------------------------------------------------------- +# Swarm +# --------------------------------------------------------------------------- + + +class SwarmConfig(_EnvBase): + """Multi-agent swarm execution parameters. + + Sources: ``src/swarm/worker.py``, ``src/tools/swarm_tool.py``, + ``src/swarm/runtime.py``, ``src/swarm/grounding.py``. + """ + + swarm_worker_timeout: int = Field(alias="SWARM_WORKER_TIMEOUT", default=300) + swarm_worker_max_iter: int = Field(alias="SWARM_WORKER_MAX_ITER", default=50) + swarm_max_workers: int = Field(alias="SWARM_MAX_WORKERS", default=4) + swarm_timeout: int = Field(alias="SWARM_TIMEOUT", default=1800) + swarm_heartbeat_interval_s: float = Field(alias="SWARM_HEARTBEAT_INTERVAL_S", default=3.0) + swarm_stream_retry_delay_s: float = Field(alias="SWARM_STREAM_RETRY_DELAY_S", default=1.0) + swarm_grounding_max_symbols: int = Field(alias="SWARM_GROUNDING_MAX_SYMBOLS", default=8) + + +# --------------------------------------------------------------------------- +# Agent Tuning +# --------------------------------------------------------------------------- + + +class AgentTuningConfig(_EnvBase): + """Agent loop tuning, content-filter, scheduler, and feature flags. + + Sources: ``src/agent/loop.py``, ``src/providers/content_filter.py``, + ``src/scheduled_research/executor.py``, ``src/live/order_guard.py``, + ``src/factors/_backend.py``, ``src/factors/bench_runner.py``, + ``src/tools/web_search_tool.py``, ``api_server.py``. + """ + + token_threshold: int = Field(alias="TOKEN_THRESHOLD", default=40000) + vt_heartbeat_interval_s: float = Field(alias="VT_HEARTBEAT_INTERVAL_S", default=3.0) + vt_reasoning_delta_min_interval_s: float = Field( + alias="VT_REASONING_DELTA_MIN_INTERVAL_S", default=1.0, + ) + vt_stream_retry_delay_s: float = Field(alias="VT_STREAM_RETRY_DELAY_S", default=1.0) + vibe_trading_tool_timeout_seconds: float = Field( + alias="VIBE_TRADING_TOOL_TIMEOUT_SECONDS", default=1800.0, + ) + vibe_trading_goal_max_continuations: int = Field( + alias="VIBE_TRADING_GOAL_MAX_CONTINUATIONS", default=3, + ) + vibe_trading_sse_timeout: int = Field(alias="VIBE_TRADING_SSE_TIMEOUT", default=90) + content_filter_warning_threshold: float = Field( + alias="CONTENT_FILTER_WARNING_THRESHOLD", default=0.05, + ) + vibe_trading_enable_advisory: EnvBool = Field( + alias="VIBE_TRADING_ENABLE_ADVISORY", default=False, + ) + vibe_trading_enable_scheduler: EnvBool = Field( + alias="VIBE_TRADING_ENABLE_SCHEDULER", default=False, + ) + vibe_trading_channels_auto_start: EnvBool = Field( + alias="VIBE_TRADING_CHANNELS_AUTO_START", default=False, + ) + vibe_trading_disable_bottleneck: EnvBool = Field( + alias="VIBE_TRADING_DISABLE_BOTTLENECK", default=False, + ) + vibe_trading_bench_workers: int = Field(alias="VIBE_TRADING_BENCH_WORKERS", default=0) + vibe_trading_search_backends: str = Field(alias="VIBE_TRADING_SEARCH_BACKENDS", default="") + vibe_trading_search_bing_fallback: EnvBool = Field( + alias="VIBE_TRADING_SEARCH_BING_FALLBACK", default=True, + ) + + +# --------------------------------------------------------------------------- +# Paths +# --------------------------------------------------------------------------- + + +class PathConfig(_EnvBase): + """File-system path overrides and session-level MCP trust. + + Sources: ``src/hypotheses/registry.py``, ``src/goal/store.py``, + ``src/config/loader.py``. + """ + + vibe_trading_hypotheses_path: str = Field(alias="VIBE_TRADING_HYPOTHESES_PATH", default="") + vibe_trading_goal_db_path: str = Field(alias="VIBE_TRADING_GOAL_DB_PATH", default="") + vibe_trading_swarm_agent_config: str = Field( + alias="VIBE_TRADING_SWARM_AGENT_CONFIG", default="", + ) + allow_session_mcp_servers: EnvBool = Field(alias="ALLOW_SESSION_MCP_SERVERS", default=False) + + +# --------------------------------------------------------------------------- +# Top-level composition +# --------------------------------------------------------------------------- + + +class EnvConfig(_EnvBase): + """Root configuration model composing all environment variable groups. + + Instantiating with no arguments reads every recognised env var from + ``os.environ`` and applies the documented defaults for missing values:: + + cfg = EnvConfig() + cfg.llm.timeout_seconds # 120 + cfg.swarm.swarm_timeout # 1800 + """ + + llm: LLMConfig = Field(default_factory=LLMConfig) + data: DataConfig = Field(default_factory=DataConfig) + api: APIConfig = Field(default_factory=APIConfig) + swarm: SwarmConfig = Field(default_factory=SwarmConfig) + agent_tuning: AgentTuningConfig = Field(default_factory=AgentTuningConfig) + paths: PathConfig = Field(default_factory=PathConfig) + + @model_validator(mode="after") + def _resolve_api_key_alias(self) -> "EnvConfig": + """Copy ``VIBE_TRADING_API_KEY`` to ``api_auth_key`` when only the alias is set. + + The CLI historically reads ``VIBE_TRADING_API_KEY`` first and falls back to + ``API_AUTH_KEY`` (``cli/_legacy.py:2834``). The API server only reads + ``API_AUTH_KEY`` (``src/api/security.py:126``). This validator closes + the semantic gap so both surfaces agree. + """ + if self.api.vibe_trading_api_key and not self.api.api_auth_key: + self.api.api_auth_key = self.api.vibe_trading_api_key + return self diff --git a/agent/src/factors/_backend.py b/agent/src/factors/_backend.py index e01eff5b..201af8e7 100644 --- a/agent/src/factors/_backend.py +++ b/agent/src/factors/_backend.py @@ -9,27 +9,60 @@ fall back to the original pandas path — identical results, slower speed. Note: ``bn.move_rank`` uses a fundamentally different normalization (Spearman rank correlation) than our ``ts_rank`` (percentile rank), so ``ts_rank`` uses numpy ``sliding_window_view`` instead. + +``HAS_BOTTLENECK`` and ``bn`` are resolved lazily on first access via +:func:`__getattr__` so the ``VIBE_TRADING_DISABLE_BOTTLENECK`` env var +is read through :func:`get_env_config` instead of at import time. """ from __future__ import annotations -import os +from typing import Any -_DISABLE = os.environ.get("VIBE_TRADING_DISABLE_BOTTLENECK", "0") == "1" +# numpy sliding_window_view — always available (numpy >= 1.20) +from numpy.lib.stride_tricks import sliding_window_view -HAS_BOTTLENECK: bool = False -bn = None +__all__ = ["HAS_BOTTLENECK", "bn", "sliding_window_view"] -if not _DISABLE: +# --------------------------------------------------------------------------- +# Lazy bottleneck initialisation +# --------------------------------------------------------------------------- + +_bn_initialised: bool = False +_has_bottleneck: bool = False +_bn_module: Any = None + + +def _ensure_bottleneck() -> None: + """Attempt to import bottleneck on first access. + + Reads ``VIBE_TRADING_DISABLE_BOTTLENECK`` through the lazy + :func:`get_env_config` accessor so runtime Settings API changes + take effect before the first factor computation. + """ + global _bn_initialised, _has_bottleneck, _bn_module # noqa: PLW0603 + if _bn_initialised: + return + _bn_initialised = True + from src.config.accessor import get_env_config + + if get_env_config().agent_tuning.vibe_trading_disable_bottleneck: + return try: import bottleneck as _bn - bn = _bn - HAS_BOTTLENECK = True + _bn_module = _bn + _has_bottleneck = True except ImportError: pass -# numpy sliding_window_view — always available (numpy >= 1.20) -from numpy.lib.stride_tricks import sliding_window_view # noqa: E402 -__all__ = ["HAS_BOTTLENECK", "bn", "sliding_window_view"] +def __getattr__(name: str) -> Any: + """Lazily resolve ``HAS_BOTTLENECK`` and ``bn`` on first attribute access.""" + if name == "HAS_BOTTLENECK": + _ensure_bottleneck() + return _has_bottleneck + if name == "bn": + _ensure_bottleneck() + return _bn_module + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/agent/src/factors/bench_runner.py b/agent/src/factors/bench_runner.py index 1f26627f..c1c37156 100644 --- a/agent/src/factors/bench_runner.py +++ b/agent/src/factors/bench_runner.py @@ -22,6 +22,7 @@ import time from concurrent.futures import ProcessPoolExecutor, as_completed from typing import Any, Callable, Iterable +from src.config.accessor import get_env_config from src.factors.factor_analysis_core import compute_ic_series from src.factors.registry import ( Registry, @@ -205,7 +206,7 @@ def run_bench( n_total = len(alpha_ids) try: - n_workers = int(os.environ.get("VIBE_TRADING_BENCH_WORKERS", "0") or os.cpu_count() or 1) + n_workers = get_env_config().agent_tuning.vibe_trading_bench_workers or os.cpu_count() or 1 except ValueError: logger.warning("invalid VIBE_TRADING_BENCH_WORKERS; falling back to sequential") n_workers = 1 diff --git a/agent/src/goal/store.py b/agent/src/goal/store.py index 74f4a10e..3173126e 100644 --- a/agent/src/goal/store.py +++ b/agent/src/goal/store.py @@ -16,6 +16,7 @@ from functools import wraps from pathlib import Path from typing import Callable, TypeVar +from src.config.accessor import get_env_config from src.goal.models import ( AuditRow, EvidenceInput, @@ -70,7 +71,7 @@ def _json_loads(value: str | None, default: object) -> object: def _default_db_path() -> Path: """Return the configured goal ledger database path.""" - raw_path = os.getenv(_DB_PATH_ENV, "").strip() + raw_path = get_env_config().paths.vibe_trading_goal_db_path.strip() if raw_path: return Path(raw_path).expanduser() return _DEFAULT_DB_PATH diff --git a/agent/src/hypotheses/registry.py b/agent/src/hypotheses/registry.py index bf6846cc..795c8cc5 100644 --- a/agent/src/hypotheses/registry.py +++ b/agent/src/hypotheses/registry.py @@ -15,6 +15,8 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +from src.config.accessor import get_env_config + HYPOTHESIS_STATUSES = ( "exploring", "testing", @@ -34,7 +36,7 @@ def default_hypotheses_path() -> Path: Env override path when ``VIBE_TRADING_HYPOTHESES_PATH`` is set, otherwise ``~/.vibe-trading/hypotheses.json``. """ - override = os.environ.get(_ENV_PATH, "").strip() + override = get_env_config().paths.vibe_trading_hypotheses_path.strip() if override: return Path(override).expanduser() return Path.home() / ".vibe-trading" / "hypotheses.json" diff --git a/agent/src/live/order_guard.py b/agent/src/live/order_guard.py index 04a33d21..68ef635a 100644 --- a/agent/src/live/order_guard.py +++ b/agent/src/live/order_guard.py @@ -42,6 +42,7 @@ import os from datetime import datetime, timezone from typing import Any +from src.config.accessor import get_env_config from src.live.advisory import ( AdvisoryContext, AdvisoryOrchestrator, @@ -390,8 +391,7 @@ class LiveOrderGuardTool(MCPRemoteTool): Never raises — all exceptions are caught and converted to REVIEW_UNAVAILABLE. """ - env_val = os.getenv(_ADVISORY_ENABLED_ENV, "").strip().lower() - if env_val not in _ADVISORY_TRUTHY: + if not get_env_config().agent_tuning.vibe_trading_enable_advisory: return None providers = get_advisory_providers() diff --git a/agent/src/preflight.py b/agent/src/preflight.py index 0867fc7a..bfe409ff 100644 --- a/agent/src/preflight.py +++ b/agent/src/preflight.py @@ -15,6 +15,8 @@ from typing import List, Optional from rich.console import Console from rich.table import Table +from src.config.accessor import get_env_config + @dataclass(frozen=True) class CheckResult: @@ -32,8 +34,9 @@ def _check_llm_provider() -> CheckResult: from src.providers.llm import _ensure_dotenv, _sync_provider_env, provider_diagnostics _ensure_dotenv() - provider = os.getenv("LANGCHAIN_PROVIDER", "").strip() - model = os.getenv("LANGCHAIN_MODEL_NAME", "").strip() + _cfg = get_env_config() + provider = _cfg.llm.langchain_provider.strip() + model = _cfg.llm.langchain_model_name.strip() if not provider: return CheckResult( @@ -185,7 +188,7 @@ def _check_yfinance() -> CheckResult: def _check_tushare() -> CheckResult: """Check Tushare token configuration.""" - token = os.getenv("TUSHARE_TOKEN", "").strip() + token = get_env_config().data.tushare_token.strip() if not token or token == "your-tushare-token": return CheckResult( name="Tushare", @@ -221,11 +224,7 @@ def _check_akshare() -> CheckResult: def _check_content_filter_threshold() -> CheckResult: """Report the configured content filter warning threshold.""" - raw = os.getenv("CONTENT_FILTER_WARNING_THRESHOLD", "0.05") - try: - threshold = float(raw) - except (TypeError, ValueError): - threshold = 0.05 + threshold = get_env_config().agent_tuning.content_filter_warning_threshold return CheckResult( name="Content Filter Threshold", status="ready", diff --git a/agent/src/providers/chat.py b/agent/src/providers/chat.py index 276d6b11..753a3987 100644 --- a/agent/src/providers/chat.py +++ b/agent/src/providers/chat.py @@ -11,6 +11,7 @@ import re from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Optional +from src.config.accessor import get_env_config from src.providers.content_filter import is_content_filter_triggered from src.providers.llm import build_llm @@ -304,8 +305,9 @@ class ChatLLM: on_text_chunk(pending_text) return response except Exception as exc: - provider = os.getenv("LANGCHAIN_PROVIDER", "openai").strip().lower() or "openai" - model = self.model_name or os.getenv("LANGCHAIN_MODEL_NAME", "").strip() or "(unset)" + _cfg = get_env_config() + provider = _cfg.llm.langchain_provider.strip().lower() or "openai" + model = self.model_name or _cfg.llm.langchain_model_name.strip() or "(unset)" raise ProviderStreamError(provider=provider, model=model, original=exc) from exc @staticmethod diff --git a/agent/src/providers/content_filter.py b/agent/src/providers/content_filter.py index 77792837..a20d2ab7 100644 --- a/agent/src/providers/content_filter.py +++ b/agent/src/providers/content_filter.py @@ -12,6 +12,8 @@ from __future__ import annotations import os +from src.config.accessor import get_env_config + CONTENT_FILTER_WARNING_THRESHOLD_ENV = "CONTENT_FILTER_WARNING_THRESHOLD" DEFAULT_CONTENT_FILTER_THRESHOLD = 0.05 @@ -67,14 +69,7 @@ def get_content_filter_threshold() -> float: Reads ``CONTENT_FILTER_WARNING_THRESHOLD`` (default 0.05 = 5%). Invalid values fall back to the default instead of crashing the run. """ - raw = os.getenv( - CONTENT_FILTER_WARNING_THRESHOLD_ENV, - str(DEFAULT_CONTENT_FILTER_THRESHOLD), - ) - try: - return float(raw) - except (TypeError, ValueError): - return DEFAULT_CONTENT_FILTER_THRESHOLD + return get_env_config().agent_tuning.content_filter_warning_threshold def compute_content_filter_warnings( diff --git a/agent/src/providers/llm.py b/agent/src/providers/llm.py index aca6f567..64cf7742 100644 --- a/agent/src/providers/llm.py +++ b/agent/src/providers/llm.py @@ -13,6 +13,7 @@ from urllib.parse import urlsplit from pydantic import PrivateAttr +from src.config.accessor import get_env_config, reset_env_config from src.providers.capabilities import get_provider_capabilities, provider_env_names try: @@ -371,7 +372,7 @@ def _redact_proxy_url(name: str, raw: str | None) -> str: def _deepseek_adapter_mode() -> str: """Return the configured DeepSeek adapter mode.""" - mode = os.getenv("VIBE_TRADING_DEEPSEEK_ADAPTER", "auto").strip().lower() + mode = get_env_config().llm.vibe_trading_deepseek_adapter.strip().lower() aliases = { "compat": "openai-compatible", "compatible": "openai-compatible", @@ -406,8 +407,8 @@ def _build_native_deepseek( return chat_deepseek( model=model, temperature=temperature, - timeout=int(os.getenv("TIMEOUT_SECONDS", "120")), - max_retries=int(os.getenv("MAX_RETRIES", "2")), + timeout=get_env_config().llm.timeout_seconds, + max_retries=get_env_config().llm.max_retries, callbacks=callbacks, api_key=api_key or None, base_url=base_url or None, @@ -447,8 +448,8 @@ def _ensure_dotenv() -> None: logger.info( "dotenv resolved from %s | provider=%s model=%s base=%s", _redact_env_source(loaded), - os.getenv("LANGCHAIN_PROVIDER", "(unset)"), - os.getenv("LANGCHAIN_MODEL_NAME", "(unset)"), + get_env_config().llm.langchain_provider, + get_env_config().llm.langchain_model_name or "(unset)", _redact_base_url_for_log(os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE")), ) @@ -471,16 +472,18 @@ def _sync_provider_env() -> None: api_key_env=None means no key required (e.g. Ollama local). """ _ensure_dotenv() - provider = os.getenv("LANGCHAIN_PROVIDER", "openai").lower() + reset_env_config() + provider = get_env_config().llm.langchain_provider.lower() if provider in {"openai-codex", "openai_codex"}: - codex_url = os.getenv("OPENAI_CODEX_BASE_URL", "https://chatgpt.com/backend-api/codex/responses") + codex_url = get_env_config().llm.openai_codex_base_url + # SDK-side env setup, not Vibe-Trading config reads os.environ["OPENAI_API_BASE"] = codex_url os.environ["OPENAI_BASE_URL"] = codex_url os.environ.pop("OPENAI_API_KEY", None) return - key_env, base_env = provider_env_names(provider, os.getenv("LANGCHAIN_MODEL_NAME", "")) + key_env, base_env = provider_env_names(provider, get_env_config().llm.langchain_model_name) # Resolve API key: provider-specific env → OPENAI_API_KEY fallback if key_env is not None: @@ -493,6 +496,7 @@ def _sync_provider_env() -> None: if provider == "ollama" and base_url: base_url = _normalize_ollama_base_url(base_url) + # SDK-side env setup, not Vibe-Trading config reads if api_key: os.environ["OPENAI_API_KEY"] = api_key if base_url: @@ -507,8 +511,8 @@ def provider_diagnostics() -> dict[str, Any]: Redacted provider/model/package/env/proxy/capability details. """ _sync_provider_env() - provider = os.getenv("LANGCHAIN_PROVIDER", "openai").strip().lower() - model = os.getenv("LANGCHAIN_MODEL_NAME", "").strip() + provider = get_env_config().llm.langchain_provider.strip().lower() + model = get_env_config().llm.langchain_model_name.strip() caps = get_provider_capabilities(provider, model) key_env, base_env = provider_env_names(provider, model) base_url = os.getenv(base_env, "") or os.getenv("OPENAI_BASE_URL", "") or os.getenv("OPENAI_API_BASE", "") @@ -545,9 +549,9 @@ def provider_diagnostics() -> dict[str, Any]: if os.getenv(name) }, "packages": {name: _package_version(name) for name in package_names}, - "timeout_seconds": int(os.getenv("TIMEOUT_SECONDS", "120")), - "max_retries": int(os.getenv("MAX_RETRIES", "2")), - "reasoning_effort": os.getenv("LANGCHAIN_REASONING_EFFORT", "").strip().lower(), + "timeout_seconds": get_env_config().llm.timeout_seconds, + "max_retries": get_env_config().llm.max_retries, + "reasoning_effort": get_env_config().llm.langchain_reasoning_effort.strip().lower(), "adapter": { "type": adapter_type, "mode": adapter_mode, @@ -577,20 +581,20 @@ def build_llm(*, model_name: Optional[str] = None, callbacks: Any = None) -> Any RuntimeError: If langchain-openai is missing or LANGCHAIN_MODEL_NAME is unset. """ _sync_provider_env() - name = model_name or os.getenv("LANGCHAIN_MODEL_NAME", "").strip() + name = model_name or get_env_config().llm.langchain_model_name.strip() if not name: raise RuntimeError("LANGCHAIN_MODEL_NAME is not set") - temperature = float(os.getenv("LANGCHAIN_TEMPERATURE", "0.0")) - provider = os.getenv("LANGCHAIN_PROVIDER", "openai").lower() + temperature = get_env_config().llm.langchain_temperature + provider = get_env_config().llm.langchain_provider.lower() caps = get_provider_capabilities(provider, name) if provider in {"openai-codex", "openai_codex"}: from src.providers.openai_codex import OpenAICodexLLM - effort = os.getenv("LANGCHAIN_REASONING_EFFORT", "").strip().lower() + effort = get_env_config().llm.langchain_reasoning_effort.strip().lower() return OpenAICodexLLM( model=name, temperature=temperature, - timeout=int(os.getenv("TIMEOUT_SECONDS", "120")), + timeout=get_env_config().llm.timeout_seconds, reasoning_effort=effort or None, ) @@ -622,12 +626,12 @@ def build_llm(*, model_name: Optional[str] = None, callbacks: Any = None) -> Any temperature = 1.0 # Optional reasoning activation for relays requiring opt-in (e.g. OpenRouter). # Moonshot/DeepSeek official APIs emit reasoning by default and ignore this field. - effort = os.getenv("LANGCHAIN_REASONING_EFFORT", "").strip().lower() + effort = get_env_config().llm.langchain_reasoning_effort.strip().lower() kwargs: dict[str, Any] = { "model": name, "temperature": temperature, - "timeout": int(os.getenv("TIMEOUT_SECONDS", "120")), - "max_retries": int(os.getenv("MAX_RETRIES", "2")), + "timeout": get_env_config().llm.timeout_seconds, + "max_retries": get_env_config().llm.max_retries, "callbacks": callbacks, "extra_body": {"reasoning": {"effort": effort}} if effort and caps.openrouter_reasoning_body else None, "vibe_provider": provider, @@ -635,7 +639,7 @@ def build_llm(*, model_name: Optional[str] = None, callbacks: Any = None) -> Any if caps.default_headers: headers = dict(caps.default_headers) if caps.name == "moonshot": - custom_ua = os.getenv("MOONSHOT_USER_AGENT", "").strip() + custom_ua = get_env_config().llm.moonshot_user_agent.strip() if custom_ua: headers["User-Agent"] = custom_ua kwargs["default_headers"] = headers diff --git a/agent/src/providers/openai_codex.py b/agent/src/providers/openai_codex.py index c78dc5f4..67ceb8c7 100644 --- a/agent/src/providers/openai_codex.py +++ b/agent/src/providers/openai_codex.py @@ -16,6 +16,8 @@ from dataclasses import dataclass, field from typing import Any, Callable, Iterable, Optional from urllib.parse import urlparse +from src.config.accessor import get_env_config + try: import httpx except ImportError: @@ -356,7 +358,7 @@ class OpenAICodexLLM: self.tools = tools or [] self.reasoning_effort = reasoning_effort self.codex_url = validate_codex_base_url( - codex_url or os.getenv("OPENAI_CODEX_BASE_URL", DEFAULT_CODEX_URL) + codex_url or get_env_config().llm.openai_codex_base_url ) def bind_tools(self, tools: list[dict[str, Any]]) -> "OpenAICodexLLM": diff --git a/agent/src/scheduled_research/executor.py b/agent/src/scheduled_research/executor.py index fbe9d730..85258bbc 100644 --- a/agent/src/scheduled_research/executor.py +++ b/agent/src/scheduled_research/executor.py @@ -15,6 +15,7 @@ import time from datetime import datetime, timedelta, timezone from typing import Awaitable, Callable +from src.config.accessor import get_env_config from src.scheduled_research.models import JobStatus, ScheduledResearchJob, validate_schedule from src.scheduled_research.store import ScheduledResearchJobStore @@ -45,8 +46,9 @@ def scheduler_enabled_from_env(value: str | None = None) -> bool: The feature is disabled by default. Pass *value* in tests to avoid mutating process environment. """ - raw = os.getenv(SCHEDULER_ENABLED_ENV, "") if value is None else value - return raw.strip().lower() in _TRUE_VALUES + if value is not None: + return value.strip().lower() in _TRUE_VALUES + return get_env_config().agent_tuning.vibe_trading_enable_scheduler def is_due(job: ScheduledResearchJob, now_ms: int) -> bool: diff --git a/agent/src/swarm/grounding.py b/agent/src/swarm/grounding.py index 72edd22e..67be2b23 100644 --- a/agent/src/swarm/grounding.py +++ b/agent/src/swarm/grounding.py @@ -55,6 +55,8 @@ import re from datetime import date, timedelta from typing import Iterable +from src.config.accessor import get_env_config + logger = logging.getLogger(__name__) @@ -143,17 +145,7 @@ def extract_symbols_from_user_vars(user_vars: dict[str, str]) -> list[str]: def max_grounding_symbols() -> int: """Return the configured cap for symbols fetched into worker prompts.""" - raw = os.getenv(MAX_SYMBOLS_ENV, "").strip() - if not raw: - return DEFAULT_MAX_SYMBOLS - try: - value = int(raw) - except ValueError: - logger.warning( - "grounding: invalid %s=%r, using default %d", - MAX_SYMBOLS_ENV, raw, DEFAULT_MAX_SYMBOLS, - ) - return DEFAULT_MAX_SYMBOLS + value = get_env_config().swarm.swarm_grounding_max_symbols return max(1, value) diff --git a/agent/src/swarm/runtime.py b/agent/src/swarm/runtime.py index c9191037..7dca76fc 100644 --- a/agent/src/swarm/runtime.py +++ b/agent/src/swarm/runtime.py @@ -20,6 +20,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Callable +from src.config.accessor import get_env_config from src.config.schema import AgentConfig from src.swarm import grounding from src.swarm.models import ( @@ -124,8 +125,9 @@ class SwarmRuntime: # provider layer uses (src/providers/llm.py:136,195) — that way an # override applied via os.environ still shows up. Per-agent overrides # remain visible on SwarmAgentSpec.model_name. - run.provider = (os.getenv("LANGCHAIN_PROVIDER") or "").strip().lower() or None - run.model = (os.getenv("LANGCHAIN_MODEL_NAME") or "").strip() or None + _cfg = get_env_config() + run.provider = _cfg.llm.langchain_provider.strip().lower() or None + run.model = _cfg.llm.langchain_model_name.strip() or None self._store.create_run(run) @@ -434,10 +436,7 @@ class SwarmRuntime: ), ) - try: - interval = float(os.getenv("SWARM_HEARTBEAT_INTERVAL_S", "3.0")) - except ValueError: - interval = 3.0 + interval = get_env_config().swarm.swarm_heartbeat_interval_s try: with HeartbeatTimer( diff --git a/agent/src/swarm/store.py b/agent/src/swarm/store.py index 68feb3a6..a9060620 100644 --- a/agent/src/swarm/store.py +++ b/agent/src/swarm/store.py @@ -18,6 +18,8 @@ import time from datetime import datetime, timezone from pathlib import Path +from src.config.accessor import get_env_config + from src.swarm.models import SwarmEvent, SwarmRun from src.tools.redaction import redact_internal_paths @@ -327,10 +329,7 @@ class SwarmStore: Returns: Seconds of event silence after which the run should be reaped. """ - try: - interval = float(os.getenv("SWARM_HEARTBEAT_INTERVAL_S", "3.0")) - except ValueError: - interval = 3.0 + interval = get_env_config().swarm.swarm_heartbeat_interval_s heartbeat_floor = max(60.0, interval * 10.0) agent_budgets = [ diff --git a/agent/src/swarm/worker.py b/agent/src/swarm/worker.py index 61644271..4182136b 100644 --- a/agent/src/swarm/worker.py +++ b/agent/src/swarm/worker.py @@ -37,8 +37,14 @@ from src.tools.redaction import is_sensitive_arg, redact_payload logger = logging.getLogger(__name__) -_DEFAULT_MAX_ITERATIONS = int(os.getenv("SWARM_WORKER_MAX_ITER", "50")) -_DEFAULT_TIMEOUT_SECONDS = int(os.getenv("SWARM_WORKER_TIMEOUT", "300")) +def _default_max_iterations() -> int: + from src.config.accessor import get_env_config + return get_env_config().swarm.swarm_worker_max_iter + + +def _default_timeout_seconds() -> int: + from src.config.accessor import get_env_config + return get_env_config().swarm.swarm_worker_timeout def _heartbeat_interval_s() -> float: @@ -334,8 +340,8 @@ def run_worker( """ agent_id = agent_spec.id task_id = task.id - max_iterations = agent_spec.max_iterations or _DEFAULT_MAX_ITERATIONS - timeout = agent_spec.timeout_seconds or _DEFAULT_TIMEOUT_SECONDS + max_iterations = agent_spec.max_iterations or _default_max_iterations() + timeout = agent_spec.timeout_seconds or _default_timeout_seconds() _emit(event_callback, "worker_started", agent_id, task_id) diff --git a/agent/src/tools/alpha_bench_tool.py b/agent/src/tools/alpha_bench_tool.py index bd898154..aac60f18 100644 --- a/agent/src/tools/alpha_bench_tool.py +++ b/agent/src/tools/alpha_bench_tool.py @@ -38,6 +38,7 @@ from typing import Any import pandas as pd from src.agent.tools import BaseTool +from src.config.accessor import get_env_config logger = logging.getLogger(__name__) @@ -251,7 +252,7 @@ def _load_csi300_panel(start: str, end: str) -> dict[str, pd.DataFrame]: the requested window; if that call fails we degrade to a 30-name blue-chip fallback so the bench still runs. """ - token = os.getenv("TUSHARE_TOKEN", "").strip() + token = get_env_config().data.tushare_token.strip() if not token or token == "your-tushare-token": raise RuntimeError( "TUSHARE_TOKEN not in agent/.env or environment; required for csi300 universe" diff --git a/agent/src/tools/fred_macro_tool.py b/agent/src/tools/fred_macro_tool.py index 724dcc66..b723f5da 100644 --- a/agent/src/tools/fred_macro_tool.py +++ b/agent/src/tools/fred_macro_tool.py @@ -25,6 +25,7 @@ from typing import Any from backtest.loaders._http import resolve_min_interval, throttled_get_json from src.agent.tools import BaseTool +from src.config.accessor import get_env_config logger = logging.getLogger(__name__) @@ -103,7 +104,7 @@ class FredMacroTool(BaseTool): ``True`` when ``FRED_API_KEY`` is set in the environment, otherwise ``False`` so the tool is silently excluded from the registry. """ - return bool(os.getenv("FRED_API_KEY")) + return bool(get_env_config().data.fred_api_key) def execute(self, **kwargs: Any) -> str: """Fetch one FRED series and return a JSON envelope. @@ -119,7 +120,7 @@ class FredMacroTool(BaseTool): "data": {"series_id", "observations": [{"date", "value"}, ...], "count"}}``. On failure: ``{"ok": false, "error": str}``. """ - api_key = os.getenv("FRED_API_KEY") + api_key = get_env_config().data.fred_api_key or None if not api_key: return _error("FRED_API_KEY is not configured") diff --git a/agent/src/tools/iwencai_tool.py b/agent/src/tools/iwencai_tool.py index 1a3b0d70..7be78213 100644 --- a/agent/src/tools/iwencai_tool.py +++ b/agent/src/tools/iwencai_tool.py @@ -23,6 +23,7 @@ from typing import Any from backtest.loaders._http import resolve_min_interval, throttled_get_json from src.agent.tools import BaseTool +from src.config.accessor import get_env_config logger = logging.getLogger(__name__) @@ -158,7 +159,7 @@ class IWenCaiSearchTool(BaseTool): value; ``False`` otherwise, which silently excludes the tool from the registry. Never raises. """ - return bool(os.getenv(_KEY_ENV)) + return bool(get_env_config().data.vibe_trading_iwencai_key) description = ( "Run a natural-language A-share research query against iWenCai (问财), a " @@ -204,7 +205,7 @@ class IWenCaiSearchTool(BaseTool): "data": {"query": ..., "count": int, "results": [...]}}``. On failure: ``{"ok": false, "error": "..."}``. """ - key = os.getenv(_KEY_ENV) + key = get_env_config().data.vibe_trading_iwencai_key or None if not key: return self._error( f"iWenCai access key not configured; set {_KEY_ENV} to enable this tool" diff --git a/agent/src/tools/path_utils.py b/agent/src/tools/path_utils.py index 156c236f..2c36caa6 100644 --- a/agent/src/tools/path_utils.py +++ b/agent/src/tools/path_utils.py @@ -20,6 +20,8 @@ from __future__ import annotations import os from pathlib import Path +from src.config.accessor import get_env_config + _ALLOWED_FILE_ROOTS_ENV = "VIBE_TRADING_ALLOWED_FILE_ROOTS" _ALLOWED_RUN_ROOTS_ENV = "VIBE_TRADING_ALLOWED_RUN_ROOTS" @@ -68,7 +70,7 @@ def _agent_root() -> Path: def _configured_file_roots() -> list[Path]: """Return file roots configured through the environment.""" - raw = os.getenv(_ALLOWED_FILE_ROOTS_ENV, "") + raw = get_env_config().api.vibe_trading_allowed_file_roots roots: list[Path] = [] for item in raw.split(","): item = item.strip() @@ -125,7 +127,7 @@ _ALLOWED_WRITE_ROOTS_ENV = "VIBE_TRADING_ALLOWED_WRITE_ROOTS" def allowed_write_roots() -> list[Path]: """Return all roots allowed for file writes and edits.""" - raw = os.getenv(_ALLOWED_WRITE_ROOTS_ENV, "") + raw = get_env_config().api.vibe_trading_allowed_write_roots configured: list[Path] = [] for item in raw.split(","): item = item.strip() @@ -213,7 +215,7 @@ def resolve_safe_path( def _allowed_run_roots() -> list[Path]: """Return all roots allowed for run_dir-based tools.""" - raw = os.getenv(_ALLOWED_RUN_ROOTS_ENV, "") + raw = get_env_config().api.vibe_trading_allowed_run_roots configured: list[Path] = [] for item in raw.split(","): item = item.strip() diff --git a/agent/src/tools/swarm_tool.py b/agent/src/tools/swarm_tool.py index 70e3486c..592c91fb 100644 --- a/agent/src/tools/swarm_tool.py +++ b/agent/src/tools/swarm_tool.py @@ -19,7 +19,15 @@ from src.agent.tools import BaseTool logger = logging.getLogger(__name__) _POLL_INTERVAL_SECONDS = 5 -_MAX_WAIT_SECONDS = int(os.getenv("SWARM_TIMEOUT", "1800")) + + +def _max_wait_seconds() -> int: + import sys as _sys + _mod = _sys.modules.get(__name__) + if _mod is not None and "_MAX_WAIT_SECONDS" in _mod.__dict__: + return _mod.__dict__["_MAX_WAIT_SECONDS"] + from src.config.accessor import get_env_config + return get_env_config().swarm.swarm_timeout # Preset matching: (preset_name, keyword_patterns, weight_boost). Patterns match user intent (EN + ZH). _PRESET_KEYWORDS: list[tuple[str, list[str], float]] = [ @@ -804,7 +812,8 @@ class SwarmTool(BaseTool): pending_live_events.clear() t0 = time.monotonic() - while time.monotonic() - t0 < _MAX_WAIT_SECONDS: + max_wait = _max_wait_seconds() + while time.monotonic() - t0 < max_wait: time.sleep(_POLL_INTERVAL_SECONDS) loaded = store.load_run(run_id) @@ -830,7 +839,7 @@ class SwarmTool(BaseTool): ) return json.dumps( - {"status": "timeout", "error": f"Swarm run {run_id} timed out after {_MAX_WAIT_SECONDS}s"}, + {"status": "timeout", "error": f"Swarm run {run_id} timed out after {max_wait}s"}, ensure_ascii=False, ) @@ -875,3 +884,9 @@ def _format_result( }, } return json.dumps(result, ensure_ascii=False, indent=2) + + +def __getattr__(name: str): + if name == "_MAX_WAIT_SECONDS": + return _max_wait_seconds() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/agent/src/tools/web_search_tool.py b/agent/src/tools/web_search_tool.py index ab65eb87..0f3057ed 100644 --- a/agent/src/tools/web_search_tool.py +++ b/agent/src/tools/web_search_tool.py @@ -18,6 +18,7 @@ import time from typing import Any from src.agent.tools import BaseTool +from src.config.accessor import _parse_bool, get_env_config from src.security.scanner import with_security_warnings logger = logging.getLogger(__name__) @@ -36,7 +37,7 @@ def _aliyun_iqs_search(query: str, max_results: int = 5) -> list[dict] | None: category and rerank. Requires ``ALIYUN_IQS_API_KEY``; returns None when unset. Pure stdlib. """ - key = os.getenv("ALIYUN_IQS_API_KEY", "").strip() + key = get_env_config().data.aliyun_iqs_api_key.strip() if not key: return None import json as _json @@ -181,12 +182,12 @@ class WebSearchTool(BaseTool): """ query = kwargs["query"] max_results = min(int(kwargs.get("max_results", 5)), 10) - backends = os.getenv("VIBE_TRADING_SEARCH_BACKENDS", _DEFAULT_BACKENDS).strip() or "auto" + backends = (get_env_config().agent_tuning.vibe_trading_search_backends or _DEFAULT_BACKENDS).strip() or "auto" # Fast path: Alibaba Cloud IQS if configured (official API, CN-direct, # ~1s, structured + snippet, best quality). Skip ddgs entirely when IQS # is available — ddgs engines are unreachable from typical CN egress. - if os.getenv("ALIYUN_IQS_API_KEY", "").strip(): + if get_env_config().data.aliyun_iqs_api_key.strip(): try: raw = _aliyun_iqs_search(query, max_results=max_results) if raw: @@ -287,7 +288,7 @@ class WebSearchTool(BaseTool): # query quality), then cn.bing (real URLs, broader). Toggle via # VIBE_TRADING_SEARCH_BING_FALLBACK (default on). fb_err = "disabled" - if os.getenv("VIBE_TRADING_SEARCH_BING_FALLBACK", "1").strip().lower() in ("1", "true", "yes"): + if get_env_config().agent_tuning.vibe_trading_search_bing_fallback: for fb_name, fb_fn in (("sogou", _sogou_search), ("bing_cn", _bing_cn_search)): try: raw = fb_fn(query, max_results=max_results) diff --git a/agent/tests/conftest.py b/agent/tests/conftest.py index 0c3d14c7..cf5be8f8 100644 --- a/agent/tests/conftest.py +++ b/agent/tests/conftest.py @@ -5,7 +5,18 @@ from __future__ import annotations import sys from pathlib import Path +import pytest + # Ensure agent/ is on sys.path so imports like `backtest.*` and `src.*` work. AGENT_DIR = Path(__file__).resolve().parent.parent if str(AGENT_DIR) not in sys.path: sys.path.insert(0, str(AGENT_DIR)) + + +@pytest.fixture(autouse=True) +def _reset_env_config(): + """Clear the cached EnvConfig before each test so monkeypatch.setenv works.""" + from src.config.accessor import reset_env_config + reset_env_config() + yield + reset_env_config() diff --git a/agent/tests/test_env_schema.py b/agent/tests/test_env_schema.py new file mode 100644 index 00000000..a4751d90 --- /dev/null +++ b/agent/tests/test_env_schema.py @@ -0,0 +1,471 @@ +"""Comprehensive tests for EnvConfig schema and accessor singleton. + +Covers: +- Default values for every sub-model +- Type coercion (int, float, bool) +- Validation errors on bad input +- Environment variable override +- API key alias resolution +- Singleton caching and reset +- Thread safety +- _parse_bool utility +- get_env_or utility +- _parse_env_bool (EnvBool BeforeValidator) +""" + +from __future__ import annotations + +import concurrent.futures + +import pytest +from pydantic import ValidationError + +from src.config.accessor import ( + _parse_bool, + get_env_config, + get_env_or, + reset_env_config, +) +from src.config.env_schema import ( + APIConfig, + AgentTuningConfig, + DataConfig, + EnvConfig, + LLMConfig, + PathConfig, + SwarmConfig, + _parse_env_bool, +) + + +# --------------------------------------------------------------------------- +# Fixture: reset singleton + clean env before each test +# --------------------------------------------------------------------------- + +# All env-var aliases used by EnvConfig sub-models. +_ALL_ALIASES: list[str] = [] +for _model in (LLMConfig, DataConfig, APIConfig, SwarmConfig, AgentTuningConfig, PathConfig): + for _info in _model.model_fields.values(): + if _info.alias: + _ALL_ALIASES.append(_info.alias) + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Remove all EnvConfig-related env vars and reset the singleton.""" + for alias in _ALL_ALIASES: + monkeypatch.delenv(alias, raising=False) + reset_env_config() + yield # type: ignore[misc] + reset_env_config() + + +# =================================================================== +# TestEnvConfigDefaults — every sub-model's defaults +# =================================================================== + + +class TestEnvConfigDefaults: + """Verify every field defaults correctly when no env vars are set.""" + + def test_llm_defaults(self) -> None: + c = EnvConfig() + assert c.llm.langchain_provider == "openai" + assert c.llm.langchain_model_name == "" + assert c.llm.langchain_temperature == 0.0 + assert c.llm.timeout_seconds == 120 + assert c.llm.max_retries == 2 + assert c.llm.langchain_reasoning_effort == "" + assert c.llm.vibe_trading_deepseek_adapter == "auto" + assert c.llm.moonshot_user_agent == "" + assert c.llm.openai_codex_base_url == "https://chatgpt.com/backend-api/codex/responses" + + def test_data_defaults(self) -> None: + c = EnvConfig() + assert c.data.tushare_token == "" + assert c.data.ccxt_exchange == "binance" + assert c.data.ccxt_timeout_ms == 15000 + assert c.data.ccxt_fetch_budget_s == 60.0 + assert c.data.futu_host == "127.0.0.1" + assert c.data.futu_port == 11111 + assert c.data.finnhub_api_key == "" + assert c.data.alphavantage_api_key == "" + assert c.data.tiingo_api_key == "" + assert c.data.fmp_api_key == "" + assert c.data.fred_api_key == "" + assert c.data.vibe_trading_iwencai_key == "" + assert c.data.vibe_trading_sec_ua == "" + assert c.data.vibe_trading_data_cache is False + assert c.data.vibe_trading_data_cache_root == "" + assert c.data.aliyun_iqs_api_key == "" + + def test_api_defaults(self) -> None: + c = EnvConfig() + assert c.api.api_auth_key == "" + assert c.api.vibe_trading_api_key == "" + assert c.api.cors_origins == "" + assert c.api.api_allowed_hosts == "" + assert c.api.enable_session_runtime is True + assert c.api.vibe_trading_trust_docker_loopback is False + assert c.api.vibe_trading_enable_shell_tools is False + assert c.api.vibe_trading_allowed_file_roots == "" + assert c.api.vibe_trading_allowed_write_roots == "" + assert c.api.vibe_trading_allowed_run_roots == "" + + def test_swarm_defaults(self) -> None: + c = EnvConfig() + assert c.swarm.swarm_worker_timeout == 300 + assert c.swarm.swarm_worker_max_iter == 50 + assert c.swarm.swarm_max_workers == 4 + assert c.swarm.swarm_timeout == 1800 + assert c.swarm.swarm_heartbeat_interval_s == 3.0 + assert c.swarm.swarm_stream_retry_delay_s == 1.0 + assert c.swarm.swarm_grounding_max_symbols == 8 + + def test_agent_tuning_defaults(self) -> None: + c = EnvConfig() + assert c.agent_tuning.token_threshold == 40000 + assert c.agent_tuning.vt_heartbeat_interval_s == 3.0 + assert c.agent_tuning.vt_reasoning_delta_min_interval_s == 1.0 + assert c.agent_tuning.vt_stream_retry_delay_s == 1.0 + assert c.agent_tuning.vibe_trading_tool_timeout_seconds == 1800.0 + assert c.agent_tuning.vibe_trading_goal_max_continuations == 3 + assert c.agent_tuning.vibe_trading_sse_timeout == 90 + assert c.agent_tuning.content_filter_warning_threshold == 0.05 + assert c.agent_tuning.vibe_trading_enable_advisory is False + assert c.agent_tuning.vibe_trading_enable_scheduler is False + assert c.agent_tuning.vibe_trading_channels_auto_start is False + assert c.agent_tuning.vibe_trading_disable_bottleneck is False + assert c.agent_tuning.vibe_trading_bench_workers == 0 + assert c.agent_tuning.vibe_trading_search_backends == "" + assert c.agent_tuning.vibe_trading_search_bing_fallback is True + + def test_path_defaults(self) -> None: + c = EnvConfig() + assert c.paths.vibe_trading_hypotheses_path == "" + assert c.paths.vibe_trading_goal_db_path == "" + assert c.paths.vibe_trading_swarm_agent_config == "" + assert c.paths.allow_session_mcp_servers is False + + +# =================================================================== +# TestEnvConfigTypeCoercion +# =================================================================== + + +class TestEnvConfigTypeCoercion: + """Verify env-var strings are coerced to the correct Python types.""" + + def test_int_coercion(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("TIMEOUT_SECONDS", "300") + c = EnvConfig() + assert c.llm.timeout_seconds == 300 + assert isinstance(c.llm.timeout_seconds, int) + + def test_float_coercion(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LANGCHAIN_TEMPERATURE", "0.5") + c = EnvConfig() + assert c.llm.langchain_temperature == 0.5 + assert isinstance(c.llm.langchain_temperature, float) + + def test_invalid_int_falls_back_to_default(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("TIMEOUT_SECONDS", "not_a_number") + c = EnvConfig() + assert c.llm.timeout_seconds == 120 + + def test_invalid_token_threshold_falls_back_to_default(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("TOKEN_THRESHOLD", "abc") + c = EnvConfig() + assert c.agent_tuning.token_threshold == 40000 + + def test_bool_coercion_from_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("VIBE_TRADING_DATA_CACHE", "true") + c = EnvConfig() + assert c.data.vibe_trading_data_cache is True + + def test_bool_false_from_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ENABLE_SESSION_RUNTIME", "false") + c = EnvConfig() + assert c.api.enable_session_runtime is False + + def test_float_swarm_heartbeat(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SWARM_HEARTBEAT_INTERVAL_S", "5.5") + c = EnvConfig() + assert c.swarm.swarm_heartbeat_interval_s == 5.5 + + +# =================================================================== +# TestEnvConfigOverride +# =================================================================== + + +class TestEnvConfigOverride: + """Verify env vars override defaults and reset restores them.""" + + def test_env_override_and_reset( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LANGCHAIN_PROVIDER", "deepseek") + monkeypatch.setenv("TIMEOUT_SECONDS", "60") + monkeypatch.setenv("TUSHARE_TOKEN", "test_token_123") + + c = EnvConfig() + assert c.llm.langchain_provider == "deepseek" + assert c.llm.timeout_seconds == 60 + assert c.data.tushare_token == "test_token_123" + + # Remove env vars and create a new config → defaults restored. + monkeypatch.delenv("LANGCHAIN_PROVIDER") + monkeypatch.delenv("TIMEOUT_SECONDS") + monkeypatch.delenv("TUSHARE_TOKEN") + + c2 = EnvConfig() + assert c2.llm.langchain_provider == "openai" + assert c2.llm.timeout_seconds == 120 + assert c2.data.tushare_token == "" + + def test_multiple_overrides_simultaneously( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("SWARM_MAX_WORKERS", "8") + monkeypatch.setenv("CCXT_EXCHANGE", "okx") + monkeypatch.setenv("CORS_ORIGINS", "http://localhost:3000") + + c = EnvConfig() + assert c.swarm.swarm_max_workers == 8 + assert c.data.ccxt_exchange == "okx" + assert c.api.cors_origins == "http://localhost:3000" + + +# =================================================================== +# TestAPIKeyAlias +# =================================================================== + + +class TestAPIKeyAlias: + """Verify VIBE_TRADING_API_KEY → api_auth_key alias resolution.""" + + def test_vibe_trading_api_key_only(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("VIBE_TRADING_API_KEY", "secret") + c = EnvConfig() + assert c.api.api_auth_key == "secret" + assert c.api.vibe_trading_api_key == "secret" + + def test_both_keys_explicit_wins(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("VIBE_TRADING_API_KEY", "secret") + monkeypatch.setenv("API_AUTH_KEY", "other") + c = EnvConfig() + assert c.api.api_auth_key == "other" + + def test_api_auth_key_only(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("API_AUTH_KEY", "other") + c = EnvConfig() + assert c.api.api_auth_key == "other" + + def test_neither_key_set(self) -> None: + c = EnvConfig() + assert c.api.api_auth_key == "" + assert c.api.vibe_trading_api_key == "" + + +# =================================================================== +# TestSingletonBehavior +# =================================================================== + + +class TestSingletonBehavior: + """Verify get_env_config / reset_env_config singleton semantics.""" + + def test_cached_instance(self) -> None: + c1 = get_env_config() + c2 = get_env_config() + assert c1 is c2 + + def test_reset_creates_new_instance(self) -> None: + c1 = get_env_config() + reset_env_config() + c3 = get_env_config() + assert c3 is not c1 + + def test_reset_picks_up_new_env( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + c1 = get_env_config() + assert c1.llm.timeout_seconds == 120 + + monkeypatch.setenv("TIMEOUT_SECONDS", "999") + reset_env_config() + + c2 = get_env_config() + assert c2.llm.timeout_seconds == 999 + assert c2 is not c1 + + +# =================================================================== +# TestThreadSafety +# =================================================================== + + +class TestThreadSafety: + """Verify concurrent access to get_env_config is safe.""" + + def test_concurrent_access_returns_same_instance(self) -> None: + results: list[EnvConfig] = [] + + def _fetch() -> None: + results.append(get_env_config()) + + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: + futures = [pool.submit(_fetch) for _ in range(10)] + concurrent.futures.wait(futures) + + # All threads got an instance (no exception). + assert len(results) == 10 + # All are the same object (singleton). + assert all(r is results[0] for r in results) + + def test_concurrent_reset_and_read(self) -> None: + """Interleaved reset + read must not raise.""" + errors: list[Exception] = [] + + def _worker(i: int) -> None: + try: + if i % 3 == 0: + reset_env_config() + else: + get_env_config() + except Exception as exc: # noqa: BLE001 + errors.append(exc) + + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: + futures = [pool.submit(_worker, i) for i in range(30)] + concurrent.futures.wait(futures) + + assert errors == [] + + +# =================================================================== +# TestParseBool — accessor._parse_bool +# =================================================================== + + +class TestParseBool: + """Verify the unified boolean parser from accessor.py.""" + + @pytest.mark.parametrize( + "value", + ["1", "true", "True", "TRUE", "yes", "Yes", "YES", "on", "On", "ON"], + ) + def test_truthy(self, value: str) -> None: + assert _parse_bool(value) is True + + @pytest.mark.parametrize( + "value", + [ + "0", "false", "False", "FALSE", + "no", "No", "NO", + "off", "Off", "OFF", + "", "random", "2", "maybe", + ], + ) + def test_falsy(self, value: str) -> None: + assert _parse_bool(value) is False + + def test_none_returns_false(self) -> None: + assert _parse_bool(None) is False + + +# =================================================================== +# TestParseEnvBool — env_schema._parse_env_bool +# =================================================================== + + +class TestParseEnvBool: + """Verify the Pydantic BeforeValidator for EnvBool fields.""" + + @pytest.mark.parametrize( + "value", + ["1", "true", "True", "TRUE", "yes", "Yes", "YES", "on", "On", "ON"], + ) + def test_truthy_strings(self, value: str) -> None: + assert _parse_env_bool(value) is True + + @pytest.mark.parametrize( + "value", + ["0", "false", "False", "FALSE", "no", "No", "NO", "off", "Off", "OFF", ""], + ) + def test_falsy_strings(self, value: str) -> None: + assert _parse_env_bool(value) is False + + def test_non_string_passthrough(self) -> None: + """Non-string values pass through for Pydantic's built-in coercion.""" + assert _parse_env_bool(True) is True + assert _parse_env_bool(False) is False + assert _parse_env_bool(1) == 1 + assert _parse_env_bool(None) is None + + def test_unrecognized_string_passthrough(self) -> None: + """Unrecognized strings pass through (Pydantic will reject them).""" + result = _parse_env_bool("random") + assert result == "random" + + +# =================================================================== +# TestGetEnvOr +# =================================================================== + + +class TestGetEnvOr: + """Verify the env-var fallback helper.""" + + def test_primary_set(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PRIMARY", "primary_val") + monkeypatch.setenv("FALLBACK", "fallback_val") + assert get_env_or("PRIMARY", "FALLBACK", "default") == "primary_val" + + def test_fallback_only(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PRIMARY", raising=False) + monkeypatch.setenv("FALLBACK", "fallback_val") + assert get_env_or("PRIMARY", "FALLBACK", "default") == "fallback_val" + + def test_neither_set_returns_default(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PRIMARY", raising=False) + monkeypatch.delenv("FALLBACK", raising=False) + assert get_env_or("PRIMARY", "FALLBACK", "default") == "default" + + def test_empty_primary_falls_to_fallback(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PRIMARY", "") + monkeypatch.setenv("FALLBACK", "fb") + assert get_env_or("PRIMARY", "FALLBACK", "default") == "fb" + + def test_empty_both_returns_default(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PRIMARY", "") + monkeypatch.setenv("FALLBACK", "") + assert get_env_or("PRIMARY", "FALLBACK", "default") == "default" + + +# =================================================================== +# TestSubModelDirectConstruction +# =================================================================== + + +class TestSubModelDirectConstruction: + """Verify sub-models can be constructed directly with kwargs.""" + + def test_llm_config_direct(self) -> None: + cfg = LLMConfig(langchain_provider="anthropic", timeout_seconds=60) + assert cfg.langchain_provider == "anthropic" + assert cfg.timeout_seconds == 60 + + def test_data_config_direct(self) -> None: + cfg = DataConfig(tushare_token="my_token", ccxt_exchange="okx") + assert cfg.tushare_token == "my_token" + assert cfg.ccxt_exchange == "okx" + + def test_swarm_config_direct(self) -> None: + cfg = SwarmConfig(swarm_max_workers=16) + assert cfg.swarm_max_workers == 16 + + def test_extra_fields_ignored(self) -> None: + cfg = LLMConfig(unknown_field="ignored") + assert not hasattr(cfg, "unknown_field") diff --git a/agent/tests/test_futu_loader.py b/agent/tests/test_futu_loader.py index 98d71668..483e024c 100644 --- a/agent/tests/test_futu_loader.py +++ b/agent/tests/test_futu_loader.py @@ -145,28 +145,44 @@ class TestNormalizeFrame: class TestIsAvailable: def test_false_when_host_missing(self, monkeypatch): - monkeypatch.delenv("FUTU_HOST", raising=False) + from src.config.accessor import reset_env_config + + monkeypatch.setenv("FUTU_HOST", "") monkeypatch.setenv("FUTU_PORT", "11111") + reset_env_config() assert FutuLoader().is_available() is False + reset_env_config() def test_false_when_port_missing(self, monkeypatch): + from src.config.accessor import reset_env_config + monkeypatch.setenv("FUTU_HOST", "127.0.0.1") - monkeypatch.delenv("FUTU_PORT", raising=False) + monkeypatch.setenv("FUTU_PORT", "0") + reset_env_config() assert FutuLoader().is_available() is False + reset_env_config() def test_true_when_connection_succeeds(self, monkeypatch): + from src.config.accessor import reset_env_config + monkeypatch.setenv("FUTU_HOST", "127.0.0.1") monkeypatch.setenv("FUTU_PORT", "11111") + reset_env_config() mock_ctx = MagicMock() _futu_stub.OpenQuoteContext.return_value = mock_ctx assert FutuLoader().is_available() is True mock_ctx.close.assert_called_once() + reset_env_config() def test_false_when_connection_raises(self, monkeypatch): + from src.config.accessor import reset_env_config + monkeypatch.setenv("FUTU_HOST", "127.0.0.1") monkeypatch.setenv("FUTU_PORT", "11111") + reset_env_config() _futu_stub.OpenQuoteContext.side_effect = OSError("connection refused") assert FutuLoader().is_available() is False + reset_env_config() # --------------------------------------------------------------------------- diff --git a/tools/ci_env_var_gate.py b/tools/ci_env_var_gate.py new file mode 100644 index 00000000..5efac187 --- /dev/null +++ b/tools/ci_env_var_gate.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""CI gate: reject raw os.getenv / os.environ reads outside config layer. + +Scans Python files under the agent source tree and rejects any direct +environment-variable **read** that bypasses the centralized +``agent/src/config/`` accessor layer. + +Detected patterns (READ calls): + os.getenv("KEY") + os.environ.get("KEY") + os.environ["KEY"] # subscript with Load context + +Allowlisted (NOT flagged): + os.environ.copy() + os.environ.items() + os.environ.setdefault(...) + os.environ["KEY"] = value # Store context in Assign / AugAssign targets + os.environ.pop() # only in src/api/settings_routes.py + src/config/ + +Warnings (non-blocking): + os.environ.pop() outside src/api/settings_routes.py and src/config/ + +Usage:: + + python tools/ci_env_var_gate.py # scan, exit 0/1 + python tools/ci_env_var_gate.py --allowlist # print allowlist and exit 0 + +Zero external dependencies — stdlib only (ast, os, sys, pathlib). +""" + +from __future__ import annotations + +import ast +import sys +from pathlib import Path + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +# Directories / files to scan (relative to repo root). +SCAN_TARGETS: list[str] = [ + "agent/src/", + "agent/backtest/", + "agent/cli/", + "agent/mcp_server.py", + "agent/api_server.py", + "agent/scripts/", +] + +# Directories always skipped (relative prefixes). +SKIP_PREFIXES: tuple[str, ...] = ( + "agent/tests/", +) + +# The allowed zone — reads here are the *point* of the config layer. +ALLOWED_PREFIX: str = "agent/src/config/" + +# File (relative) where os.environ.pop() is explicitly permitted. +POP_ALLOWLISTED_FILE: str = "agent/src/api/settings_routes.py" + +# --------------------------------------------------------------------------- +# AST visitor +# --------------------------------------------------------------------------- + + +class _EnvReadVisitor(ast.NodeVisitor): + """Walk a single file's AST and collect disallowed env-var reads.""" + + def __init__(self, filepath: Path, repo_root: Path) -> None: + self._filepath = filepath + self._rel = str(filepath.relative_to(repo_root)).replace("\\", "/") + self._violations: list[tuple[int, str]] = [] + self._warnings: list[tuple[int, str]] = [] + + # -- properties --------------------------------------------------------- + + @property + def violations(self) -> list[tuple[int, str]]: + return list(self._violations) + + @property + def warnings(self) -> list[tuple[int, str]]: + return list(self._warnings) + + # -- helpers ------------------------------------------------------------ + + def _is_os_environ(self, node: ast.expr) -> bool: + """Return True when *node* is ``os.environ``.""" + return ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "os" + and node.attr == "environ" + ) + + def _is_os_getenv(self, node: ast.expr) -> bool: + """Return True when *node* is ``os.getenv``.""" + return ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "os" + and node.attr == "getenv" + ) + + # -- visitors ----------------------------------------------------------- + + def visit_Call(self, node: ast.Call) -> None: # noqa: N802 + func = node.func + + # os.getenv(...) + if self._is_os_getenv(func): + self._violations.append( + (node.lineno, "os.getenv() — use src.config accessor") + ) + self.generic_visit(node) + return + + # os.environ.(...) + if ( + isinstance(func, ast.Attribute) + and self._is_os_environ(func.value) + ): + method = func.attr + + # Allowlisted methods + if method in ("copy", "items", "setdefault"): + self.generic_visit(node) + return + + # os.environ.get(...) — READ + if method == "get": + self._violations.append( + (node.lineno, "os.environ.get() — use src.config accessor") + ) + self.generic_visit(node) + return + + # os.environ.pop(...) — warning outside settings_routes / config + if method == "pop": + if self._rel != POP_ALLOWLISTED_FILE: + self._warnings.append( + ( + node.lineno, + "os.environ.pop() outside settings_routes.py / config/", + ) + ) + self.generic_visit(node) + return + + # Any other os.environ.() — flag as read + self._violations.append( + ( + node.lineno, + f"os.environ.{method}() — use src.config accessor", + ) + ) + + self.generic_visit(node) + + def visit_Subscript(self, node: ast.Subscript) -> None: # noqa: N802 + # os.environ["KEY"] in a Load (read) context + if ( + self._is_os_environ(node.value) + and isinstance(node.ctx, ast.Load) + ): + self._violations.append( + (node.lineno, 'os.environ["..."] read — use src.config accessor') + ) + + self.generic_visit(node) + + +# --------------------------------------------------------------------------- +# File discovery +# --------------------------------------------------------------------------- + + +def _collect_python_files(repo_root: Path) -> list[Path]: + """Return all .py files under SCAN_TARGETS, minus SKIP_PREFIXES and ALLOWED_PREFIX.""" + results: list[Path] = [] + + for target in SCAN_TARGETS: + target_path = repo_root / target + if target_path.is_file(): + # Single file target (e.g. agent/mcp_server.py) + if target_path.suffix == ".py": + rel = str(target_path.relative_to(repo_root)).replace("\\", "/") + if not _should_skip(rel): + results.append(target_path) + elif target_path.is_dir(): + for py_file in sorted(target_path.rglob("*.py")): + rel = str(py_file.relative_to(repo_root)).replace("\\", "/") + if _should_skip(rel): + continue + results.append(py_file) + + return results + + +def _should_skip(rel_path: str) -> bool: + """Return True when *rel_path* falls in a skipped or allowed zone.""" + for prefix in SKIP_PREFIXES: + if rel_path.startswith(prefix): + return True + if rel_path.startswith(ALLOWED_PREFIX): + return True + return False + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +ALLOWLIST_TEXT = """\ +Allowlisted patterns (NOT flagged by this gate): + 1. os.environ.copy() — snapshot of the full environment + 2. os.environ.items() — iteration over env vars + 3. os.environ.setdefault(...) — conditional write + 4. os.environ["KEY"] = value — explicit write (Store context) + 5. os.environ.pop() — only in src/api/settings_routes.py + src/config/ + 6. All reads inside agent/src/config/ — the config layer itself + 7. All files under agent/tests/ — test code is exempt +""" + + +def main(argv: list[str] | None = None) -> int: + args = argv if argv is not None else sys.argv[1:] + + if "--allowlist" in args: + print(ALLOWLIST_TEXT) + return 0 + + repo_root = Path(__file__).resolve().parent.parent + files = _collect_python_files(repo_root) + + all_violations: list[tuple[str, int, str]] = [] + all_warnings: list[tuple[str, int, str]] = [] + + for filepath in files: + try: + source = filepath.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + continue + + try: + tree = ast.parse(source, filename=str(filepath)) + except SyntaxError: + # Skip files that can't be parsed (e.g. generated stubs). + continue + + visitor = _EnvReadVisitor(filepath, repo_root) + visitor.visit(tree) + + rel = str(filepath.relative_to(repo_root)).replace("\\", "/") + for lineno, msg in visitor.violations: + all_violations.append((rel, lineno, msg)) + for lineno, msg in visitor.warnings: + all_warnings.append((rel, lineno, msg)) + + # Print warnings (non-blocking) + for rel, lineno, msg in all_warnings: + print(f" WARN: {rel}:{lineno}: {msg}") + + # Print violations (blocking) + if all_violations: + print() + print("FAIL: raw os.getenv / os.environ reads outside config layer:") + for rel, lineno, msg in all_violations: + print(f" {rel}:{lineno}: {msg}") + print() + print("Use the centralized config accessor in agent/src/config/ instead.") + print("Run with --allowlist to see permitted patterns.") + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci_grep_gates.sh b/tools/ci_grep_gates.sh index e9c276ac..dcaff86f 100644 --- a/tools/ci_grep_gates.sh +++ b/tools/ci_grep_gates.sh @@ -17,6 +17,9 @@ # (spec.md §"Vendor 数据 ToS"). # (d) No deprecated `datetime.utcnow(` usage or bare `datetime.now()` calls # in Python sources; `datetime.now(timezone.utc)` is allowed. +# (e) No raw `os.getenv` / `os.environ.get` / `os.environ["KEY"]` reads +# outside the centralized config layer (`agent/src/config/`). +# AST-based; uses `tools/ci_env_var_gate.py`. # # Exclusions: .git, node_modules, __pycache__, .venv, dist, build, this # script itself. The HTML scan in (c) is scoped to wiki/alpha-library/** @@ -125,6 +128,21 @@ else echo "${GREEN}ok${NC}" fi +# -------------------------------------------------------------- gate (e) +echo "[gate e] no raw os.getenv / os.environ reads outside config layer ..." +E_OUTPUT=$(python tools/ci_env_var_gate.py 2>&1) +E_RC=$? +if [ "$E_RC" -ne 0 ]; then + echo "${RED}FAIL${NC}: raw env-var reads outside agent/src/config/:" + echo "$E_OUTPUT" + FAILED=1 +else + if [ -n "$E_OUTPUT" ]; then + echo "$E_OUTPUT" + fi + echo "${GREEN}ok${NC}" +fi + # --------------------------------------------------------------- result if [ "$FAILED" -ne 0 ]; then echo diff --git a/tools/test_ci_env_var_gate.py b/tools/test_ci_env_var_gate.py new file mode 100644 index 00000000..20f92080 --- /dev/null +++ b/tools/test_ci_env_var_gate.py @@ -0,0 +1,233 @@ +"""Tests for tools/ci_env_var_gate.py. + +Each test creates a temporary Python file under a fake repo tree, runs the +AST visitor, and asserts the expected violations / warnings / clean result. + +Run with:: + + pytest tools/test_ci_env_var_gate.py -v +""" + +from __future__ import annotations + +import ast +import textwrap +from pathlib import Path + +import pytest + +# Import the visitor and helpers from the gate module. +from ci_env_var_gate import ( # type: ignore[import-untyped] + _EnvReadVisitor, + _should_skip, + main, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _run_visitor( + source: str, + rel_path: str = "agent/src/some_module.py", + repo_root: Path | None = None, +) -> tuple[list[tuple[int, str]], list[tuple[int, str]]]: + """Parse *source*, run the visitor, return (violations, warnings).""" + if repo_root is None: + repo_root = Path("/fake/repo") + + tree = ast.parse(textwrap.dedent(source)) + filepath = repo_root / rel_path + visitor = _EnvReadVisitor(filepath, repo_root) + visitor.visit(tree) + return visitor.violations, visitor.warnings + + +# --------------------------------------------------------------------------- +# Tests — violations (must be flagged) +# --------------------------------------------------------------------------- + + +class TestOsGetenvFlagged: + """os.getenv("FOO") outside config/ is a violation.""" + + def test_os_getenv_basic(self) -> None: + violations, _ = _run_visitor('import os\nx = os.getenv("FOO")\n') + assert len(violations) == 1 + assert "os.getenv" in violations[0][1] + + def test_os_getenv_with_default(self) -> None: + violations, _ = _run_visitor('import os\nx = os.getenv("FOO", "bar")\n') + assert len(violations) == 1 + + +class TestOsEnvironGetFlagged: + """os.environ.get("FOO") outside config/ is a violation.""" + + def test_environ_get_basic(self) -> None: + violations, _ = _run_visitor('import os\nx = os.environ.get("FOO")\n') + assert len(violations) == 1 + assert "os.environ.get" in violations[0][1] + + def test_environ_get_with_default(self) -> None: + violations, _ = _run_visitor('import os\nx = os.environ.get("FOO", "bar")\n') + assert len(violations) == 1 + + +class TestOsEnvironSubscriptReadFlagged: + """os.environ["FOO"] read outside config/ is a violation.""" + + def test_subscript_read(self) -> None: + violations, _ = _run_visitor('import os\nx = os.environ["FOO"]\n') + assert len(violations) == 1 + assert "read" in violations[0][1].lower() + + def test_subscript_read_in_expression(self) -> None: + violations, _ = _run_visitor( + 'import os\nif os.environ["DEBUG"] == "1": pass\n' + ) + assert len(violations) == 1 + + +# --------------------------------------------------------------------------- +# Tests — allowlisted (must NOT be flagged) +# --------------------------------------------------------------------------- + + +class TestEnvironCopyAllowed: + """os.environ.copy() is NOT flagged.""" + + def test_copy(self) -> None: + violations, _ = _run_visitor("import os\nenv = os.environ.copy()\n") + assert len(violations) == 0 + + +class TestEnvironItemsAllowed: + """os.environ.items() is NOT flagged.""" + + def test_items(self) -> None: + violations, _ = _run_visitor( + "import os\nfor k, v in os.environ.items(): pass\n" + ) + assert len(violations) == 0 + + +class TestEnvironSetdefaultAllowed: + """os.environ.setdefault() is NOT flagged.""" + + def test_setdefault(self) -> None: + violations, _ = _run_visitor( + 'import os\nos.environ.setdefault("FOO", "bar")\n' + ) + assert len(violations) == 0 + + +class TestEnvironSubscriptWriteAllowed: + """os.environ["FOO"] = "bar" (Store context) is NOT flagged.""" + + def test_subscript_write(self) -> None: + violations, _ = _run_visitor('import os\nos.environ["FOO"] = "bar"\n') + assert len(violations) == 0 + + def test_subscript_aug_assign(self) -> None: + # os.environ["PATH"] += ":/new" — still a Store context + violations, _ = _run_visitor('import os\nos.environ["PATH"] += ":/new"\n') + assert len(violations) == 0 + + +class TestConfigDirectoryAllowed: + """Files in agent/src/config/ are NOT scanned (skipped entirely).""" + + def test_config_file_skipped(self) -> None: + # Even with os.getenv in the source, the file is in config/ so + # _should_skip returns True and the scanner never visits it. + assert _should_skip("agent/src/config/accessor.py") is True + assert _should_skip("agent/src/config/env_schema.py") is True + + def test_config_visitor_clean(self) -> None: + # Even if we *did* visit a config file, the visitor itself would + # flag it — but the scanner skips it. Verify the skip logic. + violations, _ = _run_visitor( + 'import os\nx = os.getenv("FOO")\n', + rel_path="agent/src/config/env_schema.py", + ) + # The visitor doesn't know about skip rules — it just visits. + # But the scanner would never call it on this file. + # This test documents that the visitor IS strict; the skip is + # enforced at the file-discovery layer. + assert len(violations) == 1 # visitor flags it + assert _should_skip("agent/src/config/env_schema.py") # scanner skips + + +class TestTestsDirectorySkipped: + """Files in agent/tests/ are NOT scanned.""" + + def test_tests_dir_skipped(self) -> None: + assert _should_skip("agent/tests/test_foo.py") is True + assert _should_skip("agent/tests/unit/test_bar.py") is True + + +# --------------------------------------------------------------------------- +# Tests — pop() warnings +# --------------------------------------------------------------------------- + + +class TestEnvironPopWarning: + """os.environ.pop() outside settings_routes.py is a warning.""" + + def test_pop_outside_settings_routes(self) -> None: + violations, warnings = _run_visitor( + 'import os\nos.environ.pop("FOO", None)\n', + rel_path="agent/src/some_module.py", + ) + assert len(violations) == 0 # not a blocking violation + assert len(warnings) == 1 + assert "pop" in warnings[0][1] + + def test_pop_in_settings_routes_no_warning(self) -> None: + violations, warnings = _run_visitor( + 'import os\nos.environ.pop("FOO", None)\n', + rel_path="agent/src/api/settings_routes.py", + ) + assert len(violations) == 0 + assert len(warnings) == 0 + + +# --------------------------------------------------------------------------- +# Tests — multiple violations in one file +# --------------------------------------------------------------------------- + + +class TestMultipleViolations: + """A file with several patterns reports all of them.""" + + def test_mixed_reads(self) -> None: + source = """\ + import os + a = os.getenv("A") + b = os.environ.get("B") + c = os.environ["C"] + d = os.environ.copy() # allowed + os.environ["D"] = "x" # allowed (write) + """ + violations, _ = _run_visitor(source) + assert len(violations) == 3 # getenv, environ.get, subscript read + + +# --------------------------------------------------------------------------- +# Tests — CLI flags +# --------------------------------------------------------------------------- + + +class TestAllowlistFlag: + """--allowlist prints the allowlist and exits 0.""" + + def test_allowlist_flag(self, capsys: pytest.CaptureFixture[str]) -> None: + rc = main(["--allowlist"]) + assert rc == 0 + captured = capsys.readouterr() + assert "os.environ.copy()" in captured.out + assert "os.environ.items()" in captured.out + assert "os.environ.setdefault" in captured.out