fix(data): normalize baostock volume to board lots + consistency guard

Phase 2 of #1062. BaoStock was the single A-share source reporting
volume in single shares while tencent/eastmoney/akshare/mootdx/tushare
report board lots (1 lot = 100 shares) — a silent 100x discrepancy
whose value depended on which fallback source served the request.

- baostock_loader: divide native share volume by 100 at the loader
  boundary; fractional lots preserved (odd-lot trades are real);
  volume_units declaration now "lots" like every other A-share source
- Loader cache version 3 -> 4 so entries cached under the
  pre-normalization unit are never served again
- New empirical cross-source consistency test: the same settled
  trading day fetched from every reachable A-share loader must agree
  within 1% (network-guarded — skips when fewer than two sources are
  reachable); this is the runtime lock against future unit drift
- Pin tests updated to the post-normalization matrix

Verified end-to-end: 600519.SH 2026-07-31 — tencent 55,128 lots vs
baostock 55,127.52 lots (ratio 0.999991; the 0.48-lot gap is baostock's
exact share count including odd lots, within the 1% tolerance).

Signed-off-by: shadowinlife <shadowinlife@gmail.com>
This commit is contained in:
shadowinlife committed 2026-08-11 15:39:52 +08:00
1 parent da7ae2e1ae
commit b544428e41
5 files changed
+127 -13

No files matched your search

+10 -5
View File
@@ -35,11 +35,10 @@ class DataLoader:
name = "baostock"
markets = {"a_share"}
# BaoStock natively reports volume in single shares, unlike the other
# A-share sources which report board lots (HKUDS/Vibe-Trading#1062).
# Empirically verified 2026-08-11: 600519.SH 2026-07-31 = 5,512,752
# shares vs tencent/eastmoney 55,128 lots (ratio exactly 100.0).
volume_units = {"a_share": "shares"}
# BaoStock natively reports volume in single shares; fetch() normalizes
# to board lots — the A-share canonical unit shared by tencent/eastmoney/
# akshare/mootdx/tushare (HKUDS/Vibe-Trading#1062).
volume_units = {"a_share": "lots"}
requires_auth = False
def is_available(self) -> bool:
@@ -160,6 +159,12 @@ class DataLoader:
for col in ["open", "high", "low", "close", "volume", "amount"]:
df[col] = pd.to_numeric(df[col], errors="coerce")
# BaoStock reports volume in single shares; normalize to board lots
# (1 lot = 100 shares) to match every other A-share source
# (HKUDS/Vibe-Trading#1062). Fractional lots are valid — odd-lot
# trades exist — so no rounding is applied.
df["volume"] = df["volume"] / 100.0
df = df.rename(columns={"date": "trade_date"})
df = df.set_index("trade_date").sort_index()
df = df[["open", "high", "low", "close", "volume"]].dropna(
+3 -1
View File
@@ -245,7 +245,9 @@ LOADER_CACHE_ROOT_ENV = "VIBE_TRADING_DATA_CACHE_ROOT"
_LOADER_CACHE_TRUE_VALUES = {"1", "true", "yes", "on"}
# Bump when the key payload or on-disk layout changes so stale entries are
# simply never matched (old files become unreachable garbage, safe to delete).
_LOADER_CACHE_VERSION = 3
# v4: baostock volume normalized from shares to lots (#1062) — entries cached
# under the pre-normalization unit must never be served again.
_LOADER_CACHE_VERSION = 4
def loader_cache_enabled() -> bool:
+42
View File
@@ -106,3 +106,45 @@ class TestFetchOneCodeHandling:
loader._fetch_one(bs_mock, "000001.SZ", "2024-01-01", "2024-01-31")
call_args = bs_mock.query_history_k_data_plus.call_args
assert call_args[0][0] == "sz.000001"
class TestVolumeUnitNormalization:
"""BaoStock native shares are normalized to board lots (#1062)."""
def test_volume_shares_normalized_to_lots(self):
from unittest.mock import MagicMock
from backtest.loaders.baostock_loader import DataLoader
loader = DataLoader()
bs_mock = MagicMock()
mock_rs = MagicMock()
mock_rs.error_code = "0"
mock_rs.error_msg = "success"
mock_rs.next.side_effect = [True, False]
mock_rs.get_row_data.return_value = [
"2024-01-02", "10.0", "10.5", "9.8", "10.2", "5512800", "7400000000",
]
bs_mock.query_history_k_data_plus.return_value = mock_rs
df = loader._fetch_one(bs_mock, "601398.SH", "2024-01-01", "2024-01-31")
assert df["volume"].iloc[-1] == 55128.0
def test_odd_lot_volume_keeps_fractional_lots(self):
from unittest.mock import MagicMock
from backtest.loaders.baostock_loader import DataLoader
loader = DataLoader()
bs_mock = MagicMock()
mock_rs = MagicMock()
mock_rs.error_code = "0"
mock_rs.error_msg = "success"
mock_rs.next.side_effect = [True, False]
mock_rs.get_row_data.return_value = [
"2024-01-02", "10.0", "10.5", "9.8", "10.2", "5512753", "7400000000",
]
bs_mock.query_history_k_data_plus.return_value = mock_rs
df = loader._fetch_one(bs_mock, "601398.SH", "2024-01-01", "2024-01-31")
assert abs(df["volume"].iloc[-1] - 55127.53) < 1e-9
+7 -7
View File
@@ -1,11 +1,11 @@
"""Pin the volume-unit declarations of equity loaders (HKUDS/Vibe-Trading#1062).
A-share sources natively report board lots (1 lot = 100 shares) while
baostock reports single shares; the tencent/eastmoney loaders are
market-dependent (A-share lots vs HK shares). These pins lock the audit
matrix from issue #1062 so a declaration cannot drift silently — the
cross-source 100x discrepancy that motivated the fix was exactly such a
silent drift.
Every A-share source declares board lots (1 lot = 100 shares) — baostock's
native single-share volume is normalized at its loader boundary; the
tencent/eastmoney loaders are market-dependent (A-share lots vs HK shares).
These pins lock the audit matrix from issue #1062 so a declaration cannot
drift silently — the cross-source 100x discrepancy that motivated the fix
was exactly such a silent drift.
"""
from __future__ import annotations
@@ -31,7 +31,7 @@ from backtest.loaders.yfinance_loader import DataLoader as YfinanceLoader
(TencentLoader, "hk_equity", "shares"),
(EastmoneyLoader, "a_share", "lots"),
(EastmoneyLoader, "hk_equity", "shares"),
(BaostockLoader, "a_share", "shares"),
(BaostockLoader, "a_share", "lots"),
(MootdxLoader, "a_share", "lots"),
(AkshareLoader, "a_share", "lots"),
(TushareLoader, "a_share", "lots"),
@@ -0,0 +1,65 @@
"""Empirical cross-source volume consistency guard (HKUDS/Vibe-Trading#1062).
Fetches the same settled A-share trading day from every reachable loader and
asserts the reported volumes agree within tolerance. This is the runtime lock
the #1062 audit called for: unit drift between fallback sources (the original
bug: baostock shares vs tencent/eastmoney lots, exactly 100x) fails loudly
here instead of silently corrupting volume analysis.
Network-guarded by design — sources unreachable from the test environment are
skipped individually, and the test skips entirely when fewer than two sources
are reachable, so CI without market access stays green.
"""
from __future__ import annotations
import pytest
CODE = "600519.SH"
TRADE_DATE = "2026-07-31"
TOLERANCE = 0.01
def _volume_from(loader_cls) -> float | None:
try:
data = loader_cls().fetch([CODE], TRADE_DATE, TRADE_DATE)
except Exception: # noqa: BLE001 — unreachable source = not participating
return None
df = data.get(CODE) if data else None
if df is None or df.empty or "volume" not in df.columns:
return None
value = df["volume"].iloc[-1]
if value is None or value != value: # NaN guard
return None
return float(value)
def test_a_share_volume_consistent_across_sources():
from backtest.loaders.baostock_loader import DataLoader as BaostockLoader
from backtest.loaders.eastmoney_loader import DataLoader as EastmoneyLoader
from backtest.loaders.mootdx_loader import DataLoader as MootdxLoader
from backtest.loaders.tencent_loader import DataLoader as TencentLoader
candidates = [
("tencent", TencentLoader),
("eastmoney", EastmoneyLoader),
("baostock", BaostockLoader),
("mootdx", MootdxLoader),
]
volumes: dict[str, float] = {}
for name, loader_cls in candidates:
value = _volume_from(loader_cls)
if value is not None:
volumes[name] = value
if len(volumes) < 2:
pytest.skip(f"fewer than two reachable A-share sources: {sorted(volumes)}")
baseline_name = next(iter(volumes))
baseline = volumes[baseline_name]
for name, value in volumes.items():
ratio = value / baseline
assert abs(ratio - 1.0) <= TOLERANCE, (
f"{name} volume {value:,.0f} disagrees with {baseline_name} "
f"{baseline:,.0f} (ratio {ratio:.2f}) — unit drift? see #1062"
)