fix(data): normalize baostock volume to board lots + consistency guard
Phase 2 of #1062. BaoStock was the single A-share source reporting volume in single shares while tencent/eastmoney/akshare/mootdx/tushare report board lots (1 lot = 100 shares) — a silent 100x discrepancy whose value depended on which fallback source served the request. - baostock_loader: divide native share volume by 100 at the loader boundary; fractional lots preserved (odd-lot trades are real); volume_units declaration now "lots" like every other A-share source - Loader cache version 3 -> 4 so entries cached under the pre-normalization unit are never served again - New empirical cross-source consistency test: the same settled trading day fetched from every reachable A-share loader must agree within 1% (network-guarded — skips when fewer than two sources are reachable); this is the runtime lock against future unit drift - Pin tests updated to the post-normalization matrix Verified end-to-end: 600519.SH 2026-07-31 — tencent 55,128 lots vs baostock 55,127.52 lots (ratio 0.999991; the 0.48-lot gap is baostock's exact share count including odd lots, within the 1% tolerance). Signed-off-by: shadowinlife <shadowinlife@gmail.com>
This commit is contained in:
1 parent
da7ae2e1ae
commit
b544428e41
5 files changed
+127
-13
No files matched your search
@@ -35,11 +35,10 @@ class DataLoader:
|
||||
|
||||
name = "baostock"
|
||||
markets = {"a_share"}
|
||||
# BaoStock natively reports volume in single shares, unlike the other
|
||||
# A-share sources which report board lots (HKUDS/Vibe-Trading#1062).
|
||||
# Empirically verified 2026-08-11: 600519.SH 2026-07-31 = 5,512,752
|
||||
# shares vs tencent/eastmoney 55,128 lots (ratio exactly 100.0).
|
||||
volume_units = {"a_share": "shares"}
|
||||
# BaoStock natively reports volume in single shares; fetch() normalizes
|
||||
# to board lots — the A-share canonical unit shared by tencent/eastmoney/
|
||||
# akshare/mootdx/tushare (HKUDS/Vibe-Trading#1062).
|
||||
volume_units = {"a_share": "lots"}
|
||||
requires_auth = False
|
||||
|
||||
def is_available(self) -> bool:
|
||||
@@ -160,6 +159,12 @@ class DataLoader:
|
||||
for col in ["open", "high", "low", "close", "volume", "amount"]:
|
||||
df[col] = pd.to_numeric(df[col], errors="coerce")
|
||||
|
||||
# BaoStock reports volume in single shares; normalize to board lots
|
||||
# (1 lot = 100 shares) to match every other A-share source
|
||||
# (HKUDS/Vibe-Trading#1062). Fractional lots are valid — odd-lot
|
||||
# trades exist — so no rounding is applied.
|
||||
df["volume"] = df["volume"] / 100.0
|
||||
|
||||
df = df.rename(columns={"date": "trade_date"})
|
||||
df = df.set_index("trade_date").sort_index()
|
||||
df = df[["open", "high", "low", "close", "volume"]].dropna(
|
||||
|
||||
@@ -245,7 +245,9 @@ LOADER_CACHE_ROOT_ENV = "VIBE_TRADING_DATA_CACHE_ROOT"
|
||||
_LOADER_CACHE_TRUE_VALUES = {"1", "true", "yes", "on"}
|
||||
# Bump when the key payload or on-disk layout changes so stale entries are
|
||||
# simply never matched (old files become unreachable garbage, safe to delete).
|
||||
_LOADER_CACHE_VERSION = 3
|
||||
# v4: baostock volume normalized from shares to lots (#1062) — entries cached
|
||||
# under the pre-normalization unit must never be served again.
|
||||
_LOADER_CACHE_VERSION = 4
|
||||
|
||||
|
||||
def loader_cache_enabled() -> bool:
|
||||
|
||||
@@ -106,3 +106,45 @@ class TestFetchOneCodeHandling:
|
||||
loader._fetch_one(bs_mock, "000001.SZ", "2024-01-01", "2024-01-31")
|
||||
call_args = bs_mock.query_history_k_data_plus.call_args
|
||||
assert call_args[0][0] == "sz.000001"
|
||||
|
||||
|
||||
class TestVolumeUnitNormalization:
|
||||
"""BaoStock native shares are normalized to board lots (#1062)."""
|
||||
|
||||
def test_volume_shares_normalized_to_lots(self):
|
||||
from unittest.mock import MagicMock
|
||||
from backtest.loaders.baostock_loader import DataLoader
|
||||
|
||||
loader = DataLoader()
|
||||
bs_mock = MagicMock()
|
||||
mock_rs = MagicMock()
|
||||
mock_rs.error_code = "0"
|
||||
mock_rs.error_msg = "success"
|
||||
mock_rs.next.side_effect = [True, False]
|
||||
mock_rs.get_row_data.return_value = [
|
||||
"2024-01-02", "10.0", "10.5", "9.8", "10.2", "5512800", "7400000000",
|
||||
]
|
||||
bs_mock.query_history_k_data_plus.return_value = mock_rs
|
||||
|
||||
df = loader._fetch_one(bs_mock, "601398.SH", "2024-01-01", "2024-01-31")
|
||||
|
||||
assert df["volume"].iloc[-1] == 55128.0
|
||||
|
||||
def test_odd_lot_volume_keeps_fractional_lots(self):
|
||||
from unittest.mock import MagicMock
|
||||
from backtest.loaders.baostock_loader import DataLoader
|
||||
|
||||
loader = DataLoader()
|
||||
bs_mock = MagicMock()
|
||||
mock_rs = MagicMock()
|
||||
mock_rs.error_code = "0"
|
||||
mock_rs.error_msg = "success"
|
||||
mock_rs.next.side_effect = [True, False]
|
||||
mock_rs.get_row_data.return_value = [
|
||||
"2024-01-02", "10.0", "10.5", "9.8", "10.2", "5512753", "7400000000",
|
||||
]
|
||||
bs_mock.query_history_k_data_plus.return_value = mock_rs
|
||||
|
||||
df = loader._fetch_one(bs_mock, "601398.SH", "2024-01-01", "2024-01-31")
|
||||
|
||||
assert abs(df["volume"].iloc[-1] - 55127.53) < 1e-9
|
||||
@@ -1,11 +1,11 @@
|
||||
"""Pin the volume-unit declarations of equity loaders (HKUDS/Vibe-Trading#1062).
|
||||
|
||||
A-share sources natively report board lots (1 lot = 100 shares) while
|
||||
baostock reports single shares; the tencent/eastmoney loaders are
|
||||
market-dependent (A-share lots vs HK shares). These pins lock the audit
|
||||
matrix from issue #1062 so a declaration cannot drift silently — the
|
||||
cross-source 100x discrepancy that motivated the fix was exactly such a
|
||||
silent drift.
|
||||
Every A-share source declares board lots (1 lot = 100 shares) — baostock's
|
||||
native single-share volume is normalized at its loader boundary; the
|
||||
tencent/eastmoney loaders are market-dependent (A-share lots vs HK shares).
|
||||
These pins lock the audit matrix from issue #1062 so a declaration cannot
|
||||
drift silently — the cross-source 100x discrepancy that motivated the fix
|
||||
was exactly such a silent drift.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -31,7 +31,7 @@ from backtest.loaders.yfinance_loader import DataLoader as YfinanceLoader
|
||||
(TencentLoader, "hk_equity", "shares"),
|
||||
(EastmoneyLoader, "a_share", "lots"),
|
||||
(EastmoneyLoader, "hk_equity", "shares"),
|
||||
(BaostockLoader, "a_share", "shares"),
|
||||
(BaostockLoader, "a_share", "lots"),
|
||||
(MootdxLoader, "a_share", "lots"),
|
||||
(AkshareLoader, "a_share", "lots"),
|
||||
(TushareLoader, "a_share", "lots"),
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Empirical cross-source volume consistency guard (HKUDS/Vibe-Trading#1062).
|
||||
|
||||
Fetches the same settled A-share trading day from every reachable loader and
|
||||
asserts the reported volumes agree within tolerance. This is the runtime lock
|
||||
the #1062 audit called for: unit drift between fallback sources (the original
|
||||
bug: baostock shares vs tencent/eastmoney lots, exactly 100x) fails loudly
|
||||
here instead of silently corrupting volume analysis.
|
||||
|
||||
Network-guarded by design — sources unreachable from the test environment are
|
||||
skipped individually, and the test skips entirely when fewer than two sources
|
||||
are reachable, so CI without market access stays green.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
CODE = "600519.SH"
|
||||
TRADE_DATE = "2026-07-31"
|
||||
TOLERANCE = 0.01
|
||||
|
||||
|
||||
def _volume_from(loader_cls) -> float | None:
|
||||
try:
|
||||
data = loader_cls().fetch([CODE], TRADE_DATE, TRADE_DATE)
|
||||
except Exception: # noqa: BLE001 — unreachable source = not participating
|
||||
return None
|
||||
df = data.get(CODE) if data else None
|
||||
if df is None or df.empty or "volume" not in df.columns:
|
||||
return None
|
||||
value = df["volume"].iloc[-1]
|
||||
if value is None or value != value: # NaN guard
|
||||
return None
|
||||
return float(value)
|
||||
|
||||
|
||||
def test_a_share_volume_consistent_across_sources():
|
||||
from backtest.loaders.baostock_loader import DataLoader as BaostockLoader
|
||||
from backtest.loaders.eastmoney_loader import DataLoader as EastmoneyLoader
|
||||
from backtest.loaders.mootdx_loader import DataLoader as MootdxLoader
|
||||
from backtest.loaders.tencent_loader import DataLoader as TencentLoader
|
||||
|
||||
candidates = [
|
||||
("tencent", TencentLoader),
|
||||
("eastmoney", EastmoneyLoader),
|
||||
("baostock", BaostockLoader),
|
||||
("mootdx", MootdxLoader),
|
||||
]
|
||||
volumes: dict[str, float] = {}
|
||||
for name, loader_cls in candidates:
|
||||
value = _volume_from(loader_cls)
|
||||
if value is not None:
|
||||
volumes[name] = value
|
||||
|
||||
if len(volumes) < 2:
|
||||
pytest.skip(f"fewer than two reachable A-share sources: {sorted(volumes)}")
|
||||
|
||||
baseline_name = next(iter(volumes))
|
||||
baseline = volumes[baseline_name]
|
||||
for name, value in volumes.items():
|
||||
ratio = value / baseline
|
||||
assert abs(ratio - 1.0) <= TOLERANCE, (
|
||||
f"{name} volume {value:,.0f} disagrees with {baseline_name} "
|
||||
f"{baseline:,.0f} (ratio {ratio:.2f}) — unit drift? see #1062"
|
||||
)
|
||||
Reference in new issue
Block a user