fix(loaders): adjust A-share prices and use point-in-time index membership

Tushare's daily and fund_daily endpoints return raw prices, so a close-to-close
return taken across an ex-date spans the mechanical gap of a split or bonus
issue rather than the stock's actual move. Checked against Tushare's own
pct_chg over 2020-2024: 300750.SZ on 2023-04-26 reads -41.82% raw against a
true +5.40%, 300124.SZ -34.26% against -1.01%, 601012.SH -24.21% against
+6.46%. Every sampled name was hit and the error is always negative, so this
was a systematic contaminant of returns rather than noise.

The akshare and tencent loaders already request qfq; only the Tushare path did
not, and it feeds both the factor bench and A-share backtests. Adds
backtest/loaders/cn_adjust.py as the single implementation and routes both
through it. After the fix, adjusted returns match pct_chg to within 1e-5 and
the amount-derived VWAP keeps its relationship to close to 2e-16. A symbol with
no usable factors is dropped with a warning rather than served raw; indices
need no adjustment and Tushare publishes no HK factor series.

The CSI300 bench also took its roster from the newest index_weight snapshot and
applied it across the whole window, so a name was present only because it
survived to the end and every IC was measured on a hindsight-selected set. The
panel now carries every name that was ever a member and masks each date's
cross-section to the roster in force then.
This commit is contained in:
Haozhe Wu committed 2026-08-04 14:52:13 +08:00
1 parent fde1aa58f4
commit d52199ff6d
6 files changed
+410 -16

No files matched your search

+77
View File
@@ -0,0 +1,77 @@
"""Corporate-action adjustment for Tushare A-share and fund bars.
Tushare's ``daily`` and ``fund_daily`` endpoints return *unadjusted* prices, so
a close-to-close return taken across an ex-date spans the mechanical price drop
of a split, bonus issue or dividend rather than the instrument's actual move.
Measured against Tushare's own ``pct_chg`` over 2020-2024: 300750.SZ on
2023-04-26 reads -41.82% raw against a true +5.40%, 300124.SZ -34.26% against
-1.01%, 601012.SH -24.21% against +6.46%. The error is always negative, because
prices only ever gap down on an ex-date, so it is a systematic contaminant of
returns rather than noise.
The matching factor series comes from ``pro.adj_factor`` for equities and
``pro.fund_adj`` for funds; both carry the same ``(trade_date, adj_factor)``
schema. Factors are cumulative and rise across a corporate action — verified on
600519.SH 2022-06-30, where the factor steps 7.4740 -> 7.5546 as the stock goes
ex-dividend.
This module is the single implementation; the backtest loader and the alpha
bench both route through it rather than re-deriving the convention.
"""
from __future__ import annotations
import pandas as pd
def apply_qfq(df: pd.DataFrame, factor: pd.DataFrame | None) -> pd.DataFrame | None:
"""Forward-adjust raw Tushare daily bars for corporate actions.
``pro.daily`` returns *unadjusted* prices, so a close-to-close return taken
across an ex-date spans the mechanical price drop of a split, bonus issue or
dividend rather than the stock's actual move. Measured against Tushare's own
``pct_chg`` on CSI300 members: 300750.SZ on 2023-04-26 reads -41.82% raw
against a true +5.40%, 300124.SZ on 2021-06-04 reads -34.26% against -1.01%,
601012.SH on 2022-06-06 reads -24.21% against +6.46%. The error is always
negative, so it is a systematic contaminant of every cross-sectional IC the
bench reports, not noise.
Prices are scaled to the last bar in the window (前复权), matching the
``adjust="qfq"`` convention the akshare and tencent loaders already use.
``volume`` is divided by the same ratio so share counts stay on one basis
and the ``amount``-derived VWAP keeps the same relationship to ``close``;
``amount`` is a cash figure and is left untouched.
Args:
df: Raw daily bars indexed by trade date.
factor: ``pro.adj_factor`` rows for the same symbol and window.
Returns:
The adjusted frame, or ``None`` when the adjustment factors are missing
or unusable — the caller drops the symbol rather than benching it on
contaminated prices.
"""
if factor is None or getattr(factor, "empty", True):
return None
if "adj_factor" not in factor.columns or "trade_date" not in factor.columns:
return None
series = factor.copy()
series["trade_date"] = pd.to_datetime(series["trade_date"], errors="coerce")
series = (
series.dropna(subset=["trade_date"])
.set_index("trade_date")["adj_factor"]
.astype(float)
.sort_index()
)
series = series.reindex(df.index).ffill().bfill()
if series.isna().any() or (series <= 0).any():
return None
ratio = series / series.iloc[-1]
out = df.copy()
for col in ("open", "high", "low", "close"):
if col in out.columns:
out[col] = out[col] * ratio
if "volume" in out.columns:
out["volume"] = out["volume"] / ratio
return out
+25
View File
@@ -11,6 +11,7 @@ import pandas as pd
from backtest.loaders._symbol_utils import _is_etf_listed
from backtest.loaders.base import cached_loader_fetch, validate_date_range
from backtest.loaders.cn_adjust import apply_qfq
from backtest.loaders.registry import register
logger = logging.getLogger(__name__)
@@ -149,15 +150,21 @@ class DataLoader:
if _is_etf_listed(code):
endpoint_name = "fund_daily"
df = self.api.fund_daily(ts_code=code, start_date=start_date, end_date=end_date)
adjust = getattr(self.api, "fund_adj", None)
elif _is_index(code):
endpoint_name = "index_daily"
df = self.api.index_daily(ts_code=code, start_date=start_date, end_date=end_date)
# An index level is already continuous across its members' ex-dates.
adjust = None
elif _is_hk_equity(code):
endpoint_name = "hk_daily"
df = self.api.hk_daily(ts_code=code, start_date=start_date, end_date=end_date)
# Tushare publishes no HK adjustment-factor series.
adjust = None
else:
endpoint_name = "daily"
df = self.api.daily(ts_code=code, start_date=start_date, end_date=end_date)
adjust = getattr(self.api, "adj_factor", None)
if df is None or df.empty:
logger.warning("tushare returned empty for %s via %s", code, endpoint_name)
@@ -172,8 +179,26 @@ class DataLoader:
ohlcv = df[["open", "high", "low", "close", "volume"]].dropna(
subset=["open", "high", "low", "close"]
)
if adjust is None:
return ohlcv
try:
factor = adjust(ts_code=code, start_date=start_date, end_date=end_date)
except Exception as exc: # noqa: BLE001 - one bad fetch must not raise
logger.warning("tushare adjustment-factor fetch failed for %s: %s", code, exc)
factor = None
adjusted = apply_qfq(ohlcv, factor)
if adjusted is None:
# Returning the raw frame here is what produced the defect: a
# close-to-close return across an ex-date spans the mechanical gap,
# measured at -47.2%% on 300750.SZ 2023-04-26 against a true +5.4%%.
logger.warning(
"tushare: no usable adjustment factors for %s — dropping the "
"symbol rather than backtesting it on unadjusted prices",
code,
)
return adjusted
def _merge_basic_fields(
self,
result: Dict[str, pd.DataFrame],
+77 -12
View File
@@ -41,6 +41,7 @@ from typing import Any
import pandas as pd
from backtest.loaders.cn_adjust import apply_qfq as _apply_qfq
from src.agent.tools import BaseTool
from src.config.accessor import get_env_config
@@ -325,19 +326,38 @@ def _load_csi300_panel(start: str, end: str) -> dict[str, pd.DataFrame]:
codes: list[str] = []
constituent_source = "tushare index_weight"
constituent_source_date: str | None = None
membership: pd.DataFrame | None = None
try:
# Reach back before ``start`` so the snapshot that was in force on the
# first requested day is included; Tushare publishes month-end rosters.
lookback = (pd.Timestamp(start) - pd.Timedelta(days=60)).strftime("%Y%m%d")
weights = pro.index_weight(
index_code="399300.SZ", start_date=sd, end_date=ed
index_code="399300.SZ", start_date=lookback, end_date=ed
)
if weights is not None and not weights.empty:
latest_date = weights["trade_date"].max()
constituent_source_date = str(latest_date)
codes = (
weights[weights["trade_date"] == latest_date]["con_code"]
.drop_duplicates()
.tolist()
frame = weights.copy()
frame["trade_date"] = pd.to_datetime(frame["trade_date"], errors="coerce")
frame = frame.dropna(subset=["trade_date", "con_code"])
constituent_source_date = str(weights["trade_date"].max())
# Every name that was a member at any point in the window, so the
# panel can carry a name that later left the index.
codes = sorted(frame["con_code"].astype(str).unique())
membership = (
frame.assign(_member=True)
.pivot_table(
index="trade_date",
columns="con_code",
values="_member",
aggfunc="first",
)
.notna()
.sort_index()
)
logger.info(
"csi300: %d names ever a member across %d roster snapshots",
len(codes),
len(membership),
)
logger.info("csi300: %d constituents from index_weight @ %s", len(codes), latest_date)
except Exception as exc: # noqa: BLE001
logger.warning("csi300 index_weight failed (%s); using fallback list", exc)
@@ -362,7 +382,9 @@ def _load_csi300_panel(start: str, end: str) -> dict[str, pd.DataFrame]:
if col in df.columns:
df[col] = pd.to_numeric(df[col], errors="coerce")
keep = [c for c in ("open", "high", "low", "close", "volume", "amount") if c in df.columns]
return code, df[keep].dropna(subset=["open", "high", "low", "close"])
df = df[keep].dropna(subset=["open", "high", "low", "close"])
factor = _retry(lambda: pro.adj_factor(ts_code=code, start_date=sd, end_date=ed))
return code, _apply_qfq(df, factor)
fetched: dict[str, pd.DataFrame] = {}
with ThreadPoolExecutor(max_workers=_CSI300_FETCH_WORKERS) as pool:
@@ -376,6 +398,26 @@ def _load_csi300_panel(start: str, end: str) -> dict[str, pd.DataFrame]:
if frame is not None and not frame.empty:
fetched[code] = frame
# A name with no usable adjustment factors is dropped rather than benched on
# raw prices, so the drop has to be visible or it becomes its own silent bias.
dropped = sorted(set(codes) - set(fetched))
if not fetched:
raise RuntimeError(
"csi300: no symbol survived corporate-action adjustment — "
"pro.adj_factor returned nothing usable for any of the "
f"{len(codes)} names, which usually means the Tushare token lacks "
"adj_factor permission. Benching on unadjusted prices is not an "
"alternative: an ex-date injects a fabricated cross-sectional "
"return, measured at -47.2% on 300750.SZ 2023-04-26."
)
if dropped:
logger.warning(
"csi300: dropped %d/%d name(s) with no usable adjustment factors: %s",
len(dropped),
len(codes),
", ".join(dropped[:10]) + ("..." if len(dropped) > 10 else ""),
)
panel = _wide_from_fetched(fetched, include_amount=True)
# CN equity vwap: Tushare ``amount`` is in 千元, ``volume`` in 手. True VWAP
# = (amount * 1000 CNY) / (volume * 100 shares). Matches
@@ -386,15 +428,38 @@ def _load_csi300_panel(start: str, end: str) -> dict[str, pd.DataFrame]:
panel["vwap"] = safe_div(
panel["amount"] * 1000.0, panel["volume"] * 100.0 + 1.0
)
# Restrict each date's cross-section to the names that were index members on
# that date. Without this the panel carries today's roster back through the
# whole window, so a name is only present because it survived to the end —
# every IC is then measured on a set selected with hindsight.
if membership is not None:
mask = (
membership.reindex(columns=panel["close"].columns)
.reindex(index=panel["close"].index.union(membership.index))
.ffill()
.reindex(panel["close"].index)
.bfill()
.fillna(False)
.astype(bool)
)
for key, frame in panel.items():
if isinstance(frame, pd.DataFrame):
panel[key] = frame.where(mask)
panel["_meta"] = {
"universe": "csi300",
# A terminal snapshot is forward-looking relative to the start of the
# requested interval; the static fallback is survivor-selected too.
"survivorship_bias": True,
# True only on the degraded path: the hand-picked fallback is a
# survivor-selected static roster with no point-in-time membership.
"survivorship_bias": membership is None,
"pit_membership": membership is not None,
"degraded": constituent_source != "tushare index_weight",
"constituent_source": constituent_source,
"constituent_source_date": constituent_source_date,
"constituent_count": len(codes),
# Prices are corporate-action adjusted; raw pro.daily is not.
"price_adjustment": "qfq",
"dropped_unadjustable": len(dropped),
}
return panel
+134
View File
@@ -0,0 +1,134 @@
"""Regression: CSI300 bench prices must be corporate-action adjusted.
``pro.daily`` returns raw prices, so a close-to-close return taken across an
ex-date spans the mechanical drop of a split or bonus issue. Measured against
Tushare's own ``pct_chg`` over 2020-2024: 300750.SZ on 2023-04-26 reads -41.82%
raw against a true +5.40%, 300124.SZ -34.26% against -1.01%, 601012.SH -24.21%
against +6.46%. The error is always negative, so every cross-sectional IC the
bench reports carried a systematic contaminant, not noise.
These tests are offline; the live validation (adjusted return vs ``pct_chg``
within 1e-5, VWAP/close ratio preserved to 2e-16) is recorded in the audit doc.
"""
from __future__ import annotations
import pandas as pd
import pytest
from src.tools.alpha_bench_tool import _apply_qfq
_DATES = pd.to_datetime(["2023-04-24", "2023-04-25", "2023-04-26", "2023-04-27"])
def _bars(closes):
return pd.DataFrame(
{
"open": closes,
"high": [c * 1.01 for c in closes],
"low": [c * 0.99 for c in closes],
"close": closes,
"volume": [1000.0, 1000.0, 2000.0, 2000.0],
"amount": [100.0, 100.0, 100.0, 100.0],
},
index=_DATES,
)
def _factors(values):
return pd.DataFrame(
{
"trade_date": ["20230424", "20230425", "20230426", "20230427"],
"adj_factor": values,
}
)
class TestAdjustmentRemovesTheExDateJump:
"""A 2-for-1 split halves the raw price without changing the return."""
# Price halves on the third bar. Tushare's adj_factor is cumulative and
# rises across a corporate action (verified on 600519.SH 2022-06-30:
# 7.4740 -> 7.5546), so a 2-for-1 split doubles it.
_SPLIT_CLOSES = [200.0, 200.0, 100.0, 100.0]
_SPLIT_FACTORS = [1.0, 1.0, 2.0, 2.0]
def test_raw_prices_show_a_fabricated_fifty_percent_drop(self):
raw = _bars(self._SPLIT_CLOSES)["close"].pct_change()
assert raw.iloc[2] == pytest.approx(-0.5)
def test_adjusted_prices_show_no_return_on_the_ex_date(self):
adjusted = _apply_qfq(_bars(self._SPLIT_CLOSES), _factors(self._SPLIT_FACTORS))
assert adjusted["close"].pct_change().iloc[2] == pytest.approx(0.0, abs=1e-12)
def test_the_latest_bar_keeps_its_traded_price(self):
# 前复权: the most recent bar is the anchor, so it is untouched.
adjusted = _apply_qfq(_bars(self._SPLIT_CLOSES), _factors(self._SPLIT_FACTORS))
assert adjusted["close"].iloc[-1] == pytest.approx(100.0)
def test_ohlc_are_scaled_by_the_same_ratio(self):
bars = _bars(self._SPLIT_CLOSES)
adjusted = _apply_qfq(bars, _factors(self._SPLIT_FACTORS))
for column in ("open", "high", "low"):
assert adjusted[column].iloc[0] == pytest.approx(bars[column].iloc[0] * 0.5)
def test_volume_is_put_on_the_same_basis_as_price(self):
# Pre-split bars are restated into post-split shares, so the count doubles.
bars = _bars(self._SPLIT_CLOSES)
adjusted = _apply_qfq(bars, _factors(self._SPLIT_FACTORS))
assert adjusted["volume"].iloc[0] == pytest.approx(bars["volume"].iloc[0] * 2.0)
def test_amount_is_cash_and_is_left_alone(self):
bars = _bars(self._SPLIT_CLOSES)
adjusted = _apply_qfq(bars, _factors(self._SPLIT_FACTORS))
assert adjusted["amount"].tolist() == bars["amount"].tolist()
def test_vwap_keeps_its_relationship_to_close(self):
# The bench derives vwap from amount/volume, so an adjustment that moved
# close without moving volume would silently decouple the two.
bars = _bars(self._SPLIT_CLOSES)
adjusted = _apply_qfq(bars, _factors(self._SPLIT_FACTORS))
raw_ratio = (bars["amount"] * 1000 / (bars["volume"] * 100)) / bars["close"]
adj_ratio = (
adjusted["amount"] * 1000 / (adjusted["volume"] * 100)
) / adjusted["close"]
pd.testing.assert_series_equal(raw_ratio, adj_ratio)
class TestUnusableFactorsAreRefused:
"""Missing factors must drop the symbol, never fall back to raw prices."""
_CLOSES = [100.0, 100.0, 100.0, 100.0]
def test_none_is_refused(self):
assert _apply_qfq(_bars(self._CLOSES), None) is None
def test_an_empty_frame_is_refused(self):
assert _apply_qfq(_bars(self._CLOSES), pd.DataFrame()) is None
def test_a_frame_without_the_factor_column_is_refused(self):
bad = pd.DataFrame({"trade_date": ["20230424"], "something_else": [1.0]})
assert _apply_qfq(_bars(self._CLOSES), bad) is None
def test_a_non_positive_factor_is_refused(self):
assert _apply_qfq(_bars(self._CLOSES), _factors([1.0, 0.0, 1.0, 1.0])) is None
def test_a_short_factor_series_is_forward_filled_not_refused(self):
# Tushare returns one row per trading day; a trailing gap is normal and
# must not cost the symbol its whole history.
partial = pd.DataFrame(
{"trade_date": ["20230424", "20230425"], "adj_factor": [2.0, 2.0]}
)
adjusted = _apply_qfq(_bars(self._CLOSES), partial)
assert adjusted is not None
assert adjusted["close"].notna().all()
@@ -42,6 +42,10 @@ class _FakeTusharePro:
"""Return deterministic OHLCV data for every requested code."""
return _daily_frame()
def adj_factor(self, **_kwargs) -> pd.DataFrame:
"""Return a flat adjustment factor — no corporate action in the window."""
return pd.DataFrame({"trade_date": ["20240102"], "adj_factor": [1.0]})
def _install_fake_tushare(
monkeypatch: pytest.MonkeyPatch,
@@ -60,10 +64,10 @@ def _install_fake_tushare(
)
def test_csi300_reports_terminal_index_weight_snapshot(
def test_csi300_reports_point_in_time_membership(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Healthy CSI300 loads disclose their terminal Tushare snapshot."""
"""A healthy CSI300 load carries every name that was ever a member."""
weights = pd.DataFrame(
{
"trade_date": ["20240102", "20240131", "20240131"],
@@ -76,14 +80,42 @@ def test_csi300_reports_terminal_index_weight_snapshot(
assert panel["_meta"] == {
"universe": "csi300",
"survivorship_bias": True,
# Membership is per date, so a name is no longer present merely because
# it survived to the end of the window.
"survivorship_bias": False,
"pit_membership": True,
"degraded": False,
"constituent_source": "tushare index_weight",
"constituent_source_date": "20240131",
"constituent_count": 2,
# The union of both snapshots, not the terminal roster.
"constituent_count": 3,
"price_adjustment": "qfq",
"dropped_unadjustable": 0,
}
def test_csi300_masks_a_name_outside_its_membership_window(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A name that left the index must be absent from later cross-sections."""
weights = pd.DataFrame(
{
"trade_date": ["20240102", "20240102", "20240131"],
# 600000.SH is a member on the first snapshot only.
"con_code": ["600000.SH", "000001.SZ", "000001.SZ"],
}
)
_install_fake_tushare(monkeypatch, _FakeTusharePro(weights))
panel = tool._load_csi300_panel("2024-01-01", "2024-01-31")
close = panel["close"]
assert set(close.columns) == {"600000.SH", "000001.SZ"}
# The fake loader returns a single bar dated 2024-01-02, when both were in.
assert close.loc["2024-01-02", "600000.SH"] == 10.5
assert close.loc["2024-01-02", "000001.SZ"] == 10.5
def test_csi300_reports_hand_picked_fallback(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -99,11 +131,16 @@ def test_csi300_reports_hand_picked_fallback(
assert panel["_meta"] == {
"universe": "csi300",
# The fallback is a static survivor-selected roster with no membership
# history, so the bias disclosure stays on.
"survivorship_bias": True,
"pit_membership": False,
"degraded": True,
"constituent_source": "hand-picked fallback",
"constituent_source_date": None,
"constituent_count": 2,
"price_adjustment": "qfq",
"dropped_unadjustable": 0,
}
+56
View File
@@ -135,12 +135,29 @@ def _make_ohlcv_df() -> pd.DataFrame:
})
def _make_adj_df(factors=(1.0, 1.0, 2.0)) -> pd.DataFrame:
"""Build the adjustment-factor frame tushare pairs with the bars above.
The default doubles on the last bar, i.e. a 2-for-1 split, so a test can
tell an adjusted frame from a raw one.
"""
return pd.DataFrame({
"ts_code": ["X"] * 3,
"trade_date": ["20250102", "20250103", "20250106"],
"adj_factor": list(factors),
})
class TestFetchDailyFrameRouting:
"""Verify _fetch_daily_frame calls the correct tushare endpoint per symbol type."""
def _make_loader(self) -> DataLoader:
loader = object.__new__(DataLoader)
loader.api = MagicMock()
# Equities and funds are corporate-action adjusted before they are
# returned, so the factor endpoints must answer with a real frame.
loader.api.adj_factor.return_value = _make_adj_df((1.0, 1.0, 1.0))
loader.api.fund_adj.return_value = _make_adj_df((1.0, 1.0, 1.0))
return loader
def test_stock_routes_to_daily(self) -> None:
@@ -191,6 +208,45 @@ class TestFetchDailyFrameRouting:
assert result is None
loader.api.daily.assert_not_called()
def test_stock_prices_are_corporate_action_adjusted(self) -> None:
loader = self._make_loader()
loader.api.daily.return_value = _make_ohlcv_df()
loader.api.adj_factor.return_value = _make_adj_df((1.0, 1.0, 2.0))
result = loader._fetch_daily_frame("000001.SZ", "20250102", "20250110")
loader.api.adj_factor.assert_called_once()
# Forward-adjusted to the last bar: earlier closes are halved, the last
# keeps its traded price.
assert result["close"].tolist() == [5.25, 5.5, 11.5]
def test_etf_prices_are_corporate_action_adjusted(self) -> None:
loader = self._make_loader()
loader.api.fund_daily.return_value = _make_ohlcv_df()
loader.api.fund_adj.return_value = _make_adj_df((1.0, 1.0, 2.0))
result = loader._fetch_daily_frame("510050.SH", "20250102", "20250110")
loader.api.fund_adj.assert_called_once()
assert result["close"].tolist() == [5.25, 5.5, 11.5]
def test_a_symbol_with_no_factors_is_dropped_not_returned_raw(self) -> None:
# Falling back to raw prices is the defect this guard exists to stop.
loader = self._make_loader()
loader.api.daily.return_value = _make_ohlcv_df()
loader.api.adj_factor.return_value = pd.DataFrame()
assert loader._fetch_daily_frame("000001.SZ", "20250102", "20250110") is None
def test_an_index_is_not_adjusted(self) -> None:
loader = self._make_loader()
loader.api.index_daily.return_value = _make_ohlcv_df()
result = loader._fetch_daily_frame("000001.SH", "20250102", "20250110")
loader.api.adj_factor.assert_not_called()
assert result["close"].tolist() == [10.5, 11.0, 11.5]
def test_empty_result_warns(self) -> None:
loader = self._make_loader()
loader.api.daily.return_value = pd.DataFrame()