fix(backtest): optimizers see the window in position space, so shorts hedge (#1548)

#1548 signed ``mu`` and left ``cov`` on raw asset returns, which put the
objective's numerator in position space and its variance term in asset space:
``(w @ mu) / sqrt(w @ cov @ w)`` then reads a long/short pair of correlated
assets as correlated when the positions actually hedge. Measured on the new
fixture (asset correlation +0.92): the asset covariance funds the long with the
whole book (1.00 / 0.00), the position covariance splits it 0.514 / -0.486.

``BaseOptimizer.optimize`` now negates a short's column before calling
``_build_context``, so every context is built from position returns: ``mu`` is
the position's expected return, ``cov`` is D Sigma D, and risk-parity /
max-diversification contexts get the position covariance for free. This also
retires the ``self._active_signs`` instance state the PR introduced between the
loop and the hook. Volatility-only contexts are unaffected -- std(-r) == std(r).

agent/backtest/optimizers/base.py

@@ -71,13 +71,17 @@ class BaseOptimizer(ABC):
                 continue
 
             signs = np.array([np.sign(pos.at[dt, c]) for c in active])
-            # Exposed for subclasses (e.g. mean-variance style optimizers)
-            # that need the direction of the position actually being sized,
-            # not just its unsigned covariance/volatility structure -- a
-            # short's expected return is -1 * the asset's raw drift, not the
-            # raw drift itself.
-            self._active_signs = signs
-            ctx = self._build_context(window, active)
+            # Hand subclasses the window in POSITION space: a short's column is
+            # negated, so every context built from it describes what is actually
+            # being sized. ``mu`` becomes the position's expected return (a
+            # short earns the negative of its asset's drift) and ``cov`` becomes
+            # D Sigma D, whose cross terms flip sign for a long/short pair --
+            # signing only ``mu`` would score the numerator in position space
+            # and the variance in asset space, so a hedged pair would still
+            # read as correlated. Volatility-only contexts are unaffected
+            # (std(-r) == std(r)).
+            signed = window.mul(pd.Series(signs, index=window.columns), axis=1)
+            ctx = self._build_context(signed, active)
             if ctx is None:
                 continue
 
@@ -103,7 +107,10 @@ class BaseOptimizer(ABC):
         Return None to skip the date.
 
         Args:
-            window: Return window for active assets.
+            window: Return window for active assets, in POSITION space -- a
+                short's column is already negated by ``optimize``, so a mean
+                taken here is the position's expected return and a covariance
+                is the position covariance.
             active: Active asset codes.
 
         Returns:

agent/backtest/optimizers/mean_variance.py

@@ -18,17 +18,16 @@ class MeanVarianceOptimizer(BaseOptimizer):
     def _build_context(
         self, window: pd.DataFrame, active: List[str]
     ) -> "Dict[str, Any] | None":
-        """Mean vector and covariance.
+        """Mean vector and covariance of the positions being sized.
 
-        ``mu`` is the expected return of the position actually being sized
-        (``sign(pos) * raw asset drift``), not the raw asset drift -- a
-        short's expected return is the negative of the asset's own drift.
-        Without this, a strong short candidate (very negative raw drift)
-        scores as a bad "long" in the Sharpe objective below and is starved
-        of capital relative to a weak short (near-zero drift).
+        ``window`` arrives in position space (see ``BaseOptimizer.optimize``),
+        so ``mu`` is the expected return of the position and ``cov`` its
+        covariance. On raw asset returns a strong short candidate (very
+        negative drift) scored as a bad "long" in the Sharpe objective below
+        and was starved of capital relative to a weak short, and a long/short
+        pair that hedges read as correlated.
         """
-        signs = self._active_signs
-        mu = signs * window.mean().values
+        mu = window.mean().values
         cov = window.cov().values
         if np.isnan(cov).any() or np.isnan(mu).any():
             return None

agent/backtest/optimizers/turnover_aware.py

@@ -97,14 +97,13 @@ class TurnoverAwareOptimizer(BaseOptimizer):
     ) -> "Dict[str, Any] | None":
         """Mean vector, covariance, and active codes for the current window.
 
-        ``mu`` is the expected return of the position actually being sized
-        (``sign(pos) * raw asset drift``), not the raw asset drift -- see
-        ``MeanVarianceOptimizer._build_context`` for the same fix and its
-        rationale; this optimizer's ``objective`` has the identical
-        ``w @ mu`` return term.
+        ``window`` arrives in position space (see ``BaseOptimizer.optimize``),
+        so ``mu`` is the position's expected return and ``cov`` the position
+        covariance -- this optimizer's ``objective`` has the same ``w @ mu``
+        term and the same ``w @ cov @ w`` variance term as the mean-variance
+        one, and needs both in the same space.
         """
-        signs = self._active_signs
-        mu = signs * window.mean().values
+        mu = window.mean().values
         cov = window.cov().values
         if np.isnan(cov).any() or np.isnan(mu).any():
             return None

agent/tests/test_mean_variance_optimizer.py

@@ -53,6 +53,41 @@ class TestMeanVarianceOptimize:
 
         assert abs(last["STRONG"]) > abs(last["WEAK"])
 
+    def test_hedged_pair_is_not_starved_by_asset_space_covariance(self) -> None:
+        """A long and a short of two positively correlated assets hedge each
+        other, so both legs must be funded.
+
+        Regression on the other half of the same bug: signing only ``mu`` left
+        the variance term in ASSET space, where the pair still reads +0.92
+        correlated and diversification looks worthless. Measured on this
+        fixture, the asset-covariance objective put 100% of the book on the
+        long and 0% on the short; the position covariance (D Sigma D, off
+        diagonal -9.4e-05 instead of +9.4e-05) splits it 0.514 / -0.486.
+        """
+        rng = np.random.default_rng(11)
+        n = 200
+        common = rng.normal(0, 0.01, n)
+        dates = pd.bdate_range("2025-01-01", periods=n)
+        ret = pd.DataFrame(
+            {
+                "LONG": 0.002 + common + rng.normal(0, 0.003, n),
+                "SHORT": -0.0015 + common + rng.normal(0, 0.003, n),
+            },
+            index=dates,
+        )
+        assert ret.corr().iloc[0, 1] > 0.9  # the pair really is correlated
+
+        pos = pd.DataFrame(0.0, index=dates, columns=["LONG", "SHORT"])
+        pos.iloc[150:, 0] = 1.0
+        pos.iloc[150:, 1] = -1.0
+
+        result = MeanVarianceOptimizer(lookback=150).optimize(ret, pos, dates)
+        last = result.iloc[-1]
+
+        assert last["LONG"] > 0 and last["SHORT"] < 0
+        assert abs(last["SHORT"]) > 0.3, "the hedging short must be funded"
+        assert abs(last["LONG"]) > 0.3
+
     def test_single_asset_unchanged(self) -> None:
         dates = pd.bdate_range("2025-01-01", periods=100)
         ret = pd.DataFrame(

agent/tests/test_turnover_aware_optimizer.py

@@ -140,6 +140,42 @@ class TestTurnoverAwareOptimize:
         last = result.iloc[-1]
         assert abs(last["STRONG"]) > abs(last["WEAK"])
 
+    def test_hedged_pair_is_not_starved_by_asset_space_covariance(self) -> None:
+        """Same second half as the mean-variance optimizer: this objective's
+        ``lam * (w @ cov @ w)`` risk term needs the POSITION covariance, or a
+        long/short pair of correlated assets reads as correlated and the
+        hedging short is starved.
+
+        Measured on this fixture at ``risk_aversion=100`` (daily variances run
+        ~1e-4, so the default lam=1 makes the risk term negligible and either
+        covariance concentrates): asset covariance gives 0.972 / -0.028, the
+        position covariance 0.521 / -0.479.
+        """
+        rng = np.random.default_rng(11)
+        n = 200
+        common = rng.normal(0, 0.01, n)
+        dates = pd.bdate_range("2025-01-01", periods=n)
+        ret = pd.DataFrame(
+            {
+                "LONG": 0.002 + common + rng.normal(0, 0.003, n),
+                "SHORT": -0.0015 + common + rng.normal(0, 0.003, n),
+            },
+            index=dates,
+        )
+        pos = pd.DataFrame(0.0, index=dates, columns=["LONG", "SHORT"])
+        pos.iloc[150:, 0] = 1.0
+        pos.iloc[150:, 1] = -1.0
+
+        result = optimize(
+            ret, pos, dates, lookback=150, turnover_penalty=0.0,
+            risk_aversion=100.0,
+        )
+        last = result.iloc[-1]
+
+        assert last["LONG"] > 0 and last["SHORT"] < 0
+        assert abs(last["SHORT"]) > 0.3, "the hedging short must be funded"
+        assert abs(last["LONG"]) > 0.3
+
     def test_short_window_and_nan_do_not_raise(self) -> None:
         ret, pos, dates = _sample_data(n_days=80)
         ret.iloc[10:20, 0] = np.nan