Add allow_nonpositive_prices: open on negative-price bars, still reject zero (#816)

Markets like European day-ahead power clear negative routinely, but the
loader's validate_ohlc drops any bar with a non-positive OHLC value and the
engine rejects any open at price <= 0. On real EU power data this silently
deletes ~27% of DE-LU days (every bar with a sub-zero hourly-derived low),
punching holes in rolling statistics (e.g. a z-score entry signal) with no
warning.

Add an opt-in `allow_nonpositive_prices` flag (default False — existing
behavior byte-identical for all positive-price markets):

- loaders/base.py: validate_ohlc gains the flag. Structural invariants
  (high<low, high/low bracketing open/close) always enforced; when the flag
  is set, negative prices pass through and only an exactly-zero price is
  rejected. Zero is genuinely undefined for `size = notional / price` and
  margin; a negative price is well-defined via abs().
- engines/base.py: BaseEngine reads the flag from config. _plan_open_order
  rejects an exactly-zero open always, negatives only when the flag is off.
  _calc_raw_size / _calc_margin divide by abs(price) so a negative entry
  price yields a positive size magnitude (direction carries the sign) and
  positive collateral — unchanged for positive prices. _calc_pnl already
  uses a price difference, well-defined through zero.

Tests open a position on a negative-price bar and assert size/margin/PnL are
well-defined through zero, that an exactly-zero price is still rejected with
the flag on, and that the default still rejects negatives. A regression test
pins *why* zero stays dropped rather than kept: a surviving 0.00 close yields
inf at the `close.pct_change().fillna(0.0)` sites in benchmark.py and
engines/base.py (fillna fills NaN, not inf), so keeping it would corrupt the
return series. Fixture carries the 5 real non-positive-close bars from a 2025
NO2/DE-LU window (ENTSO-E via Energy-Charts / Bundesnetzagentur | SMARD,
CC BY 4.0). Existing engine and OHLC-validation suites pass unchanged
(140 tests total).

Co-authored-by: Ali Emadi <ali@gatekeeperai.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

agent/backtest/engines/base.py

@@ -360,6 +360,14 @@ class BaseEngine(ABC):
         self.config = config
         self.initial_capital: float = config.get("initial_cash", 1_000_000)
         self.default_leverage: float = config.get("leverage", 1.0)
+        # Markets that clear at or below zero (e.g. EU day-ahead power) opt in
+        # to opening on negative-price bars. Default False preserves the legacy
+        # "reject any open_price <= 0" behavior. An exactly-zero open is always
+        # rejected (undefined size = notional / price); negatives are handled by
+        # abs()-based sizing and margin below.
+        self.allow_nonpositive_prices: bool = bool(
+            config.get("allow_nonpositive_prices", False)
+        )
         self.capital: float = self.initial_capital
         self.positions: Dict[str, Position] = {}
         self.trades: List[TradeRecord] = []
@@ -439,14 +447,24 @@ class BaseEngine(ABC):
     def _calc_margin(
         self, symbol: str, size: float, price: float, leverage: float,
     ) -> float:
-        """Margin (collateral) required for a position."""
-        return size * price / leverage
+        """Margin (collateral) required for a position.
+
+        ``abs(price)`` so collateral stays positive when the entry price is
+        negative; for the usual positive price this is unchanged.
+        """
+        return size * abs(price) / leverage
 
     def _calc_raw_size(
         self, symbol: str, target_notional: float, price: float,
     ) -> float:
-        """Convert target notional exposure to number of units/contracts."""
-        return target_notional / price
+        """Convert target notional exposure to number of units/contracts.
+
+        Size is a positive magnitude — direction carries the sign elsewhere —
+        so divide by ``abs(price)``; a negative entry price must not flip the
+        size negative (which the ``size <= 0`` guard would then reject). For a
+        positive price this is unchanged.
+        """
+        return target_notional / abs(price)
 
     def _leverage_for_symbol(self, symbol: str) -> float:
         """Return leverage used to size and margin one symbol."""
@@ -949,7 +967,10 @@ class BaseEngine(ABC):
         if not self.can_execute(symbol, direction, bar):
             return None
         open_price = float(bar.get("open", bar.get("close", 0)))
-        if open_price <= 0:
+        # Zero is always rejected (size = notional / price is undefined);
+        # negatives are rejected unless this engine opted into non-positive
+        # prices, in which case abs()-based sizing/margin below handle them.
+        if open_price == 0 or (open_price < 0 and not self.allow_nonpositive_prices):
             return None
         price = self.apply_slippage(open_price, direction)
         leverage = self._leverage_for_symbol(symbol)

agent/backtest/loaders/base.py

@@ -47,7 +47,12 @@ def validate_date_range(start_date: str, end_date: str) -> None:
         raise ValueError(f"start_date ({start_date}) > end_date ({end_date})")
 
 
-def validate_ohlc(frame: pd.DataFrame, *, strategy: str = "drop") -> pd.DataFrame:
+def validate_ohlc(
+    frame: pd.DataFrame,
+    *,
+    strategy: str = "drop",
+    allow_nonpositive_prices: bool = False,
+) -> pd.DataFrame:
     """Drop, flag, or reject bars that violate OHLC invariants.
 
     Loaders only drop NaN rows, so structurally dirty bars — ``high < low``,
@@ -57,11 +62,24 @@ def validate_ohlc(frame: pd.DataFrame, *, strategy: str = "drop") -> pd.DataFram
     canonical loader-boundary check; call it after the existing ``dropna`` so a
     single sanity pass guards every source.
 
+    Structural invariants (``high < low`` and high/low failing to bracket
+    open/close) are always enforced. The *positivity* invariant is
+    configurable: some markets clear at or below zero legitimately (European
+    day-ahead power routinely prints negative), and a rolling statistic over a
+    silently gap-filled series is worse than a well-defined negative bar. When
+    ``allow_nonpositive_prices`` is set, negative prices pass through and only
+    an exactly-zero price is rejected — zero is genuinely undefined for
+    notional sizing (``size = notional / price``) and margin, whereas a
+    negative price is handled by ``abs()``-based sizing in the engine.
+
     Args:
         frame: OHLCV frame with at least ``open``/``high``/``low``/``close``
             columns. NaN handling is left to the caller's ``dropna``.
         strategy: ``"drop"`` (remove offending rows, default), ``"warn"``
             (log and keep), or ``"raise"`` (raise on any violation).
+        allow_nonpositive_prices: when ``True``, keep bars with negative
+            prices and reject only exact zeros; when ``False`` (default,
+            unchanged behavior) reject any price ``<= 0``.
 
     Returns:
         The frame with invalid rows removed (``"drop"``) or unchanged
@@ -76,17 +94,18 @@ def validate_ohlc(frame: pd.DataFrame, *, strategy: str = "drop") -> pd.DataFram
         return frame
 
     open_, high, low, close = (frame[c] for c in required)
-    invalid = (
+    structural = (
         (high < low)
         | (high < open_)
         | (high < close)
         | (low > open_)
         | (low > close)
-        | (open_ <= 0)
-        | (high <= 0)
-        | (low <= 0)
-        | (close <= 0)
     )
+    if allow_nonpositive_prices:
+        nonpositive = (open_ == 0) | (high == 0) | (low == 0) | (close == 0)
+    else:
+        nonpositive = (open_ <= 0) | (high <= 0) | (low <= 0) | (close <= 0)
+    invalid = structural | nonpositive
     n_invalid = int(invalid.sum())
     if n_invalid == 0:
         return frame

agent/tests/fixtures/negative_close_bars.csv

@@ -0,0 +1,14 @@
+# Real EU day-ahead power bars that clear at or below zero.
+# Source: ENTSO-E day-ahead prices via Energy-Charts (Fraunhofer ISE),
+#   Bundesnetzagentur | SMARD.de. Licensed CC BY 4.0.
+# Window 2025-07-20..2025-12-31. These are the daily bars whose *close* is
+# non-positive: four strictly negative (kept when allow_nonpositive_prices)
+# and one exactly zero (POWER-DA-DELU 2025-10-26, always rejected). Two also
+# open negative (POWER-DA-DELU 2025-10-04). Hourly-derived daily low goes far
+# below zero on solar-heavy DELU days.
+code,trade_date,open,high,low,close,volume
+POWER-DA-NO2,2025-10-04,11.22,12.64,-0.09,-0.09,0.0
+POWER-DA-DELU,2025-09-15,1.39,54.31,-14.27,-0.09,0.0
+POWER-DA-DELU,2025-10-03,82.76,128.62,-0.08,-0.02,0.0
+POWER-DA-DELU,2025-10-04,-0.01,4.02,-5.02,-1.03,0.0
+POWER-DA-DELU,2025-10-26,3.99,49.34,-1.05,0.00,0.0

agent/tests/test_nonpositive_prices.py

@@ -0,0 +1,192 @@
+"""allow_nonpositive_prices: open on negative-price bars, still reject zero.
+
+Markets like European day-ahead power clear negative routinely. The default
+(flag off) still drops/rejects any non-positive price, so nothing changes for
+existing markets; when the flag is on, negative prices flow through and only an
+exactly-zero price is rejected (size = notional / price and margin are
+undefined at zero, but well-defined for negatives via abs()).
+
+Fixture `fixtures/negative_close_bars.csv` holds the five real bars from a
+2025 NO2/DE-LU window whose close is non-positive (four negative, one exactly
+zero). Source: ENTSO-E via Energy-Charts (Fraunhofer ISE),
+Bundesnetzagentur | SMARD.de, CC BY 4.0.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from backtest.engines.base import BaseEngine
+from backtest.loaders.base import validate_ohlc
+
+FIXTURE = Path(__file__).parent / "fixtures" / "negative_close_bars.csv"
+
+
+def _negative_close_frame() -> pd.DataFrame:
+    df = pd.read_csv(FIXTURE, comment="#")
+    df.index = pd.to_datetime(df.pop("trade_date"))
+    df.index.name = "trade_date"
+    return df
+
+
+# ---------------------------------------------------------------------------
+# loader: validate_ohlc positivity gate
+# ---------------------------------------------------------------------------
+
+
+def test_default_drops_every_nonpositive_bar() -> None:
+    """Unchanged behavior: with the flag off, all five non-positive-close bars
+    (and their sub-zero lows) are dropped."""
+    frame = _negative_close_frame()
+    assert len(frame) == 5
+    cleaned = validate_ohlc(frame)
+    assert cleaned.empty
+
+
+def test_allow_keeps_negatives_rejects_exact_zero() -> None:
+    """With the flag on, the four negative-close bars survive and only the
+    exactly-zero close (DELU 2025-10-26) is dropped — structural invariants
+    (high brackets low/open/close) still hold for all five real bars."""
+    frame = _negative_close_frame()
+    cleaned = validate_ohlc(frame, allow_nonpositive_prices=True)
+    assert len(cleaned) == 4
+    kept_closes = sorted(round(c, 2) for c in cleaned["close"])
+    assert kept_closes == [-1.03, -0.09, -0.09, -0.02]
+    assert 0.0 not in list(cleaned["close"])
+
+
+def test_why_loader_drops_exact_zero_inf_downstream() -> None:
+    """Pins *why* an exact-zero close is dropped at the loader, not kept.
+
+    Both the benchmark (``benchmark.py``) and the position simulation
+    (``engines/base.py``) compute per-bar returns as
+    ``close.pct_change().fillna(0.0)`` over a *raw* close series. If a ``0.00``
+    clear survived into that series, the bar *after* it divides by zero and
+    yields ``inf`` — and ``fillna(0.0)`` does not neutralize it (it fills NaN,
+    not inf), so the benchmark's ``(1 + r).prod()`` collapses to ``nan``.
+
+    Negatives divide cleanly (engine sizing uses ``abs(price)``), so they are
+    kept; exact zero is genuinely undefined and is therefore rejected at the
+    loader *and* the engine. This test encodes that contract so the asymmetry
+    is not silently "fixed" by later relaxing the loader. See #571.
+    """
+    idx = pd.to_datetime(["2025-10-24", "2025-10-25", "2025-10-26", "2025-10-27"])
+
+    # If the loader had KEPT the 0.00 close, the raw return series blows up.
+    kept = pd.Series([42.0, -1.03, 0.00, 42.0], index=idx)
+    ret_if_kept = kept.pct_change().fillna(0.0)  # the exact downstream expression
+    assert np.isinf(ret_if_kept.iloc[-1])                       # 0.00 -> inf next bar
+    with np.errstate(invalid="ignore"):  # the nan-from-inf is the point, not a bug
+        bench_total = float((1 + ret_if_kept).prod())
+    assert not np.isfinite(bench_total)                         # benchmark total -> nan
+
+    # The loader drops the exact-zero bar, so the surviving series is inf-free.
+    frame = pd.DataFrame(
+        {
+            "open": kept,
+            "high": kept.abs() + 1.0,
+            "low": kept - 1.0,
+            "close": kept,
+        },
+        index=idx,
+    )
+    cleaned = validate_ohlc(frame, allow_nonpositive_prices=True)
+    assert 0.0 not in list(cleaned["close"])                    # zero bar removed
+    safe_ret = cleaned["close"].pct_change().fillna(0.0)
+    assert np.isfinite(safe_ret.to_numpy()).all()               # negatives stay finite
+
+
+def test_allow_still_enforces_structural_invariants() -> None:
+    """The flag relaxes only positivity, never the OHLC bracket invariants."""
+    frame = pd.DataFrame(
+        [(-5.0, -1.0, -8.0, -12.0, 0.0)],  # close -12 < low -8 -> invalid bracket
+        columns=["open", "high", "low", "close", "volume"],
+        index=pd.to_datetime(["2025-10-04"]),
+    )
+    assert validate_ohlc(frame, allow_nonpositive_prices=True).empty
+
+
+# ---------------------------------------------------------------------------
+# engine: opening / sizing / margin / pnl through zero
+# ---------------------------------------------------------------------------
+
+
+class _PlainEngine(BaseEngine):
+    """Minimal concrete engine: identity slippage/rounding, zero commission,
+    all trades allowed — isolates BaseEngine's price handling."""
+
+    def can_execute(self, symbol: str, direction: int, bar: pd.Series) -> bool:
+        return True
+
+    def round_size(self, raw_size: float, price: float) -> float:
+        return raw_size
+
+    def calc_commission(self, size: float, price: float, direction: int, is_open: bool) -> float:
+        return 0.0
+
+    def apply_slippage(self, price: float, direction: int) -> float:
+        return price
+
+
+def _engine(*, allow: bool) -> _PlainEngine:
+    return _PlainEngine({"initial_cash": 1_000_000, "allow_nonpositive_prices": allow})
+
+
+def _bar_df(open_px: float, ts: str = "2025-10-04") -> tuple[pd.DataFrame, pd.Timestamp]:
+    idx = pd.to_datetime([ts])
+    df = pd.DataFrame(
+        {"open": [open_px], "high": [max(open_px, 1.0)], "low": [open_px], "close": [open_px]},
+        index=idx,
+    )
+    return df, idx[0]
+
+
+def test_opens_on_negative_price_bar_when_allowed() -> None:
+    eng = _engine(allow=True)
+    df, ts = _bar_df(-5.0)  # DELU 2025-10-04 opened at -0.01; use -5 for headroom
+    order = eng._plan_open_order("POWER-DA-DELU", 0.5, df, ts, equity=1_000_000)
+    assert order is not None
+    assert order.direction == 1
+    # Size is a positive magnitude despite the negative price (abs-based).
+    assert order.size == pytest.approx(0.5 * 1_000_000 / 5.0)
+    # Margin (collateral) is positive, not negated by the negative price.
+    assert order.margin == pytest.approx(order.size * 5.0)
+    assert order.cost > 0
+
+
+def test_default_still_rejects_negative_open() -> None:
+    eng = _engine(allow=False)
+    df, ts = _bar_df(-5.0)
+    assert eng._plan_open_order("POWER-DA-DELU", 0.5, df, ts, equity=1_000_000) is None
+
+
+def test_zero_price_rejected_even_when_allowed() -> None:
+    """DELU 2025-10-26 cleared at exactly 0 — undefined sizing, always rejected."""
+    eng = _engine(allow=True)
+    df, ts = _bar_df(0.0)
+    assert eng._plan_open_order("POWER-DA-DELU", 0.5, df, ts, equity=1_000_000) is None
+
+
+def test_margin_and_pnl_well_defined_through_zero() -> None:
+    """Collateral positive at a negative entry; PnL correct crossing zero."""
+    eng = _engine(allow=True)
+    size = 1_000.0
+    margin = eng._calc_margin("POWER-DA-DELU", size, price=-5.0, leverage=1.0)
+    assert margin == pytest.approx(5_000.0)  # abs(price), positive collateral
+
+    # Long from -5 to +3: gains the full 8 EUR/MWh move.
+    long_pnl = eng._calc_pnl("POWER-DA-DELU", 1, size, entry_price=-5.0, exit_price=3.0)
+    assert long_pnl == pytest.approx(size * 8.0)
+    # Short from -5 to -8 (price falls further negative): profits.
+    short_pnl = eng._calc_pnl("POWER-DA-DELU", -1, size, entry_price=-5.0, exit_price=-8.0)
+    assert short_pnl == pytest.approx(size * 3.0)
+
+
+def test_raw_size_positive_for_negative_price() -> None:
+    eng = _engine(allow=True)
+    size = eng._calc_raw_size("POWER-DA-DELU", target_notional=500_000.0, price=-5.0)
+    assert size == pytest.approx(100_000.0)  # not -100_000