Merge pull request #730 from yxhuang/fix/us-hk-stock-news

fix(tools): return real Yahoo news articles for US/HK stock news

agent/backtest/loaders/yahoo_client.py

@@ -387,3 +387,28 @@ def search(query: str) -> List[Dict[str, Any]]:
     )
     quotes = (payload or {}).get("quotes") or []
     return [quote for quote in quotes if isinstance(quote, dict)]
+
+
+def search_news(query: str, count: int) -> List[Dict[str, Any]]:
+    """Look up matching news via the v1 search endpoint.
+
+    Args:
+        query: Free-text query (ticker fragment or company name).
+        count: Maximum number of news items requested from Yahoo.
+
+    Returns:
+        The ``news`` list (each a dict with ``title``, ``publisher``, ``link``,
+        ``providerPublishTime``, ``relatedTickers``, ...), or an empty list when
+        none match.
+
+    Raises:
+        requests.RequestException: On a network/HTTP failure.
+    """
+    payload = throttled_get_json(
+        _SEARCH_BASE,
+        host_key=HOST_KEY,
+        min_interval=_min_interval(),
+        params={"q": query, "newsCount": count},
+    )
+    news = (payload or {}).get("news") or []
+    return [item for item in news if isinstance(item, dict)]

agent/src/tools/stock_news_tool.py

@@ -7,11 +7,8 @@ Two public, no-auth news surfaces are wrapped behind one BaseTool contract:
   surface it rate-limits by source IP, so the request routes through the frozen,
   IP-throttled :mod:`backtest.loaders.eastmoney_client` rather than touching the
   host directly.
-* US / HK have no free no-auth article feed here: the frozen
-  :func:`backtest.loaders.yahoo_client.search` helper exposes only the search
-  endpoint's instrument ``quotes`` (not its ``news`` array), so the tool returns
-  honest related-instrument *matches* under a ``matches`` key — never instrument
-  hits relabelled as articles.
+* US / HK headlines come from Yahoo Finance's public v1 search-news surface via
+  the frozen, IP-throttled :mod:`backtest.loaders.yahoo_client`.
 
 The tool never re-implements provider plumbing and never issues an un-throttled
 request: every outbound call goes through a frozen client.
@@ -29,6 +26,7 @@ from __future__ import annotations
 
 import json
 import logging
+from datetime import datetime, timezone
 from typing import Any
 
 from backtest.loaders import eastmoney_client, yahoo_client
@@ -172,7 +170,14 @@ def _fetch_eastmoney_news(query: str, limit: int) -> list[dict[str, Any]]:
             "type": ["cmsArticleWebOld"],
             "client": "web",
             "clientType": "web",
-            "param": {"cmsArticleWebOld": {"searchScope": "default", "sort": "default", "pageIndex": 1, "pageSize": limit}},
+            "param": {
+                "cmsArticleWebOld": {
+                    "searchScope": "default",
+                    "sort": "default",
+                    "pageIndex": 1,
+                    "pageSize": limit,
+                }
+            },
         },
         ensure_ascii=False,
     )
@@ -192,47 +197,49 @@ def _fetch_eastmoney_news(query: str, limit: int) -> list[dict[str, Any]]:
     return [_em_article(a) for a in articles if isinstance(a, dict)][:limit]
 
 
-def _yahoo_match(raw: dict[str, Any]) -> dict[str, Any]:
-    """Project one Yahoo search match into a compact headline-style record.
-
-    Yahoo's ``search`` helper returns instrument matches; each is surfaced as a
-    related-instrument headline so callers get the symbol, name and exchange.
+def _yahoo_article(raw: dict[str, Any]) -> dict[str, Any]:
+    """Project one Yahoo search-news item into the shared article shape.
 
     Args:
-        raw: A single quote dict from :func:`yahoo_client.search`.
+        raw: A single news dict from :func:`yahoo_client.search_news`.
 
     Returns:
-        A flat ``{title, symbol, exchange, quote_type}`` record.
+        A flat ``{title, url, source, published, snippet}`` record.
     """
-    name = raw.get("shortname") or raw.get("longname") or raw.get("symbol")
+    published = None
+    try:
+        timestamp = float(raw.get("providerPublishTime"))
+        published = datetime.fromtimestamp(timestamp, tz=timezone.utc).strftime(
+            "%Y-%m-%d %H:%M:%S"
+        )
+    except (TypeError, ValueError, OSError, OverflowError):
+        pass
     return {
-        "title": _snippet(name),
-        "symbol": raw.get("symbol"),
-        "exchange": raw.get("exchange"),
-        "quote_type": raw.get("quoteType"),
+        "title": _snippet(raw.get("title")),
+        "url": raw.get("link"),
+        "source": raw.get("publisher"),
+        "published": published,
+        "snippet": _snippet(raw.get("summary")),
     }
 
 
-def _fetch_yahoo_matches(query: str, limit: int) -> list[dict[str, Any]]:
-    """Fetch US/HK related-instrument matches for a query via Yahoo search.
-
-    Yahoo's public search endpoint also carries a ``news`` array, but the frozen
-    :func:`yahoo_client.search` helper exposes only the instrument ``quotes``.
-    Rather than mislabel those quotes as articles, this returns them honestly as
-    related-instrument matches.
+def _fetch_yahoo_news(query: str, limit: int) -> list[dict[str, Any]]:
+    """Fetch US/HK news articles for a query via Yahoo search.
 
     Args:
         query: Free-text search term (bare ticker or keyword).
         limit: Maximum number of records to return.
 
     Returns:
-        A capped list of compact instrument-match records; empty when none.
+        A capped list of compact article records; empty when none.
 
     Raises:
         requests.RequestException: Network/HTTP failure, propagated to caller.
     """
-    matches = yahoo_client.search(query)
-    return [_yahoo_match(m) for m in matches if isinstance(m, dict)][:limit]
+    articles = yahoo_client.search_news(query, limit)
+    return [
+        _yahoo_article(article) for article in articles if isinstance(article, dict)
+    ][:limit]
 
 
 class StockNewsTool(BaseTool):
@@ -243,9 +250,7 @@ class StockNewsTool(BaseTool):
         "Fetch recent financial news headlines, read-only and no auth. Markets: "
         "China A-share (SH/SZ/BJ) returns Eastmoney news ARTICLES "
         "(title/url/source/published/snippet) under 'articles'. US (.US) and Hong "
-        "Kong (.HK) do NOT return news articles: Yahoo's public search surface "
-        "yields related-instrument MATCHES (symbol/name/exchange/quote_type), "
-        "returned under 'matches' (result_type='matches'), not 'articles'. Use "
+        "Kong (.HK) return Yahoo Finance news articles with the same fields. Use "
         "scope 'stock' with a 'code', or scope 'global' (no code) for broad "
         "China-market finance articles. "
         'Example: {"code": "600519.SH", "scope": "stock", "limit": 10}.'
@@ -296,7 +301,9 @@ class StockNewsTool(BaseTool):
         """
         scope = kwargs.get("scope", "stock")
         if scope not in ("stock", "global"):
-            return self._error(f"invalid scope: {scope!r}; expected 'stock' or 'global'")
+            return self._error(
+                f"invalid scope: {scope!r}; expected 'stock' or 'global'"
+            )
 
         limit = _clamp_limit(kwargs.get("limit"))
 
@@ -318,7 +325,9 @@ class StockNewsTool(BaseTool):
         except Exception as exc:  # noqa: BLE001 - surface any fetch failure as envelope
             logger.warning("global news fetch failed: %s", exc)
             return self._error(f"eastmoney news fetch failed: {exc}")
-        return self._ok("global", "eastmoney", {"scope": "global", "articles": articles})
+        return self._ok(
+            "global", "eastmoney", {"scope": "global", "articles": articles}
+        )
 
     def _run_stock(self, code_arg: Any, limit: int) -> str:
         """Fetch single-security headlines, routing by exchange suffix.
@@ -331,7 +340,9 @@ class StockNewsTool(BaseTool):
             A success or error JSON envelope.
         """
         if not isinstance(code_arg, str) or not code_arg.strip():
-            return self._error("missing required parameter: code (required when scope='stock')")
+            return self._error(
+                "missing required parameter: code (required when scope='stock')"
+            )
 
         code = code_arg.strip()
         suffix = _suffix_of(code)
@@ -356,26 +367,23 @@ class StockNewsTool(BaseTool):
             logger.warning("eastmoney news fetch failed for %s: %s", code, exc)
             return self._error(f"eastmoney news fetch failed: {exc}")
         return self._ok(
-            "a_share", "eastmoney", {"scope": "stock", "code": code, "articles": articles}
+            "a_share",
+            "eastmoney",
+            {"scope": "stock", "code": code, "articles": articles},
         )
 
     def _stock_via_yahoo(self, code: str, query: str, limit: int) -> str:
-        """Fetch US/HK related-instrument matches from Yahoo for one code.
-
-        Yahoo's public ``search`` surface yields instrument matches, not news
-        articles, so the payload is labelled ``matches`` (never ``articles``) to
-        avoid passing instrument hits off as headlines.
-        """
+        """Fetch US/HK news articles from Yahoo for one code."""
         market = "hk" if _suffix_of(code) == "HK" else "us"
         try:
-            matches = _fetch_yahoo_matches(query, limit)
+            articles = _fetch_yahoo_news(query, limit)
         except Exception as exc:  # noqa: BLE001 - surface any fetch failure as envelope
-            logger.warning("yahoo search fetch failed for %s: %s", code, exc)
-            return self._error(f"yahoo search fetch failed: {exc}")
+            logger.warning("yahoo news fetch failed for %s: %s", code, exc)
+            return self._error(f"yahoo news fetch failed: {exc}")
         return self._ok(
             market,
             "yahoo",
-            {"scope": "stock", "code": code, "result_type": "matches", "matches": matches},
+            {"scope": "stock", "code": code, "articles": articles},
         )
 
     @staticmethod

agent/tests/test_stock_news_tool.py

@@ -2,7 +2,7 @@
 
 No request leaves the process: the Eastmoney HTTP boundary
 (:func:`backtest.loaders.eastmoney_client.throttled_get_json`) and the Yahoo
-:func:`backtest.loaders.yahoo_client.search` helper are mocked so the real
+:func:`backtest.loaders.yahoo_client.search_news` helper are mocked so the real
 client + tool parsing run fully offline.
 """
 
@@ -48,20 +48,23 @@ def _em_news_payload() -> dict[str, Any]:
     }
 
 
-def _yahoo_matches() -> list[dict[str, Any]]:
-    """A Yahoo search result list with two instrument matches."""
+def _yahoo_news() -> list[dict[str, Any]]:
+    """A Yahoo search result list with two news articles."""
     return [
         {
-            "symbol": "AAPL",
-            "shortname": "Apple Inc.",
-            "exchange": "NMS",
-            "quoteType": "EQUITY",
+            "title": "Apple unveils new products",
+            "publisher": "Reuters",
+            "link": "https://example.com/apple-products",
+            "providerPublishTime": 1704067200,
+            "summary": "Apple announced a new product lineup. " * 30,
+            "relatedTickers": ["AAPL"],
         },
         {
-            "symbol": "AAPL.MX",
-            "shortname": "Apple Inc.",
-            "exchange": "MEX",
-            "quoteType": "EQUITY",
+            "title": "Apple shares rise",
+            "publisher": "Bloomberg",
+            "link": "https://example.com/apple-shares",
+            "providerPublishTime": 1704153600,
+            "relatedTickers": ["AAPL"],
         },
     ]
 
@@ -90,6 +93,19 @@ class TestHelpers:
         assert len(out) <= 281
         assert out.endswith("…")
 
+    def test_search_news_filters_to_dict_items(self, monkeypatch) -> None:
+        def fake_get_json(url: str, **kwargs: Any) -> dict[str, Any]:
+            assert url == yahoo_client._SEARCH_BASE
+            assert kwargs["host_key"] == yahoo_client.HOST_KEY
+            assert kwargs["params"] == {"q": "apple", "newsCount": 2}
+            return {"news": [_yahoo_news()[0], "garbage", _yahoo_news()[1]]}
+
+        monkeypatch.setattr(yahoo_client, "throttled_get_json", fake_get_json)
+
+        articles = yahoo_client.search_news("apple", 2)
+
+        assert articles == _yahoo_news()
+
 
 class TestToolContract:
     def test_name_and_schema(self) -> None:
@@ -98,10 +114,10 @@ class TestToolContract:
         assert tool.is_readonly is True
         assert tool.parameters["required"] == []
         assert tool.parameters["properties"]["scope"]["enum"] == ["stock", "global"]
-        # Description must honestly state US/HK returns matches, not articles (B6).
+        # Description must advertise the shared article contract for every market.
         desc = tool.description.lower()
-        assert "matches" in desc
-        assert "not return news articles" in desc or "not 'articles'" in desc
+        assert "yahoo finance news articles" in desc
+        assert "matches" not in desc
 
 
 class TestExecuteSuccess:
@@ -139,36 +155,54 @@ class TestExecuteSuccess:
         assert out["data"]["scope"] == "global"
         assert len(out["data"]["articles"]) == 2
 
-    def test_us_stock_via_yahoo_returns_matches_not_articles(self) -> None:
+    def test_us_stock_via_yahoo_returns_articles(self) -> None:
         tool = StockNewsTool()
-        with patch.object(yahoo_client, "search", return_value=_yahoo_matches()) as srch:
+        with patch.object(
+            yahoo_client, "search_news", return_value=_yahoo_news()
+        ) as srch:
             out = json.loads(tool.execute(code="AAPL.US", limit=1))
 
-        srch.assert_called_once_with("AAPL")
+        srch.assert_called_once_with("AAPL", 1)
         assert out["ok"] is True
         assert out["market"] == "us"
         assert out["source"] == "yahoo"
-        # Instrument hits must NOT be mislabelled as articles (B6 regression).
-        assert "articles" not in out["data"]
-        assert out["data"]["result_type"] == "matches"
-        # limit=1 caps the two matches to one.
-        assert len(out["data"]["matches"]) == 1
-        first = out["data"]["matches"][0]
-        assert first["symbol"] == "AAPL"
-        assert first["exchange"] == "NMS"
-        assert first["quote_type"] == "EQUITY"
+        assert len(out["data"]["articles"]) == 1
+        first = out["data"]["articles"][0]
+        assert first == {
+            "title": "Apple unveils new products",
+            "url": "https://example.com/apple-products",
+            "source": "Reuters",
+            "published": "2024-01-01 00:00:00",
+            "snippet": ("Apple announced a new product lineup. " * 30)[:280].rstrip()
+            + "…",
+        }
 
-    def test_hk_stock_routes_to_yahoo(self) -> None:
+    def test_hk_stock_via_yahoo_returns_articles(self) -> None:
         tool = StockNewsTool()
-        with patch.object(yahoo_client, "search", return_value=[]):
+        with patch.object(
+            yahoo_client, "search_news", return_value=_yahoo_news()[:1]
+        ) as srch:
             out = json.loads(tool.execute(code="00700.HK"))
 
+        srch.assert_called_once_with("00700", 20)
         assert out["ok"] is True
         assert out["market"] == "hk"
         assert out["source"] == "yahoo"
-        assert "articles" not in out["data"]
-        assert out["data"]["result_type"] == "matches"
-        assert out["data"]["matches"] == []
+        assert out["data"]["articles"][0]["title"] == "Apple unveils new products"
+
+    def test_yahoo_empty_news_returns_empty_articles(self) -> None:
+        with patch.object(yahoo_client, "search_news", return_value=[]):
+            out = json.loads(StockNewsTool().execute(code="AAPL.US"))
+
+        assert out["ok"] is True
+        assert out["data"]["articles"] == []
+
+    def test_yahoo_limit_is_clamped_before_request(self) -> None:
+        with patch.object(yahoo_client, "search_news", return_value=[]) as srch:
+            out = json.loads(StockNewsTool().execute(code="AAPL.US", limit=999))
+
+        assert out["ok"] is True
+        srch.assert_called_once_with("AAPL", 50)
 
 
 class TestExecuteError:
@@ -202,10 +236,10 @@ class TestExecuteError:
     def test_yahoo_failure_envelope(self) -> None:
         tool = StockNewsTool()
         with patch.object(
-            yahoo_client, "search", side_effect=RuntimeError("yahoo 429")
+            yahoo_client, "search_news", side_effect=RuntimeError("yahoo 429")
         ):
             out = json.loads(tool.execute(code="AAPL.US"))
 
         assert out["ok"] is False
         assert "yahoo 429" in out["error"]
-        assert "yahoo search fetch failed" in out["error"]
+        assert "yahoo news fetch failed" in out["error"]