From afea650e16c544457efed023c1df05fadd86c500 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Tue, 2 Jun 2026 06:31:37 -0700 Subject: [PATCH] fix(model-picker): OpenAI shows curated models; OpenRouter no longer phantom-shows (#37404) The model picker now matches `hermes model` for OpenAI, and OpenRouter stops appearing as authenticated when only OPENAI_API_KEY is set. - models.py: provider_model_ids() for the default api.openai.com endpoint intersects the live /v1/models dump (120+ entries incl. embeddings, whisper, tts, dall-e, moderation, legacy chat) with the curated agentic list, preserving curated order. Custom OpenAI-compatible endpoints keep the live list verbatim so discovery still works. - providers.py: drop extra_env_vars=("OPENAI_API_KEY",) from the openrouter overlay. list_authenticated_providers reads extra_env_vars to decide whether a provider is authenticated, so any OpenAI user saw a phantom OpenRouter row. Runtime OpenRouter credential resolution still falls back to OPENAI_API_KEY (runtime_provider.py), independent of the overlay. - Regression tests for both paths. --- hermes_cli/models.py | 23 +++++ hermes_cli/providers.py | 1 - .../hermes_cli/test_openai_picker_curated.py | 96 +++++++++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 tests/hermes_cli/test_openai_picker_curated.py diff --git a/hermes_cli/models.py b/hermes_cli/models.py index e1e066851..99f9d462c 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2106,9 +2106,32 @@ def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False) if api_key: base_raw = os.getenv("OPENAI_BASE_URL", "").strip().rstrip("/") base = base_raw or "https://api.openai.com/v1" + # Custom OpenAI-compatible endpoints (proxies, gateways, self-hosted) + # may serve a small curated catalog — use the live list verbatim so + # discovery works. But the canonical api.openai.com /v1/models dump + # is 120+ entries of embeddings, whisper, tts, dall-e, moderation and + # legacy chat models — none of which belong in the agent model picker. + # For the default endpoint, intersect the live list with our curated + # agentic catalog so ``/model`` matches what ``hermes model`` shows. + is_default_openai = base.rstrip("/") in ( + "https://api.openai.com/v1", + "https://api.openai.com", + ) try: live = fetch_api_models(api_key, base) if live: + if is_default_openai: + live_lower = {m.lower() for m in live} + curated = list(_PROVIDER_MODELS.get(normalized, [])) + # Keep curated order; only surface curated models the + # account actually has access to. + filtered = [m for m in curated if m.lower() in live_lower] + if filtered: + return filtered + # Account serves none of the curated models (rare — + # e.g. org without GPT-5 access). Fall back to curated + # so the picker still offers sane defaults. + return curated or live return live except Exception: pass diff --git a/hermes_cli/providers.py b/hermes_cli/providers.py index f81790aa2..ba25f7e63 100644 --- a/hermes_cli/providers.py +++ b/hermes_cli/providers.py @@ -47,7 +47,6 @@ HERMES_OVERLAYS: Dict[str, HermesOverlay] = { "openrouter": HermesOverlay( transport="openai_chat", is_aggregator=True, - extra_env_vars=("OPENAI_API_KEY",), base_url_env_var="OPENROUTER_BASE_URL", ), "nous": HermesOverlay( diff --git a/tests/hermes_cli/test_openai_picker_curated.py b/tests/hermes_cli/test_openai_picker_curated.py new file mode 100644 index 000000000..67b7f2c11 --- /dev/null +++ b/tests/hermes_cli/test_openai_picker_curated.py @@ -0,0 +1,96 @@ +"""Regression tests for two OpenAI/OpenRouter model-picker bugs. + +Bug 1 — OpenAI picker dumped the raw ``/v1/models`` catalog + ``provider_model_ids("openai")`` hit ``api.openai.com/v1/models`` and + returned the full 120+ entry catalog (embeddings, whisper, tts, dall-e, + moderation, gpt-3.5, …). The ``hermes model`` CLI shows only the curated + agentic list. The picker now intersects the live default-endpoint catalog + with the curated list (preserving curated order) so both surfaces match. + Custom OpenAI-compatible endpoints (proxies, gateways) keep the live list + verbatim so discovery still works. + +Bug 2 — OpenRouter appeared authenticated whenever OPENAI_API_KEY was set + OpenRouter's HermesOverlay carried ``extra_env_vars=("OPENAI_API_KEY",)``. + ``list_authenticated_providers`` reads ``extra_env_vars`` to decide whether + a provider has credentials, so any OpenAI user saw a phantom OpenRouter + row. The overlay entry is removed; runtime credential resolution still + falls back to OPENAI_API_KEY for explicitly-selected OpenRouter (handled + in runtime_provider.py, independent of the overlay). +""" + +import os +from unittest.mock import patch + +import pytest + +from hermes_cli import models as M +from hermes_cli.providers import HERMES_OVERLAYS + + +# --- Bug 2: overlay no longer lists OPENAI_API_KEY -------------------------- + +def test_openrouter_overlay_does_not_list_openai_api_key(): + overlay = HERMES_OVERLAYS["openrouter"] + assert "OPENAI_API_KEY" not in overlay.extra_env_vars + + +# --- Bug 1: default OpenAI endpoint filters to curated agentic models ------- + +def test_default_openai_endpoint_filters_to_curated(monkeypatch): + """The 126-model /v1/models dump is intersected with the curated list.""" + monkeypatch.setenv("OPENAI_API_KEY", "sk-fake") + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + + curated = M._PROVIDER_MODELS["openai-api"] + # Live catalog: every curated model PLUS a pile of non-agentic junk. + live = list(curated) + [ + "text-embedding-3-large", "whisper-1", "tts-1", "dall-e-3", + "gpt-3.5-turbo", "davinci-002", "omni-moderation-latest", + ] + with patch.object(M, "fetch_api_models", return_value=live): + result = M.provider_model_ids("openai-api", force_refresh=True) + + # Only curated models survive, in curated order, no junk. + assert result == list(curated) + for m in result: + assert m in curated + + +def test_default_openai_endpoint_intersects_account_access(monkeypatch): + """Curated models the account can't access are dropped (intersection).""" + monkeypatch.setenv("OPENAI_API_KEY", "sk-fake") + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + + curated = M._PROVIDER_MODELS["openai-api"] + # Account only serves the first two curated models. + live = list(curated[:2]) + ["text-embedding-3-large", "whisper-1"] + with patch.object(M, "fetch_api_models", return_value=live): + result = M.provider_model_ids("openai-api", force_refresh=True) + + assert result == list(curated[:2]) + + +def test_default_openai_endpoint_falls_back_when_no_curated_access(monkeypatch): + """If the account serves none of the curated models, fall back to curated.""" + monkeypatch.setenv("OPENAI_API_KEY", "sk-fake") + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + + curated = M._PROVIDER_MODELS["openai-api"] + live = ["text-embedding-3-large", "whisper-1", "tts-1"] # all junk + with patch.object(M, "fetch_api_models", return_value=live): + result = M.provider_model_ids("openai-api", force_refresh=True) + + # No curated overlap -> serve the curated defaults so the picker isn't empty. + assert result == list(curated) + + +def test_custom_openai_compatible_endpoint_keeps_live_list(monkeypatch): + """Custom OPENAI_BASE_URL endpoints keep the live catalog verbatim.""" + monkeypatch.setenv("OPENAI_API_KEY", "sk-fake") + monkeypatch.setenv("OPENAI_BASE_URL", "https://my-proxy.example.com/v1") + + live = ["custom-model-a", "custom-model-b", "some-embedding-model"] + with patch.object(M, "fetch_api_models", return_value=live): + result = M.provider_model_ids("openai-api", force_refresh=True) + + assert result == live