fix(model-picker): OpenAI shows curated models; OpenRouter no longer phantom-shows (#37404)

The model picker now matches `hermes model` for OpenAI, and OpenRouter
stops appearing as authenticated when only OPENAI_API_KEY is set.

- models.py: provider_model_ids() for the default api.openai.com endpoint
  intersects the live /v1/models dump (120+ entries incl. embeddings,
  whisper, tts, dall-e, moderation, legacy chat) with the curated agentic
  list, preserving curated order. Custom OpenAI-compatible endpoints keep
  the live list verbatim so discovery still works.
- providers.py: drop extra_env_vars=("OPENAI_API_KEY",) from the openrouter
  overlay. list_authenticated_providers reads extra_env_vars to decide
  whether a provider is authenticated, so any OpenAI user saw a phantom
  OpenRouter row. Runtime OpenRouter credential resolution still falls back
  to OPENAI_API_KEY (runtime_provider.py), independent of the overlay.
- Regression tests for both paths.
This commit is contained in:
Teknium
2026-06-02 06:31:37 -07:00
committed by GitHub
parent 195c4d2a98
commit afea650e16
3 changed files with 119 additions and 1 deletions

View File

@ -2106,9 +2106,32 @@ def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False)
if api_key: if api_key:
base_raw = os.getenv("OPENAI_BASE_URL", "").strip().rstrip("/") base_raw = os.getenv("OPENAI_BASE_URL", "").strip().rstrip("/")
base = base_raw or "https://api.openai.com/v1" base = base_raw or "https://api.openai.com/v1"
# Custom OpenAI-compatible endpoints (proxies, gateways, self-hosted)
# may serve a small curated catalog — use the live list verbatim so
# discovery works. But the canonical api.openai.com /v1/models dump
# is 120+ entries of embeddings, whisper, tts, dall-e, moderation and
# legacy chat models — none of which belong in the agent model picker.
# For the default endpoint, intersect the live list with our curated
# agentic catalog so ``/model`` matches what ``hermes model`` shows.
is_default_openai = base.rstrip("/") in (
"https://api.openai.com/v1",
"https://api.openai.com",
)
try: try:
live = fetch_api_models(api_key, base) live = fetch_api_models(api_key, base)
if live: if live:
if is_default_openai:
live_lower = {m.lower() for m in live}
curated = list(_PROVIDER_MODELS.get(normalized, []))
# Keep curated order; only surface curated models the
# account actually has access to.
filtered = [m for m in curated if m.lower() in live_lower]
if filtered:
return filtered
# Account serves none of the curated models (rare —
# e.g. org without GPT-5 access). Fall back to curated
# so the picker still offers sane defaults.
return curated or live
return live return live
except Exception: except Exception:
pass pass

View File

@ -47,7 +47,6 @@ HERMES_OVERLAYS: Dict[str, HermesOverlay] = {
"openrouter": HermesOverlay( "openrouter": HermesOverlay(
transport="openai_chat", transport="openai_chat",
is_aggregator=True, is_aggregator=True,
extra_env_vars=("OPENAI_API_KEY",),
base_url_env_var="OPENROUTER_BASE_URL", base_url_env_var="OPENROUTER_BASE_URL",
), ),
"nous": HermesOverlay( "nous": HermesOverlay(

View File

@ -0,0 +1,96 @@
"""Regression tests for two OpenAI/OpenRouter model-picker bugs.
Bug 1 — OpenAI picker dumped the raw ``/v1/models`` catalog
``provider_model_ids("openai")`` hit ``api.openai.com/v1/models`` and
returned the full 120+ entry catalog (embeddings, whisper, tts, dall-e,
moderation, gpt-3.5, …). The ``hermes model`` CLI shows only the curated
agentic list. The picker now intersects the live default-endpoint catalog
with the curated list (preserving curated order) so both surfaces match.
Custom OpenAI-compatible endpoints (proxies, gateways) keep the live list
verbatim so discovery still works.
Bug 2 — OpenRouter appeared authenticated whenever OPENAI_API_KEY was set
OpenRouter's HermesOverlay carried ``extra_env_vars=("OPENAI_API_KEY",)``.
``list_authenticated_providers`` reads ``extra_env_vars`` to decide whether
a provider has credentials, so any OpenAI user saw a phantom OpenRouter
row. The overlay entry is removed; runtime credential resolution still
falls back to OPENAI_API_KEY for explicitly-selected OpenRouter (handled
in runtime_provider.py, independent of the overlay).
"""
import os
from unittest.mock import patch
import pytest
from hermes_cli import models as M
from hermes_cli.providers import HERMES_OVERLAYS
# --- Bug 2: overlay no longer lists OPENAI_API_KEY --------------------------
def test_openrouter_overlay_does_not_list_openai_api_key():
overlay = HERMES_OVERLAYS["openrouter"]
assert "OPENAI_API_KEY" not in overlay.extra_env_vars
# --- Bug 1: default OpenAI endpoint filters to curated agentic models -------
def test_default_openai_endpoint_filters_to_curated(monkeypatch):
"""The 126-model /v1/models dump is intersected with the curated list."""
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
curated = M._PROVIDER_MODELS["openai-api"]
# Live catalog: every curated model PLUS a pile of non-agentic junk.
live = list(curated) + [
"text-embedding-3-large", "whisper-1", "tts-1", "dall-e-3",
"gpt-3.5-turbo", "davinci-002", "omni-moderation-latest",
]
with patch.object(M, "fetch_api_models", return_value=live):
result = M.provider_model_ids("openai-api", force_refresh=True)
# Only curated models survive, in curated order, no junk.
assert result == list(curated)
for m in result:
assert m in curated
def test_default_openai_endpoint_intersects_account_access(monkeypatch):
"""Curated models the account can't access are dropped (intersection)."""
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
curated = M._PROVIDER_MODELS["openai-api"]
# Account only serves the first two curated models.
live = list(curated[:2]) + ["text-embedding-3-large", "whisper-1"]
with patch.object(M, "fetch_api_models", return_value=live):
result = M.provider_model_ids("openai-api", force_refresh=True)
assert result == list(curated[:2])
def test_default_openai_endpoint_falls_back_when_no_curated_access(monkeypatch):
"""If the account serves none of the curated models, fall back to curated."""
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
curated = M._PROVIDER_MODELS["openai-api"]
live = ["text-embedding-3-large", "whisper-1", "tts-1"] # all junk
with patch.object(M, "fetch_api_models", return_value=live):
result = M.provider_model_ids("openai-api", force_refresh=True)
# No curated overlap -> serve the curated defaults so the picker isn't empty.
assert result == list(curated)
def test_custom_openai_compatible_endpoint_keeps_live_list(monkeypatch):
"""Custom OPENAI_BASE_URL endpoints keep the live catalog verbatim."""
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
monkeypatch.setenv("OPENAI_BASE_URL", "https://my-proxy.example.com/v1")
live = ["custom-model-a", "custom-model-b", "some-embedding-model"]
with patch.object(M, "fetch_api_models", return_value=live):
result = M.provider_model_ids("openai-api", force_refresh=True)
assert result == live