fix(model-picker): OpenAI shows curated models; OpenRouter no longer phantom-shows (#37404)
The model picker now matches `hermes model` for OpenAI, and OpenRouter
stops appearing as authenticated when only OPENAI_API_KEY is set.
- models.py: provider_model_ids() for the default api.openai.com endpoint
intersects the live /v1/models dump (120+ entries incl. embeddings,
whisper, tts, dall-e, moderation, legacy chat) with the curated agentic
list, preserving curated order. Custom OpenAI-compatible endpoints keep
the live list verbatim so discovery still works.
- providers.py: drop extra_env_vars=("OPENAI_API_KEY",) from the openrouter
overlay. list_authenticated_providers reads extra_env_vars to decide
whether a provider is authenticated, so any OpenAI user saw a phantom
OpenRouter row. Runtime OpenRouter credential resolution still falls back
to OPENAI_API_KEY (runtime_provider.py), independent of the overlay.
- Regression tests for both paths.
This commit is contained in:
@ -2106,9 +2106,32 @@ def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False)
|
||||
if api_key:
|
||||
base_raw = os.getenv("OPENAI_BASE_URL", "").strip().rstrip("/")
|
||||
base = base_raw or "https://api.openai.com/v1"
|
||||
# Custom OpenAI-compatible endpoints (proxies, gateways, self-hosted)
|
||||
# may serve a small curated catalog — use the live list verbatim so
|
||||
# discovery works. But the canonical api.openai.com /v1/models dump
|
||||
# is 120+ entries of embeddings, whisper, tts, dall-e, moderation and
|
||||
# legacy chat models — none of which belong in the agent model picker.
|
||||
# For the default endpoint, intersect the live list with our curated
|
||||
# agentic catalog so ``/model`` matches what ``hermes model`` shows.
|
||||
is_default_openai = base.rstrip("/") in (
|
||||
"https://api.openai.com/v1",
|
||||
"https://api.openai.com",
|
||||
)
|
||||
try:
|
||||
live = fetch_api_models(api_key, base)
|
||||
if live:
|
||||
if is_default_openai:
|
||||
live_lower = {m.lower() for m in live}
|
||||
curated = list(_PROVIDER_MODELS.get(normalized, []))
|
||||
# Keep curated order; only surface curated models the
|
||||
# account actually has access to.
|
||||
filtered = [m for m in curated if m.lower() in live_lower]
|
||||
if filtered:
|
||||
return filtered
|
||||
# Account serves none of the curated models (rare —
|
||||
# e.g. org without GPT-5 access). Fall back to curated
|
||||
# so the picker still offers sane defaults.
|
||||
return curated or live
|
||||
return live
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@ -47,7 +47,6 @@ HERMES_OVERLAYS: Dict[str, HermesOverlay] = {
|
||||
"openrouter": HermesOverlay(
|
||||
transport="openai_chat",
|
||||
is_aggregator=True,
|
||||
extra_env_vars=("OPENAI_API_KEY",),
|
||||
base_url_env_var="OPENROUTER_BASE_URL",
|
||||
),
|
||||
"nous": HermesOverlay(
|
||||
|
||||
96
tests/hermes_cli/test_openai_picker_curated.py
Normal file
96
tests/hermes_cli/test_openai_picker_curated.py
Normal file
@ -0,0 +1,96 @@
|
||||
"""Regression tests for two OpenAI/OpenRouter model-picker bugs.
|
||||
|
||||
Bug 1 — OpenAI picker dumped the raw ``/v1/models`` catalog
|
||||
``provider_model_ids("openai")`` hit ``api.openai.com/v1/models`` and
|
||||
returned the full 120+ entry catalog (embeddings, whisper, tts, dall-e,
|
||||
moderation, gpt-3.5, …). The ``hermes model`` CLI shows only the curated
|
||||
agentic list. The picker now intersects the live default-endpoint catalog
|
||||
with the curated list (preserving curated order) so both surfaces match.
|
||||
Custom OpenAI-compatible endpoints (proxies, gateways) keep the live list
|
||||
verbatim so discovery still works.
|
||||
|
||||
Bug 2 — OpenRouter appeared authenticated whenever OPENAI_API_KEY was set
|
||||
OpenRouter's HermesOverlay carried ``extra_env_vars=("OPENAI_API_KEY",)``.
|
||||
``list_authenticated_providers`` reads ``extra_env_vars`` to decide whether
|
||||
a provider has credentials, so any OpenAI user saw a phantom OpenRouter
|
||||
row. The overlay entry is removed; runtime credential resolution still
|
||||
falls back to OPENAI_API_KEY for explicitly-selected OpenRouter (handled
|
||||
in runtime_provider.py, independent of the overlay).
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli import models as M
|
||||
from hermes_cli.providers import HERMES_OVERLAYS
|
||||
|
||||
|
||||
# --- Bug 2: overlay no longer lists OPENAI_API_KEY --------------------------
|
||||
|
||||
def test_openrouter_overlay_does_not_list_openai_api_key():
|
||||
overlay = HERMES_OVERLAYS["openrouter"]
|
||||
assert "OPENAI_API_KEY" not in overlay.extra_env_vars
|
||||
|
||||
|
||||
# --- Bug 1: default OpenAI endpoint filters to curated agentic models -------
|
||||
|
||||
def test_default_openai_endpoint_filters_to_curated(monkeypatch):
|
||||
"""The 126-model /v1/models dump is intersected with the curated list."""
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
|
||||
curated = M._PROVIDER_MODELS["openai-api"]
|
||||
# Live catalog: every curated model PLUS a pile of non-agentic junk.
|
||||
live = list(curated) + [
|
||||
"text-embedding-3-large", "whisper-1", "tts-1", "dall-e-3",
|
||||
"gpt-3.5-turbo", "davinci-002", "omni-moderation-latest",
|
||||
]
|
||||
with patch.object(M, "fetch_api_models", return_value=live):
|
||||
result = M.provider_model_ids("openai-api", force_refresh=True)
|
||||
|
||||
# Only curated models survive, in curated order, no junk.
|
||||
assert result == list(curated)
|
||||
for m in result:
|
||||
assert m in curated
|
||||
|
||||
|
||||
def test_default_openai_endpoint_intersects_account_access(monkeypatch):
|
||||
"""Curated models the account can't access are dropped (intersection)."""
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
|
||||
curated = M._PROVIDER_MODELS["openai-api"]
|
||||
# Account only serves the first two curated models.
|
||||
live = list(curated[:2]) + ["text-embedding-3-large", "whisper-1"]
|
||||
with patch.object(M, "fetch_api_models", return_value=live):
|
||||
result = M.provider_model_ids("openai-api", force_refresh=True)
|
||||
|
||||
assert result == list(curated[:2])
|
||||
|
||||
|
||||
def test_default_openai_endpoint_falls_back_when_no_curated_access(monkeypatch):
|
||||
"""If the account serves none of the curated models, fall back to curated."""
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
|
||||
curated = M._PROVIDER_MODELS["openai-api"]
|
||||
live = ["text-embedding-3-large", "whisper-1", "tts-1"] # all junk
|
||||
with patch.object(M, "fetch_api_models", return_value=live):
|
||||
result = M.provider_model_ids("openai-api", force_refresh=True)
|
||||
|
||||
# No curated overlap -> serve the curated defaults so the picker isn't empty.
|
||||
assert result == list(curated)
|
||||
|
||||
|
||||
def test_custom_openai_compatible_endpoint_keeps_live_list(monkeypatch):
|
||||
"""Custom OPENAI_BASE_URL endpoints keep the live catalog verbatim."""
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
|
||||
monkeypatch.setenv("OPENAI_BASE_URL", "https://my-proxy.example.com/v1")
|
||||
|
||||
live = ["custom-model-a", "custom-model-b", "some-embedding-model"]
|
||||
with patch.object(M, "fetch_api_models", return_value=live):
|
||||
result = M.provider_model_ids("openai-api", force_refresh=True)
|
||||
|
||||
assert result == live
|
||||
Reference in New Issue
Block a user