fix(model-picker): OpenAI shows curated models; OpenRouter no longer phantom-shows (#37404)
The model picker now matches `hermes model` for OpenAI, and OpenRouter
stops appearing as authenticated when only OPENAI_API_KEY is set.
- models.py: provider_model_ids() for the default api.openai.com endpoint
intersects the live /v1/models dump (120+ entries incl. embeddings,
whisper, tts, dall-e, moderation, legacy chat) with the curated agentic
list, preserving curated order. Custom OpenAI-compatible endpoints keep
the live list verbatim so discovery still works.
- providers.py: drop extra_env_vars=("OPENAI_API_KEY",) from the openrouter
overlay. list_authenticated_providers reads extra_env_vars to decide
whether a provider is authenticated, so any OpenAI user saw a phantom
OpenRouter row. Runtime OpenRouter credential resolution still falls back
to OPENAI_API_KEY (runtime_provider.py), independent of the overlay.
- Regression tests for both paths.
This commit is contained in:
@ -2106,9 +2106,32 @@ def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False)
|
|||||||
if api_key:
|
if api_key:
|
||||||
base_raw = os.getenv("OPENAI_BASE_URL", "").strip().rstrip("/")
|
base_raw = os.getenv("OPENAI_BASE_URL", "").strip().rstrip("/")
|
||||||
base = base_raw or "https://api.openai.com/v1"
|
base = base_raw or "https://api.openai.com/v1"
|
||||||
|
# Custom OpenAI-compatible endpoints (proxies, gateways, self-hosted)
|
||||||
|
# may serve a small curated catalog — use the live list verbatim so
|
||||||
|
# discovery works. But the canonical api.openai.com /v1/models dump
|
||||||
|
# is 120+ entries of embeddings, whisper, tts, dall-e, moderation and
|
||||||
|
# legacy chat models — none of which belong in the agent model picker.
|
||||||
|
# For the default endpoint, intersect the live list with our curated
|
||||||
|
# agentic catalog so ``/model`` matches what ``hermes model`` shows.
|
||||||
|
is_default_openai = base.rstrip("/") in (
|
||||||
|
"https://api.openai.com/v1",
|
||||||
|
"https://api.openai.com",
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
live = fetch_api_models(api_key, base)
|
live = fetch_api_models(api_key, base)
|
||||||
if live:
|
if live:
|
||||||
|
if is_default_openai:
|
||||||
|
live_lower = {m.lower() for m in live}
|
||||||
|
curated = list(_PROVIDER_MODELS.get(normalized, []))
|
||||||
|
# Keep curated order; only surface curated models the
|
||||||
|
# account actually has access to.
|
||||||
|
filtered = [m for m in curated if m.lower() in live_lower]
|
||||||
|
if filtered:
|
||||||
|
return filtered
|
||||||
|
# Account serves none of the curated models (rare —
|
||||||
|
# e.g. org without GPT-5 access). Fall back to curated
|
||||||
|
# so the picker still offers sane defaults.
|
||||||
|
return curated or live
|
||||||
return live
|
return live
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|||||||
@ -47,7 +47,6 @@ HERMES_OVERLAYS: Dict[str, HermesOverlay] = {
|
|||||||
"openrouter": HermesOverlay(
|
"openrouter": HermesOverlay(
|
||||||
transport="openai_chat",
|
transport="openai_chat",
|
||||||
is_aggregator=True,
|
is_aggregator=True,
|
||||||
extra_env_vars=("OPENAI_API_KEY",),
|
|
||||||
base_url_env_var="OPENROUTER_BASE_URL",
|
base_url_env_var="OPENROUTER_BASE_URL",
|
||||||
),
|
),
|
||||||
"nous": HermesOverlay(
|
"nous": HermesOverlay(
|
||||||
|
|||||||
96
tests/hermes_cli/test_openai_picker_curated.py
Normal file
96
tests/hermes_cli/test_openai_picker_curated.py
Normal file
@ -0,0 +1,96 @@
|
|||||||
|
"""Regression tests for two OpenAI/OpenRouter model-picker bugs.
|
||||||
|
|
||||||
|
Bug 1 — OpenAI picker dumped the raw ``/v1/models`` catalog
|
||||||
|
``provider_model_ids("openai")`` hit ``api.openai.com/v1/models`` and
|
||||||
|
returned the full 120+ entry catalog (embeddings, whisper, tts, dall-e,
|
||||||
|
moderation, gpt-3.5, …). The ``hermes model`` CLI shows only the curated
|
||||||
|
agentic list. The picker now intersects the live default-endpoint catalog
|
||||||
|
with the curated list (preserving curated order) so both surfaces match.
|
||||||
|
Custom OpenAI-compatible endpoints (proxies, gateways) keep the live list
|
||||||
|
verbatim so discovery still works.
|
||||||
|
|
||||||
|
Bug 2 — OpenRouter appeared authenticated whenever OPENAI_API_KEY was set
|
||||||
|
OpenRouter's HermesOverlay carried ``extra_env_vars=("OPENAI_API_KEY",)``.
|
||||||
|
``list_authenticated_providers`` reads ``extra_env_vars`` to decide whether
|
||||||
|
a provider has credentials, so any OpenAI user saw a phantom OpenRouter
|
||||||
|
row. The overlay entry is removed; runtime credential resolution still
|
||||||
|
falls back to OPENAI_API_KEY for explicitly-selected OpenRouter (handled
|
||||||
|
in runtime_provider.py, independent of the overlay).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from hermes_cli import models as M
|
||||||
|
from hermes_cli.providers import HERMES_OVERLAYS
|
||||||
|
|
||||||
|
|
||||||
|
# --- Bug 2: overlay no longer lists OPENAI_API_KEY --------------------------
|
||||||
|
|
||||||
|
def test_openrouter_overlay_does_not_list_openai_api_key():
|
||||||
|
overlay = HERMES_OVERLAYS["openrouter"]
|
||||||
|
assert "OPENAI_API_KEY" not in overlay.extra_env_vars
|
||||||
|
|
||||||
|
|
||||||
|
# --- Bug 1: default OpenAI endpoint filters to curated agentic models -------
|
||||||
|
|
||||||
|
def test_default_openai_endpoint_filters_to_curated(monkeypatch):
|
||||||
|
"""The 126-model /v1/models dump is intersected with the curated list."""
|
||||||
|
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
|
||||||
|
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||||
|
|
||||||
|
curated = M._PROVIDER_MODELS["openai-api"]
|
||||||
|
# Live catalog: every curated model PLUS a pile of non-agentic junk.
|
||||||
|
live = list(curated) + [
|
||||||
|
"text-embedding-3-large", "whisper-1", "tts-1", "dall-e-3",
|
||||||
|
"gpt-3.5-turbo", "davinci-002", "omni-moderation-latest",
|
||||||
|
]
|
||||||
|
with patch.object(M, "fetch_api_models", return_value=live):
|
||||||
|
result = M.provider_model_ids("openai-api", force_refresh=True)
|
||||||
|
|
||||||
|
# Only curated models survive, in curated order, no junk.
|
||||||
|
assert result == list(curated)
|
||||||
|
for m in result:
|
||||||
|
assert m in curated
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_openai_endpoint_intersects_account_access(monkeypatch):
|
||||||
|
"""Curated models the account can't access are dropped (intersection)."""
|
||||||
|
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
|
||||||
|
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||||
|
|
||||||
|
curated = M._PROVIDER_MODELS["openai-api"]
|
||||||
|
# Account only serves the first two curated models.
|
||||||
|
live = list(curated[:2]) + ["text-embedding-3-large", "whisper-1"]
|
||||||
|
with patch.object(M, "fetch_api_models", return_value=live):
|
||||||
|
result = M.provider_model_ids("openai-api", force_refresh=True)
|
||||||
|
|
||||||
|
assert result == list(curated[:2])
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_openai_endpoint_falls_back_when_no_curated_access(monkeypatch):
|
||||||
|
"""If the account serves none of the curated models, fall back to curated."""
|
||||||
|
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
|
||||||
|
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||||
|
|
||||||
|
curated = M._PROVIDER_MODELS["openai-api"]
|
||||||
|
live = ["text-embedding-3-large", "whisper-1", "tts-1"] # all junk
|
||||||
|
with patch.object(M, "fetch_api_models", return_value=live):
|
||||||
|
result = M.provider_model_ids("openai-api", force_refresh=True)
|
||||||
|
|
||||||
|
# No curated overlap -> serve the curated defaults so the picker isn't empty.
|
||||||
|
assert result == list(curated)
|
||||||
|
|
||||||
|
|
||||||
|
def test_custom_openai_compatible_endpoint_keeps_live_list(monkeypatch):
|
||||||
|
"""Custom OPENAI_BASE_URL endpoints keep the live catalog verbatim."""
|
||||||
|
monkeypatch.setenv("OPENAI_API_KEY", "sk-fake")
|
||||||
|
monkeypatch.setenv("OPENAI_BASE_URL", "https://my-proxy.example.com/v1")
|
||||||
|
|
||||||
|
live = ["custom-model-a", "custom-model-b", "some-embedding-model"]
|
||||||
|
with patch.object(M, "fetch_api_models", return_value=live):
|
||||||
|
result = M.provider_model_ids("openai-api", force_refresh=True)
|
||||||
|
|
||||||
|
assert result == live
|
||||||
Reference in New Issue
Block a user