openclaw/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 9 Packages Projects Releases Wiki Activity

test: remove low-value model-catalog mirror tests

Browse Source 
These tests asserted that hardcoded curated model lists/constants still
contained specific model strings (e.g. 'glm-5' in provider_model_ids('zai'),
exact context-length values per model key, PROVIDER_TO_MODELS_DEV entries).
They mirror a constant rather than exercise logic, so they only ever break
when models are added/retired and never catch a real bug.

Removed 22 such functions across 7 files (149 deletions, 0 additions).
Behavioral siblings are kept: live-catalog-wins, fallback ordering,
substring/longest-match resolution, normalization, credential discovery,
and probe-tier stepping all still tested.
This commit is contained in:
kshitijk4poor
2026-05-30 11:39:25 +05:30
committed by kshitij
parent 0437137fff
commit 7b0915037c
7 changed files with 0 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
tests/agent/test_model_metadata.py
View File

@ -123,55 +123,6 @@ class TestEstimateMessagesTokensRough:
# =========================================================================
class TestDefaultContextLengths:
def test_claude_models_context_lengths(self):
for key, value in DEFAULT_CONTEXT_LENGTHS.items():
if "claude" not in key:
continue
# Claude 4.6+ models (4.6, 4.7, 4.8) have 1M context at standard
# API pricing (no long-context premium). Older Claude 4.x and
# 3.x models cap at 200k.
if any(tag in key for tag in ("4.6", "4-6", "4.7", "4-7", "4.8", "4-8")):
assert value == 1000000, f"{key} should be 1000000"
else:
assert value == 200000, f"{key} should be 200000"
def test_gpt4_models_128k_or_1m(self):
# gpt-4.1 and gpt-4.1-mini have 1M context; other gpt-4* have 128k
for key, value in DEFAULT_CONTEXT_LENGTHS.items():
if "gpt-4" in key and "gpt-4.1" not in key:
assert value == 128000, f"{key} should be 128000"
def test_gpt41_models_1m(self):
for key, value in DEFAULT_CONTEXT_LENGTHS.items():
if "gpt-4.1" in key:
assert value == 1047576, f"{key} should be 1047576"
def test_gemini_models_1m(self):
for key, value in DEFAULT_CONTEXT_LENGTHS.items():
if "gemini" in key:
assert value == 1048576, f"{key} should be 1048576"
def test_grok_models_context_lengths(self):
# xAI /v1/models does not return context_length metadata, so
# DEFAULT_CONTEXT_LENGTHS must cover the Grok family explicitly.
# Values sourced from models.dev (2026-04).
expected = {
"grok-4.20": 2000000,
"grok-4-fast": 2000000,
"grok-4": 256000,
"grok-build": 256000,
"grok-code-fast": 256000,
"grok-3": 131072,
"grok-2": 131072,
"grok-2-vision": 8192,
"grok": 131072,
}
for key, value in expected.items():
assert key in DEFAULT_CONTEXT_LENGTHS, f"{key} missing from DEFAULT_CONTEXT_LENGTHS"
assert DEFAULT_CONTEXT_LENGTHS[key] == value, (
f"{key} should be {value}, got {DEFAULT_CONTEXT_LENGTHS[key]}"
)
def test_grok_substring_matching(self):
# Longest-first substring matching must resolve the real xAI model
# IDs to the correct fallback entries without 128k probe-down.
@ -268,13 +219,6 @@ class TestDefaultContextLengths:
f"{model_id}: expected {expected_ctx}, got {actual}"
)
def test_all_values_positive(self):
for key, value in DEFAULT_CONTEXT_LENGTHS.items():
assert value > 0, f"{key} has non-positive context length"
def test_dict_is_not_empty(self):
assert len(DEFAULT_CONTEXT_LENGTHS) >= 10
# =========================================================================
# Codex OAuth context-window resolution (provider="openai-codex")
@ -1141,12 +1085,6 @@ class TestContextProbeTiers:
for i in range(len(CONTEXT_PROBE_TIERS) - 1):
assert CONTEXT_PROBE_TIERS[i] > CONTEXT_PROBE_TIERS[i + 1]
def test_first_tier_is_256k(self):
assert CONTEXT_PROBE_TIERS[0] == 256_000
def test_last_tier_is_8k(self):
assert CONTEXT_PROBE_TIERS[-1] == 8_000
class TestGetNextProbeTier:
def test_from_256k(self):

11
tests/agent/test_models_dev.py
View File

@ -82,17 +82,6 @@ SAMPLE_REGISTRY = {
class TestProviderMapping:
def test_all_mapped_providers_are_strings(self):
for hermes_id, mdev_id in PROVIDER_TO_MODELS_DEV.items():
assert isinstance(hermes_id, str)
assert isinstance(mdev_id, str)
def test_known_providers_mapped(self):
assert PROVIDER_TO_MODELS_DEV["anthropic"] == "anthropic"
assert PROVIDER_TO_MODELS_DEV["copilot"] == "github-copilot"
assert PROVIDER_TO_MODELS_DEV["stepfun"] == "stepfun"
assert PROVIDER_TO_MODELS_DEV["kilocode"] == "kilo"
def test_xai_oauth_uses_xai_catalog(self):
assert PROVIDER_TO_MODELS_DEV["xai"] == "xai"
assert PROVIDER_TO_MODELS_DEV["xai-oauth"] == "xai"

19
tests/hermes_cli/test_copilot_in_model_list.py
View File

@ -6,25 +6,6 @@ from unittest.mock import patch
from hermes_cli.model_switch import list_authenticated_providers
@patch.dict(os.environ, {"GH_TOKEN": "test-key"}, clear=False)
def test_copilot_picker_keeps_curated_copilot_models_when_live_catalog_unavailable():
with patch("agent.models_dev.fetch_models_dev", return_value={}), \
patch("hermes_cli.models._resolve_copilot_catalog_api_key", return_value="gh-token"), \
patch("hermes_cli.models._fetch_github_models", return_value=None):
providers = list_authenticated_providers(current_provider="openrouter", max_models=50)
copilot = next((p for p in providers if p["slug"] == "copilot"), None)
assert copilot is not None
assert "gpt-5.4" in copilot["models"]
assert "claude-sonnet-4.6" in copilot["models"]
assert "claude-sonnet-4" in copilot["models"]
assert "claude-sonnet-4.5" in copilot["models"]
assert "claude-haiku-4.5" in copilot["models"]
assert "gemini-3.1-pro-preview" in copilot["models"]
assert "claude-opus-4.6" not in copilot["models"]
@patch.dict(os.environ, {"GH_TOKEN": "test-key"}, clear=False)
def test_copilot_picker_uses_live_catalog_when_available():
live_models = ["gpt-5.4", "claude-sonnet-4.6", "gemini-3.1-pro-preview"]

13
tests/hermes_cli/test_gmi_provider.py
View File

@ -80,14 +80,6 @@ class TestGmiConfigRegistry:
class TestGmiModelCatalog:
def test_static_model_fallback_exists(self):
assert "gmi" in _PROVIDER_MODELS
models = _PROVIDER_MODELS["gmi"]
assert "zai-org/GLM-5.1-FP8" in models
assert "deepseek-ai/DeepSeek-V3.2" in models
assert "moonshotai/Kimi-K2.5" in models
assert "anthropic/claude-sonnet-4.6" in models
def test_canonical_provider_entry(self):
slugs = [p.slug for p in CANONICAL_PROVIDERS]
assert "gmi" in slugs
@ -267,11 +259,6 @@ class TestGmiModelMetadata:
class TestGmiAuxiliary:
def test_aux_default_model(self):
from agent.auxiliary_client import _get_aux_model_for_provider
assert _get_aux_model_for_provider("gmi") == "google/gemini-3.1-flash-lite-preview"
def test_resolve_provider_client_uses_gmi_aux_default(self, monkeypatch):
monkeypatch.setenv("GMI_API_KEY", "gmi-test-key")

32
tests/hermes_cli/test_model_validation.py
View File

@ -142,10 +142,6 @@ class TestCuratedModelsForProvider:
assert len(models) > 0
assert any("claude" in m[0] for m in models)
def test_zai_returns_glm_models(self):
models = curated_models_for_provider("zai")
assert any("glm" in m[0] for m in models)
def test_unknown_provider_returns_empty(self):
assert curated_models_for_provider("totally-unknown") == []
@ -199,9 +195,6 @@ class TestProviderModelIds:
def test_unknown_provider_returns_empty(self):
assert provider_model_ids("some-unknown-provider") == []
def test_zai_returns_glm_models(self):
assert "glm-5" in provider_model_ids("zai")
def test_stepfun_prefers_live_catalog(self):
with patch(
"hermes_cli.auth.resolve_api_key_provider_credentials",
@ -222,31 +215,6 @@ class TestProviderModelIds:
patch("hermes_cli.models._fetch_github_models", return_value=["gpt-5.4", "claude-sonnet-4.6"]):
assert provider_model_ids("copilot-acp") == ["gpt-5.4", "claude-sonnet-4.6"]
def test_copilot_falls_back_to_curated_defaults_without_stale_opus(self):
with patch("hermes_cli.models._resolve_copilot_catalog_api_key", return_value="gh-token"), \
patch("hermes_cli.models._fetch_github_models", return_value=None):
ids = provider_model_ids("copilot")
assert "gpt-5.4" in ids
assert "claude-sonnet-4.6" in ids
assert "claude-sonnet-4" in ids
assert "claude-sonnet-4.5" in ids
assert "claude-haiku-4.5" in ids
assert "gemini-3.1-pro-preview" in ids
assert "claude-opus-4.6" not in ids
def test_copilot_acp_falls_back_to_copilot_defaults(self):
with patch("hermes_cli.models._resolve_copilot_catalog_api_key", return_value="gh-token"), \
patch("hermes_cli.models._fetch_github_models", return_value=None):
ids = provider_model_ids("copilot-acp")
assert "gpt-5.4" in ids
assert "claude-sonnet-4.6" in ids
assert "claude-sonnet-4" in ids
assert "gemini-3.1-pro-preview" in ids
assert "copilot-acp" not in ids
assert "claude-opus-4.6" not in ids
# -- fetch_api_models --------------------------------------------------------

4
tests/hermes_cli/test_models.py
View File

@ -56,10 +56,6 @@ class TestOpenRouterModels:
assert isinstance(mid, str) and len(mid) > 0
assert isinstance(desc, str)
def test_at_least_5_models(self):
"""Sanity check that the models list hasn't been accidentally truncated."""
assert len(OPENROUTER_MODELS) >= 5
class TestFetchOpenRouterModels:
def test_live_fetch_recomputes_free_tags(self, monkeypatch):

9
tests/hermes_cli/test_ollama_cloud_provider.py
View File

@ -495,12 +495,3 @@ class TestOllamaCloudSuffixStripping:
assert _strip_ollama_cloud_suffix("qwen3-coder:480b-cloud") == "qwen3-coder:480b"
assert _strip_ollama_cloud_suffix("nemotron-3-nano:30b") == "nemotron-3-nano:30b"
assert _strip_ollama_cloud_suffix("") == ""
# ── Auxiliary Model ──
class TestOllamaCloudAuxiliary:
def test_aux_model_defined(self):
from agent.auxiliary_client import _API_KEY_PROVIDER_AUX_MODELS
assert "ollama-cloud" in _API_KEY_PROVIDER_AUX_MODELS
assert _API_KEY_PROVIDER_AUX_MODELS["ollama-cloud"] == "nemotron-3-nano:30b"