336 lines
12 KiB
Python
336 lines
12 KiB
Python
"""Tests for OpenCode Zen model registry functionality."""
|
|
|
|
import json
|
|
import os
|
|
import tempfile
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from providers.registries.zen import ZenModelRegistry
|
|
from providers.shared import ProviderType
|
|
|
|
|
|
class TestZenModelRegistry:
|
|
"""Test cases for Zen model registry."""
|
|
|
|
def test_registry_initialization(self):
|
|
"""Test registry initializes with default config."""
|
|
registry = ZenModelRegistry()
|
|
|
|
# Should load models from default location
|
|
assert len(registry.list_models()) > 0
|
|
assert len(registry.list_aliases()) > 0
|
|
|
|
# Should include our configured models
|
|
assert "claude-sonnet-4-5" in registry.list_models()
|
|
assert "gpt-5.1-codex" in registry.list_models()
|
|
|
|
def test_custom_config_path(self):
|
|
"""Test registry with custom config path."""
|
|
# Create temporary config
|
|
config_data = {
|
|
"models": [
|
|
{
|
|
"model_name": "test/zen-model-1",
|
|
"aliases": ["zen-test1", "zt1"],
|
|
"context_window": 4096,
|
|
"max_output_tokens": 2048,
|
|
"intelligence_score": 15,
|
|
}
|
|
]
|
|
}
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
|
|
json.dump(config_data, f)
|
|
temp_path = f.name
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as live_file:
|
|
json.dump({"models": []}, live_file)
|
|
live_path = live_file.name
|
|
|
|
try:
|
|
registry = ZenModelRegistry(config_path=temp_path, live_config_path=live_path)
|
|
assert len(registry.list_models()) == 1
|
|
assert "test/zen-model-1" in registry.list_models()
|
|
assert "zen-test1" in registry.list_aliases()
|
|
assert "zt1" in registry.list_aliases()
|
|
finally:
|
|
os.unlink(temp_path)
|
|
os.unlink(live_path)
|
|
|
|
def test_get_capabilities(self):
|
|
"""Test capability retrieval."""
|
|
registry = ZenModelRegistry()
|
|
|
|
# Test getting capabilities for a known model
|
|
caps = registry.get_capabilities("claude-sonnet-4-5")
|
|
assert caps is not None
|
|
assert caps.provider == ProviderType.ZEN
|
|
assert caps.model_name == "claude-sonnet-4-5"
|
|
assert caps.friendly_name == "OpenCode Zen (claude-sonnet-4-5)"
|
|
assert caps.context_window == 200000
|
|
assert caps.intelligence_score == 17
|
|
|
|
# Test getting capabilities for unknown model
|
|
caps = registry.get_capabilities("unknown-model")
|
|
assert caps is None
|
|
|
|
def test_resolve_model(self):
|
|
"""Test model resolution with aliases."""
|
|
registry = ZenModelRegistry()
|
|
|
|
# Test resolving a direct model name
|
|
config = registry.resolve("claude-sonnet-4-5")
|
|
assert config is not None
|
|
assert config.model_name == "claude-sonnet-4-5"
|
|
|
|
# Test resolving an alias
|
|
config = registry.resolve("zen-sonnet")
|
|
assert config is not None
|
|
assert config.model_name == "claude-sonnet-4-5"
|
|
|
|
# Test resolving unknown model
|
|
config = registry.resolve("unknown-model")
|
|
assert config is None
|
|
|
|
def test_list_aliases(self):
|
|
"""Test alias listing."""
|
|
registry = ZenModelRegistry()
|
|
|
|
aliases = registry.list_aliases()
|
|
assert isinstance(aliases, list)
|
|
assert len(aliases) > 0
|
|
|
|
# Should include our configured aliases
|
|
assert "zen-sonnet" in aliases
|
|
assert "zen-codex" in aliases
|
|
assert "zen-gemini" in aliases
|
|
|
|
def test_environment_config_path(self):
|
|
"""Test registry respects environment variable for config path."""
|
|
config_data = {
|
|
"models": [
|
|
{
|
|
"model_name": "env/test-model",
|
|
"aliases": ["env-test"],
|
|
"context_window": 8192,
|
|
"max_output_tokens": 4096,
|
|
"intelligence_score": 10,
|
|
}
|
|
]
|
|
}
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
|
|
json.dump(config_data, f)
|
|
temp_path = f.name
|
|
|
|
try:
|
|
with patch.dict(os.environ, {"ZEN_MODELS_CONFIG_PATH": temp_path}):
|
|
registry = ZenModelRegistry()
|
|
assert "env/test-model" in registry.list_models()
|
|
assert "env-test" in registry.list_aliases()
|
|
finally:
|
|
os.unlink(temp_path)
|
|
|
|
def test_malformed_config(self):
|
|
"""Test registry handles malformed config gracefully."""
|
|
malformed_config = {
|
|
"models": [
|
|
{
|
|
"model_name": "test/bad-model",
|
|
# Missing required fields
|
|
}
|
|
]
|
|
}
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
|
|
json.dump(malformed_config, f)
|
|
temp_path = f.name
|
|
|
|
try:
|
|
registry = ZenModelRegistry(config_path=temp_path)
|
|
# Should still initialize but model may not load properly
|
|
# This tests error handling in config loading
|
|
registry.list_models() # Test that this doesn't crash
|
|
# May or may not include the malformed model depending on validation
|
|
finally:
|
|
os.unlink(temp_path)
|
|
|
|
def test_empty_config(self):
|
|
"""Test registry with empty config."""
|
|
empty_config = {"models": []}
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
|
|
json.dump(empty_config, f)
|
|
temp_path = f.name
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as live_file:
|
|
json.dump({"models": []}, live_file)
|
|
live_path = live_file.name
|
|
|
|
try:
|
|
registry = ZenModelRegistry(config_path=temp_path, live_config_path=live_path)
|
|
assert len(registry.list_models()) == 0
|
|
assert len(registry.list_aliases()) == 0
|
|
finally:
|
|
os.unlink(temp_path)
|
|
os.unlink(live_path)
|
|
|
|
def test_live_catalogue_adds_unsynced_model_ids(self):
|
|
curated_data = {
|
|
"models": [
|
|
{
|
|
"model_name": "gpt-5.1",
|
|
"aliases": ["zen-gpt5.1"],
|
|
"context_window": 400000,
|
|
"max_output_tokens": 64000,
|
|
"intelligence_score": 16,
|
|
}
|
|
]
|
|
}
|
|
live_data = {
|
|
"models": [
|
|
{
|
|
"model_name": "gpt-5.4",
|
|
"aliases": [],
|
|
"context_window": 1050000,
|
|
"max_output_tokens": 128000,
|
|
"supports_extended_thinking": True,
|
|
"supports_json_mode": True,
|
|
"supports_function_calling": True,
|
|
"supports_images": True,
|
|
"max_image_size_mb": 20.0,
|
|
"supports_temperature": True,
|
|
"temperature_constraint": "range",
|
|
"description": "Live-only GPT-5.4 entry",
|
|
"use_openai_response_api": True,
|
|
}
|
|
]
|
|
}
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as curated_file:
|
|
json.dump(curated_data, curated_file)
|
|
curated_path = curated_file.name
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as live_file:
|
|
json.dump(live_data, live_file)
|
|
live_path = live_file.name
|
|
|
|
try:
|
|
registry = ZenModelRegistry(config_path=curated_path, live_config_path=live_path)
|
|
assert "gpt-5.4" in registry.list_models()
|
|
caps = registry.resolve("gpt-5.4")
|
|
assert caps is not None
|
|
assert caps.description == "Live-only GPT-5.4 entry"
|
|
assert caps.use_openai_response_api is True
|
|
finally:
|
|
os.unlink(curated_path)
|
|
os.unlink(live_path)
|
|
|
|
def test_curated_manifest_overrides_live_metadata(self):
|
|
curated_data = {
|
|
"models": [
|
|
{
|
|
"model_name": "gpt-5.4",
|
|
"aliases": ["zen-gpt5.4"],
|
|
"context_window": 1050000,
|
|
"max_output_tokens": 128000,
|
|
"supports_extended_thinking": True,
|
|
"supports_json_mode": True,
|
|
"supports_function_calling": True,
|
|
"supports_images": True,
|
|
"max_image_size_mb": 20.0,
|
|
"supports_temperature": False,
|
|
"temperature_constraint": "fixed",
|
|
"description": "Curated override",
|
|
"intelligence_score": 19,
|
|
"allow_code_generation": True,
|
|
"use_openai_response_api": True,
|
|
}
|
|
]
|
|
}
|
|
live_data = {
|
|
"models": [
|
|
{
|
|
"model_name": "gpt-5.4",
|
|
"aliases": [],
|
|
"context_window": 400000,
|
|
"max_output_tokens": 64000,
|
|
"supports_extended_thinking": False,
|
|
"supports_json_mode": True,
|
|
"supports_function_calling": True,
|
|
"supports_images": False,
|
|
"max_image_size_mb": 0.0,
|
|
"supports_temperature": True,
|
|
"temperature_constraint": "range",
|
|
"description": "Live baseline",
|
|
"intelligence_score": 10,
|
|
"allow_code_generation": False,
|
|
}
|
|
]
|
|
}
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as curated_file:
|
|
json.dump(curated_data, curated_file)
|
|
curated_path = curated_file.name
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as live_file:
|
|
json.dump(live_data, live_file)
|
|
live_path = live_file.name
|
|
|
|
try:
|
|
registry = ZenModelRegistry(config_path=curated_path, live_config_path=live_path)
|
|
caps = registry.resolve("zen-gpt5.4")
|
|
assert caps is not None
|
|
assert caps.model_name == "gpt-5.4"
|
|
assert caps.description == "Curated override"
|
|
assert caps.context_window == 1050000
|
|
assert caps.max_output_tokens == 128000
|
|
assert caps.supports_images is True
|
|
assert caps.supports_temperature is False
|
|
assert caps.allow_code_generation is True
|
|
assert caps.use_openai_response_api is True
|
|
finally:
|
|
os.unlink(curated_path)
|
|
os.unlink(live_path)
|
|
|
|
def test_missing_live_catalogue_keeps_curated_models_working(self, monkeypatch):
|
|
missing_live_path = Path(tempfile.gettempdir()) / "pal-missing-zen-live.json"
|
|
if missing_live_path.exists():
|
|
missing_live_path.unlink()
|
|
|
|
monkeypatch.setenv("ZEN_LIVE_MODELS_CONFIG_PATH", str(missing_live_path))
|
|
|
|
registry = ZenModelRegistry()
|
|
assert "gpt-5.1" in registry.list_models()
|
|
assert registry.resolve("zen-gpt5.1") is not None
|
|
|
|
def test_invalid_live_json_keeps_curated_models_working(self):
|
|
curated_data = {
|
|
"models": [
|
|
{
|
|
"model_name": "gpt-5.1",
|
|
"aliases": ["zen-gpt5.1"],
|
|
"context_window": 400000,
|
|
"max_output_tokens": 64000,
|
|
"intelligence_score": 16,
|
|
}
|
|
]
|
|
}
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as curated_file:
|
|
json.dump(curated_data, curated_file)
|
|
curated_path = curated_file.name
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as live_file:
|
|
live_file.write("{ invalid json }")
|
|
live_path = live_file.name
|
|
|
|
try:
|
|
registry = ZenModelRegistry(config_path=curated_path, live_config_path=live_path)
|
|
assert "gpt-5.1" in registry.list_models()
|
|
assert registry.resolve("zen-gpt5.1") is not None
|
|
finally:
|
|
os.unlink(curated_path)
|
|
os.unlink(live_path)
|