Fix for: https://github.com/BeehiveInnovations/zen-mcp-server/issues/102 - Removed centralized MODEL_CAPABILITIES_DESC from config.py - Added model descriptions to individual provider SUPPORTED_MODELS - Updated _get_available_models() to use ModelProviderRegistry for API key filtering - Added comprehensive test suite validating bug reproduction and fix
This commit is contained in:
Fahad
@@ -76,33 +76,26 @@ class TestModelEnumeration:
|
||||
|
||||
importlib.reload(tools.base)
|
||||
|
||||
def test_native_models_always_included(self):
|
||||
"""Test that native models from MODEL_CAPABILITIES_DESC are always included."""
|
||||
def test_no_models_when_no_providers_configured(self):
|
||||
"""Test that no native models are included when no providers are configured."""
|
||||
self._setup_environment({}) # No providers configured
|
||||
|
||||
tool = AnalyzeTool()
|
||||
models = tool._get_available_models()
|
||||
|
||||
# All native models should be present
|
||||
native_models = [
|
||||
"flash",
|
||||
"pro", # Gemini aliases
|
||||
"o3",
|
||||
"o3-mini",
|
||||
"o3-pro",
|
||||
"o4-mini",
|
||||
"o4-mini-high", # OpenAI models
|
||||
"grok",
|
||||
"grok-3",
|
||||
"grok-3-fast",
|
||||
"grok3",
|
||||
"grokfast", # X.AI models
|
||||
"gemini-2.5-flash",
|
||||
"gemini-2.5-pro", # Full Gemini names
|
||||
# After the fix, models should only be shown from enabled providers
|
||||
# With no API keys configured, no providers should be enabled
|
||||
# Only OpenRouter aliases might still appear if they're in the registry
|
||||
|
||||
# Filter out OpenRouter aliases that might still appear
|
||||
non_openrouter_models = [
|
||||
m for m in models if "/" not in m and m not in ["gemini", "pro", "flash", "opus", "sonnet", "haiku"]
|
||||
]
|
||||
|
||||
for model in native_models:
|
||||
assert model in models, f"Native model {model} should always be in enum"
|
||||
# No native provider models should be present without API keys
|
||||
assert (
|
||||
len(non_openrouter_models) == 0
|
||||
), f"No native models should be available without API keys, but found: {non_openrouter_models}"
|
||||
|
||||
@pytest.mark.skip(reason="Complex integration test - rely on simulator tests for provider testing")
|
||||
def test_openrouter_models_with_api_key(self):
|
||||
@@ -179,116 +172,36 @@ class TestModelEnumeration:
|
||||
@pytest.mark.parametrize(
|
||||
"model_name,should_exist",
|
||||
[
|
||||
("flash", True), # Native Gemini
|
||||
("o3", True), # Native OpenAI
|
||||
("grok", True), # Native X.AI
|
||||
("gemini-2.5-flash", True), # Full native name
|
||||
("o4-mini-high", True), # Native OpenAI variant
|
||||
("grok-3-fast", True), # Native X.AI variant
|
||||
("flash", False), # Gemini - not available without API key
|
||||
("o3", False), # OpenAI - not available without API key
|
||||
("grok", False), # X.AI - not available without API key
|
||||
("gemini-2.5-flash", False), # Full Gemini name - not available without API key
|
||||
("o4-mini-high", False), # OpenAI variant - not available without API key
|
||||
("grok-3-fast", False), # X.AI variant - not available without API key
|
||||
],
|
||||
)
|
||||
def test_specific_native_models_always_present(self, model_name, should_exist):
|
||||
"""Test that specific native models are always present regardless of configuration."""
|
||||
def test_specific_native_models_only_with_api_keys(self, model_name, should_exist):
|
||||
"""Test that native models are only present when their provider has API keys configured."""
|
||||
self._setup_environment({}) # No providers
|
||||
|
||||
tool = AnalyzeTool()
|
||||
models = tool._get_available_models()
|
||||
|
||||
if should_exist:
|
||||
assert model_name in models, f"Native model {model_name} should always be present"
|
||||
assert model_name in models, f"Model {model_name} should be present"
|
||||
else:
|
||||
assert model_name not in models, f"Model {model_name} should not be present"
|
||||
assert model_name not in models, f"Native model {model_name} should not be present without API key"
|
||||
|
||||
def test_auto_mode_behavior_with_environment_variables(self):
|
||||
"""Test auto mode behavior with various environment variable combinations."""
|
||||
|
||||
# Test different environment scenarios for auto mode
|
||||
test_scenarios = [
|
||||
{"name": "no_providers", "env": {}, "expected_behavior": "should_include_native_only"},
|
||||
{
|
||||
"name": "gemini_only",
|
||||
"env": {"GEMINI_API_KEY": "test-key"},
|
||||
"expected_behavior": "should_include_gemini_models",
|
||||
},
|
||||
{
|
||||
"name": "openai_only",
|
||||
"env": {"OPENAI_API_KEY": "test-key"},
|
||||
"expected_behavior": "should_include_openai_models",
|
||||
},
|
||||
{"name": "xai_only", "env": {"XAI_API_KEY": "test-key"}, "expected_behavior": "should_include_xai_models"},
|
||||
{
|
||||
"name": "multiple_providers",
|
||||
"env": {"GEMINI_API_KEY": "test-key", "OPENAI_API_KEY": "test-key", "XAI_API_KEY": "test-key"},
|
||||
"expected_behavior": "should_include_all_native_models",
|
||||
},
|
||||
]
|
||||
# DELETED: test_auto_mode_behavior_with_environment_variables
|
||||
# This test was fundamentally broken due to registry corruption.
|
||||
# It cleared ModelProviderRegistry._instance without re-registering providers,
|
||||
# causing impossible test conditions (expecting models when no providers exist).
|
||||
# Functionality is already covered by test_auto_mode_comprehensive.py
|
||||
|
||||
for scenario in test_scenarios:
|
||||
# Test each scenario independently
|
||||
self._setup_environment(scenario["env"])
|
||||
|
||||
tool = AnalyzeTool()
|
||||
models = tool._get_available_models()
|
||||
|
||||
# Always expect native models regardless of configuration
|
||||
native_models = ["flash", "pro", "o3", "o3-mini", "grok"]
|
||||
for model in native_models:
|
||||
assert model in models, f"Native model {model} missing in {scenario['name']} scenario"
|
||||
|
||||
# Verify auto mode detection
|
||||
assert tool.is_effective_auto_mode(), f"Auto mode should be active in {scenario['name']} scenario"
|
||||
|
||||
# Verify model schema includes model field in auto mode
|
||||
schema = tool.get_input_schema()
|
||||
assert "model" in schema["required"], f"Model field should be required in auto mode for {scenario['name']}"
|
||||
assert "model" in schema["properties"], f"Model field should be in properties for {scenario['name']}"
|
||||
|
||||
# Verify enum contains expected models
|
||||
model_enum = schema["properties"]["model"]["enum"]
|
||||
for model in native_models:
|
||||
assert model in model_enum, f"Native model {model} should be in enum for {scenario['name']}"
|
||||
|
||||
def test_auto_mode_model_selection_validation(self):
|
||||
"""Test that auto mode properly validates model selection."""
|
||||
self._setup_environment({"DEFAULT_MODEL": "auto", "GEMINI_API_KEY": "test-key"})
|
||||
|
||||
tool = AnalyzeTool()
|
||||
|
||||
# Verify auto mode is active
|
||||
assert tool.is_effective_auto_mode()
|
||||
|
||||
# Test valid model selection
|
||||
available_models = tool._get_available_models()
|
||||
assert len(available_models) > 0, "Should have available models in auto mode"
|
||||
|
||||
# Test that model validation works
|
||||
schema = tool.get_input_schema()
|
||||
model_enum = schema["properties"]["model"]["enum"]
|
||||
|
||||
# All enum models should be in available models
|
||||
for enum_model in model_enum:
|
||||
assert enum_model in available_models, f"Enum model {enum_model} should be available"
|
||||
|
||||
# All available models should be in enum
|
||||
for available_model in available_models:
|
||||
assert available_model in model_enum, f"Available model {available_model} should be in enum"
|
||||
|
||||
def test_environment_variable_precedence(self):
|
||||
"""Test that environment variables are properly handled for model availability."""
|
||||
# Test that setting DEFAULT_MODEL to auto enables auto mode
|
||||
self._setup_environment({"DEFAULT_MODEL": "auto"})
|
||||
tool = AnalyzeTool()
|
||||
assert tool.is_effective_auto_mode(), "DEFAULT_MODEL=auto should enable auto mode"
|
||||
|
||||
# Test environment variable combinations with auto mode
|
||||
self._setup_environment({"DEFAULT_MODEL": "auto", "GEMINI_API_KEY": "test-key", "OPENAI_API_KEY": "test-key"})
|
||||
tool = AnalyzeTool()
|
||||
models = tool._get_available_models()
|
||||
|
||||
# Should include native models from providers that are theoretically configured
|
||||
native_models = ["flash", "pro", "o3", "o3-mini", "grok"]
|
||||
for model in native_models:
|
||||
assert model in models, f"Native model {model} should be available in auto mode"
|
||||
|
||||
# Verify auto mode is still active
|
||||
assert tool.is_effective_auto_mode(), "Auto mode should remain active with multiple providers"
|
||||
# DELETED: test_auto_mode_model_selection_validation
|
||||
# DELETED: test_environment_variable_precedence
|
||||
# Both tests suffered from the same registry corruption issue as the deleted test above.
|
||||
# They cleared ModelProviderRegistry._instance without re-registering providers,
|
||||
# causing empty model lists and impossible test conditions.
|
||||
# Auto mode functionality is already comprehensively tested in test_auto_mode_comprehensive.py
|
||||
|
||||
Reference in New Issue
Block a user