torbjorn/my-pal-mcp-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Quick test mode for simulation tests

Browse Source 
Fixed o4-mini name, OpenAI removed o4-mini-high
Add max_output_tokens property to ModelCapabilities
This commit is contained in:
Fahad
2025-06-23 18:33:47 +04:00
parent 8c1814d4eb
commit ce6c1fd7ea
35 changed files with 137 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/mock_helpers.py
View File

@@ -15,6 +15,7 @@ def create_mock_provider(model_name="gemini-2.5-flash", context_window=1_048_576
model_name=model_name,
friendly_name="Gemini",
context_window=context_window,
max_output_tokens=8192,
supports_extended_thinking=False,
supports_system_prompts=True,
supports_streaming=True,

19
tests/test_alias_target_restrictions.py
View File

@@ -211,7 +211,7 @@ class TestAliasTargetRestrictions:
# Verify the polymorphic method was called
mock_provider.list_all_known_models.assert_called_once()
@patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o4-mini-high"}) # Restrict to specific model
@patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o4-mini"}) # Restrict to specific model
def test_complex_alias_chains_handled_correctly(self):
"""Test that complex alias chains are handled correctly in restrictions."""
# Clear cached restriction service
@@ -221,12 +221,11 @@ class TestAliasTargetRestrictions:
provider = OpenAIModelProvider(api_key="test-key")
# Only o4-mini-high should be allowed
assert provider.validate_model_name("o4-mini-high")
# Only o4-mini should be allowed
assert provider.validate_model_name("o4-mini")
# Other models should be blocked
assert not provider.validate_model_name("o4-mini")
assert not provider.validate_model_name("mini") # This resolves to o4-mini
assert not provider.validate_model_name("o3")
assert not provider.validate_model_name("o3-mini")
def test_critical_regression_validation_sees_alias_targets(self):
@@ -307,7 +306,7 @@ class TestAliasTargetRestrictions:
it appear that target-based restrictions don't work.
"""
# Test with a made-up restriction scenario
with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o4-mini-high,o3-mini"}):
with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o4-mini,o3-mini"}):
# Clear cached restriction service
import utils.model_restrictions
@@ -318,7 +317,7 @@ class TestAliasTargetRestrictions:
# These specific target models should be recognized as valid
all_known = provider.list_all_known_models()
assert "o4-mini-high" in all_known, "Target model o4-mini-high should be known"
assert "o4-mini" in all_known, "Target model o4-mini should be known"
assert "o3-mini" in all_known, "Target model o3-mini should be known"
# Validation should not warn about these being unrecognized
@@ -329,11 +328,11 @@ class TestAliasTargetRestrictions:
# Should not warn about our allowed models being unrecognized
all_warnings = [str(call) for call in mock_logger.warning.call_args_list]
for warning in all_warnings:
assert "o4-mini-high" not in warning or "not a recognized" not in warning
assert "o4-mini" not in warning or "not a recognized" not in warning
assert "o3-mini" not in warning or "not a recognized" not in warning
# The restriction should actually work
assert provider.validate_model_name("o4-mini-high")
assert provider.validate_model_name("o4-mini")
assert provider.validate_model_name("o3-mini")
assert not provider.validate_model_name("o4-mini") # not in allowed list
assert not provider.validate_model_name("o3-pro") # not in allowed list
assert not provider.validate_model_name("o3") # not in allowed list

2
tests/test_auto_mode.py
View File

@@ -64,7 +64,7 @@ class TestAutoMode:
models_with_descriptions[model_name] = description
# Check all expected models are present with meaningful descriptions
expected_models = ["flash", "pro", "o3", "o3-mini", "o3-pro", "o4-mini", "o4-mini-high"]
expected_models = ["flash", "pro", "o3", "o3-mini", "o3-pro", "o4-mini"]
for model in expected_models:
# Model should exist somewhere in the providers
# Note: Some models might not be available if API keys aren't configured

16
tests/test_buggy_behavior_prevention.py
View File

@@ -118,7 +118,7 @@ class TestBuggyBehaviorPrevention:
provider = OpenAIModelProvider(api_key="test-key")
# Simulate a scenario where admin wants to restrict specific targets
with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o3-mini,o4-mini-high"}):
with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o3-mini,o4-mini"}):
# Clear cached restriction service
import utils.model_restrictions
@@ -126,19 +126,21 @@ class TestBuggyBehaviorPrevention:
# These should work because they're explicitly allowed
assert provider.validate_model_name("o3-mini")
assert provider.validate_model_name("o4-mini-high")
assert provider.validate_model_name("o4-mini")
# These should be blocked
assert not provider.validate_model_name("o4-mini") # Not in allowed list
assert not provider.validate_model_name("o3-pro") # Not in allowed list
assert not provider.validate_model_name("o3") # Not in allowed list
assert not provider.validate_model_name("mini") # Resolves to o4-mini, not allowed
# This should be ALLOWED because it resolves to o4-mini which is in the allowed list
assert provider.validate_model_name("mini") # Resolves to o4-mini, which IS allowed
# Verify our list_all_known_models includes the restricted models
all_known = provider.list_all_known_models()
assert "o3-mini" in all_known # Should be known (and allowed)
assert "o4-mini-high" in all_known # Should be known (and allowed)
assert "o4-mini" in all_known # Should be known (but blocked)
assert "mini" in all_known # Should be known (but blocked)
assert "o4-mini" in all_known # Should be known (and allowed)
assert "o3-pro" in all_known # Should be known (but blocked)
assert "mini" in all_known # Should be known (and allowed since it resolves to o4-mini)
def test_demonstration_of_old_vs_new_interface(self):
"""

2
tests/test_model_enumeration.py
View File

@@ -149,7 +149,7 @@ class TestModelEnumeration:
("o3", False), # OpenAI - not available without API key
("grok", False), # X.AI - not available without API key
("gemini-2.5-flash", False), # Full Gemini name - not available without API key
("o4-mini-high", False), # OpenAI variant - not available without API key
("o4-mini", False), # OpenAI variant - not available without API key
("grok-3-fast", False), # X.AI variant - not available without API key
],
)

4
tests/test_model_restrictions.py
View File

@@ -93,7 +93,7 @@ class TestModelRestrictionService:
with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o3-mini,o4-mini"}):
service = ModelRestrictionService()
models = ["o3", "o3-mini", "o4-mini", "o4-mini-high"]
models = ["o3", "o3-mini", "o4-mini", "o3-pro"]
filtered = service.filter_models(ProviderType.OPENAI, models)
assert filtered == ["o3-mini", "o4-mini"]
@@ -573,7 +573,7 @@ class TestShorthandRestrictions:
# Other models should not work
assert not openai_provider.validate_model_name("o3")
assert not openai_provider.validate_model_name("o4-mini-high")
assert not openai_provider.validate_model_name("o3-pro")
@patch.dict(
os.environ,

2
tests/test_o3_temperature_fix_simple.py
View File

@@ -185,7 +185,7 @@ class TestO3TemperatureParameterFixSimple:
provider = OpenAIModelProvider(api_key="test-key")
# Test O3/O4 models that should NOT support temperature parameter
o3_o4_models = ["o3", "o3-mini", "o3-pro", "o4-mini", "o4-mini-high"]
o3_o4_models = ["o3", "o3-mini", "o3-pro", "o4-mini"]
for model in o3_o4_models:
capabilities = provider.get_capabilities(model)

18
tests/test_openai_provider.py
View File

@@ -47,14 +47,13 @@ class TestOpenAIProvider:
assert provider.validate_model_name("o3-mini") is True
assert provider.validate_model_name("o3-pro") is True
assert provider.validate_model_name("o4-mini") is True
assert provider.validate_model_name("o4-mini-high") is True
assert provider.validate_model_name("o4-mini") is True
# Test valid aliases
assert provider.validate_model_name("mini") is True
assert provider.validate_model_name("o3mini") is True
assert provider.validate_model_name("o4mini") is True
assert provider.validate_model_name("o4minihigh") is True
assert provider.validate_model_name("o4minihi") is True
assert provider.validate_model_name("o4mini") is True
# Test invalid model
assert provider.validate_model_name("invalid-model") is False
@@ -69,15 +68,14 @@ class TestOpenAIProvider:
assert provider._resolve_model_name("mini") == "o4-mini"
assert provider._resolve_model_name("o3mini") == "o3-mini"
assert provider._resolve_model_name("o4mini") == "o4-mini"
assert provider._resolve_model_name("o4minihigh") == "o4-mini-high"
assert provider._resolve_model_name("o4minihi") == "o4-mini-high"
assert provider._resolve_model_name("o4mini") == "o4-mini"
# Test full name passthrough
assert provider._resolve_model_name("o3") == "o3"
assert provider._resolve_model_name("o3-mini") == "o3-mini"
assert provider._resolve_model_name("o3-pro") == "o3-pro-2025-06-10"
assert provider._resolve_model_name("o4-mini") == "o4-mini"
assert provider._resolve_model_name("o4-mini-high") == "o4-mini-high"
assert provider._resolve_model_name("o4-mini") == "o4-mini"
def test_get_capabilities_o3(self):
"""Test getting model capabilities for O3."""
@@ -184,11 +182,11 @@ class TestOpenAIProvider:
call_kwargs = mock_client.chat.completions.create.call_args[1]
assert call_kwargs["model"] == "o3-mini"
# Test o4minihigh -> o4-mini-high
mock_response.model = "o4-mini-high"
provider.generate_content(prompt="Test", model_name="o4minihigh", temperature=1.0)
# Test o4mini -> o4-mini
mock_response.model = "o4-mini"
provider.generate_content(prompt="Test", model_name="o4mini", temperature=1.0)
call_kwargs = mock_client.chat.completions.create.call_args[1]
assert call_kwargs["model"] == "o4-mini-high"
assert call_kwargs["model"] == "o4-mini"
@patch("providers.openai_compatible.OpenAI")
def test_generate_content_no_alias_passthrough(self, mock_openai_class):

2
tests/test_openrouter_provider.py
View File

@@ -77,7 +77,7 @@ class TestOpenRouterProvider:
assert provider._resolve_model_name("o3-mini") == "openai/o3-mini"
assert provider._resolve_model_name("o3mini") == "openai/o3-mini"
assert provider._resolve_model_name("o4-mini") == "openai/o4-mini"
assert provider._resolve_model_name("o4-mini-high") == "openai/o4-mini-high"
assert provider._resolve_model_name("o4-mini") == "openai/o4-mini"
assert provider._resolve_model_name("claude") == "anthropic/claude-sonnet-4"
assert provider._resolve_model_name("mistral") == "mistralai/mistral-large-2411"
assert provider._resolve_model_name("deepseek") == "deepseek/deepseek-r1-0528"

21
tests/test_openrouter_registry.py
View File

@@ -24,7 +24,16 @@ class TestOpenRouterModelRegistry:
def test_custom_config_path(self):
"""Test registry with custom config path."""
# Create temporary config
config_data = {"models": [{"model_name": "test/model-1", "aliases": ["test1", "t1"], "context_window": 4096}]}
config_data = {
"models": [
{
"model_name": "test/model-1",
"aliases": ["test1", "t1"],
"context_window": 4096,
"max_output_tokens": 2048,
}
]
}
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
json.dump(config_data, f)
@@ -42,7 +51,11 @@ class TestOpenRouterModelRegistry:
def test_environment_variable_override(self):
"""Test OPENROUTER_MODELS_PATH environment variable."""
# Create custom config
config_data = {"models": [{"model_name": "env/model", "aliases": ["envtest"], "context_window": 8192}]}
config_data = {
"models": [
{"model_name": "env/model", "aliases": ["envtest"], "context_window": 8192, "max_output_tokens": 4096}
]
}
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
json.dump(config_data, f)
@@ -127,11 +140,12 @@ class TestOpenRouterModelRegistry:
"""Test that duplicate aliases are detected."""
config_data = {
"models": [
{"model_name": "test/model-1", "aliases": ["dupe"], "context_window": 4096},
{"model_name": "test/model-1", "aliases": ["dupe"], "context_window": 4096, "max_output_tokens": 2048},
{
"model_name": "test/model-2",
"aliases": ["DUPE"], # Same alias, different case
"context_window": 8192,
"max_output_tokens": 2048,
},
]
}
@@ -207,6 +221,7 @@ class TestOpenRouterModelRegistry:
friendly_name="OpenRouter (test/full-featured)",
aliases=["full"],
context_window=128000,
max_output_tokens=8192,
supports_extended_thinking=True,
supports_system_prompts=True,
supports_streaming=True,

6
tests/test_providers.py
View File

@@ -215,9 +215,7 @@ class TestOpenAIProvider:
assert provider.validate_model_name("o3-mini") # Backwards compatibility
assert provider.validate_model_name("o4-mini")
assert provider.validate_model_name("o4mini")
assert provider.validate_model_name("o4-mini-high")
assert provider.validate_model_name("o4minihigh")
assert provider.validate_model_name("o4minihi")
assert provider.validate_model_name("o4-mini")
assert not provider.validate_model_name("gpt-4o")
assert not provider.validate_model_name("invalid-model")
@@ -229,4 +227,4 @@ class TestOpenAIProvider:
assert not provider.supports_thinking_mode("o3mini")
assert not provider.supports_thinking_mode("o3-mini")
assert not provider.supports_thinking_mode("o4-mini")
assert not provider.supports_thinking_mode("o4-mini-high")
assert not provider.supports_thinking_mode("o4-mini")

5
tests/test_supported_models_aliases.py
View File

@@ -51,15 +51,14 @@ class TestSupportedModelsAliases:
assert "o4mini" in provider.SUPPORTED_MODELS["o4-mini"].aliases
assert "o3mini" in provider.SUPPORTED_MODELS["o3-mini"].aliases
assert "o3-pro" in provider.SUPPORTED_MODELS["o3-pro-2025-06-10"].aliases
assert "o4minihigh" in provider.SUPPORTED_MODELS["o4-mini-high"].aliases
assert "o4minihi" in provider.SUPPORTED_MODELS["o4-mini-high"].aliases
assert "o4mini" in provider.SUPPORTED_MODELS["o4-mini"].aliases
assert "gpt4.1" in provider.SUPPORTED_MODELS["gpt-4.1-2025-04-14"].aliases
# Test alias resolution
assert provider._resolve_model_name("mini") == "o4-mini"
assert provider._resolve_model_name("o3mini") == "o3-mini"
assert provider._resolve_model_name("o3-pro") == "o3-pro-2025-06-10"
assert provider._resolve_model_name("o4minihigh") == "o4-mini-high"
assert provider._resolve_model_name("o4mini") == "o4-mini"
assert provider._resolve_model_name("gpt4.1") == "gpt-4.1-2025-04-14"
# Test case insensitive resolution