torbjorn/my-pal-mcp-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GPT-5, GPT-5-mini support

Browse Source 
Improvements to model name resolution
Improved instructions for multi-step workflows when continuation is available
Improved instructions for chat tool
Improved preferred model resolution, moved code from registry -> each provider
Updated tests
This commit is contained in:
Fahad
2025-08-08 08:51:34 +05:00
parent 9a4791cb06
commit 1a8ec2e12f
30 changed files with 792 additions and 483 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
tests/test_openai_provider.py
View File

@@ -48,12 +48,17 @@ class TestOpenAIProvider:
assert provider.validate_model_name("o3-pro") is True
assert provider.validate_model_name("o4-mini") is True
assert provider.validate_model_name("o4-mini") is True
assert provider.validate_model_name("gpt-5") is True
assert provider.validate_model_name("gpt-5-mini") is True
# Test valid aliases
assert provider.validate_model_name("mini") is True
assert provider.validate_model_name("o3mini") is True
assert provider.validate_model_name("o4mini") is True
assert provider.validate_model_name("o4mini") is True
assert provider.validate_model_name("gpt5") is True
assert provider.validate_model_name("gpt5-mini") is True
assert provider.validate_model_name("gpt5mini") is True
# Test invalid model
assert provider.validate_model_name("invalid-model") is False
@@ -65,17 +70,22 @@ class TestOpenAIProvider:
provider = OpenAIModelProvider("test-key")
# Test shorthand resolution
assert provider._resolve_model_name("mini") == "o4-mini"
assert provider._resolve_model_name("mini") == "gpt-5-mini" # "mini" now resolves to gpt-5-mini
assert provider._resolve_model_name("o3mini") == "o3-mini"
assert provider._resolve_model_name("o4mini") == "o4-mini"
assert provider._resolve_model_name("o4mini") == "o4-mini"
assert provider._resolve_model_name("gpt5") == "gpt-5"
assert provider._resolve_model_name("gpt5-mini") == "gpt-5-mini"
assert provider._resolve_model_name("gpt5mini") == "gpt-5-mini"
# Test full name passthrough
assert provider._resolve_model_name("o3") == "o3"
assert provider._resolve_model_name("o3-mini") == "o3-mini"
assert provider._resolve_model_name("o3-pro") == "o3-pro-2025-06-10"
assert provider._resolve_model_name("o3-pro") == "o3-pro"
assert provider._resolve_model_name("o4-mini") == "o4-mini"
assert provider._resolve_model_name("o4-mini") == "o4-mini"
assert provider._resolve_model_name("gpt-5") == "gpt-5"
assert provider._resolve_model_name("gpt-5-mini") == "gpt-5-mini"
def test_get_capabilities_o3(self):
"""Test getting model capabilities for O3."""
@@ -99,11 +109,43 @@ class TestOpenAIProvider:
provider = OpenAIModelProvider("test-key")
capabilities = provider.get_capabilities("mini")
assert capabilities.model_name == "o4-mini" # Capabilities should show resolved model name
assert capabilities.friendly_name == "OpenAI (O4-mini)"
assert capabilities.context_window == 200_000
assert capabilities.model_name == "gpt-5-mini" # "mini" now resolves to gpt-5-mini
assert capabilities.friendly_name == "OpenAI (GPT-5-mini)"
assert capabilities.context_window == 400_000
assert capabilities.provider == ProviderType.OPENAI
def test_get_capabilities_gpt5(self):
"""Test getting model capabilities for GPT-5."""
provider = OpenAIModelProvider("test-key")
capabilities = provider.get_capabilities("gpt-5")
assert capabilities.model_name == "gpt-5"
assert capabilities.friendly_name == "OpenAI (GPT-5)"
assert capabilities.context_window == 400_000
assert capabilities.max_output_tokens == 128_000
assert capabilities.provider == ProviderType.OPENAI
assert capabilities.supports_extended_thinking is True
assert capabilities.supports_system_prompts is True
assert capabilities.supports_streaming is True
assert capabilities.supports_function_calling is True
assert capabilities.supports_temperature is True
def test_get_capabilities_gpt5_mini(self):
"""Test getting model capabilities for GPT-5-mini."""
provider = OpenAIModelProvider("test-key")
capabilities = provider.get_capabilities("gpt-5-mini")
assert capabilities.model_name == "gpt-5-mini"
assert capabilities.friendly_name == "OpenAI (GPT-5-mini)"
assert capabilities.context_window == 400_000
assert capabilities.max_output_tokens == 128_000
assert capabilities.provider == ProviderType.OPENAI
assert capabilities.supports_extended_thinking is True
assert capabilities.supports_system_prompts is True
assert capabilities.supports_streaming is True
assert capabilities.supports_function_calling is True
assert capabilities.supports_temperature is True
@patch("providers.openai_compatible.OpenAI")
def test_generate_content_resolves_alias_before_api_call(self, mock_openai_class):
"""Test that generate_content resolves aliases before making API calls.
@@ -132,21 +174,19 @@ class TestOpenAIProvider:
provider = OpenAIModelProvider("test-key")
# Call generate_content with alias 'gpt4.1' (resolves to gpt-4.1-2025-04-14, supports temperature)
# Call generate_content with alias 'gpt4.1' (resolves to gpt-4.1, supports temperature)
result = provider.generate_content(
prompt="Test prompt",
model_name="gpt4.1",
temperature=1.0, # This should be resolved to "gpt-4.1-2025-04-14"
temperature=1.0, # This should be resolved to "gpt-4.1"
)
# Verify the API was called with the RESOLVED model name
mock_client.chat.completions.create.assert_called_once()
call_kwargs = mock_client.chat.completions.create.call_args[1]
# CRITICAL ASSERTION: The API should receive "gpt-4.1-2025-04-14", not "gpt4.1"
assert (
call_kwargs["model"] == "gpt-4.1-2025-04-14"
), f"Expected 'gpt-4.1-2025-04-14' but API received '{call_kwargs['model']}'"
# CRITICAL ASSERTION: The API should receive "gpt-4.1", not "gpt4.1"
assert call_kwargs["model"] == "gpt-4.1", f"Expected 'gpt-4.1' but API received '{call_kwargs['model']}'"
# Verify other parameters (gpt-4.1 supports temperature unlike O3/O4 models)
assert call_kwargs["temperature"] == 1.0
@@ -156,7 +196,7 @@ class TestOpenAIProvider:
# Verify response
assert result.content == "Test response"
assert result.model_name == "gpt-4.1-2025-04-14" # Should be the resolved name
assert result.model_name == "gpt-4.1" # Should be the resolved name
@patch("providers.openai_compatible.OpenAI")
def test_generate_content_other_aliases(self, mock_openai_class):
@@ -213,14 +253,22 @@ class TestOpenAIProvider:
assert call_kwargs["model"] == "o3-mini" # Should be unchanged
def test_supports_thinking_mode(self):
"""Test thinking mode support (currently False for all OpenAI models)."""
"""Test thinking mode support."""
provider = OpenAIModelProvider("test-key")
# All OpenAI models currently don't support thinking mode
# GPT-5 models support thinking mode (reasoning tokens)
assert provider.supports_thinking_mode("gpt-5") is True
assert provider.supports_thinking_mode("gpt-5-mini") is True
assert provider.supports_thinking_mode("gpt5") is True # Test with alias
assert provider.supports_thinking_mode("gpt5mini") is True # Test with alias
# O3/O4 models don't support thinking mode
assert provider.supports_thinking_mode("o3") is False
assert provider.supports_thinking_mode("o3-mini") is False
assert provider.supports_thinking_mode("o4-mini") is False
assert provider.supports_thinking_mode("mini") is False # Test with alias too
assert (
provider.supports_thinking_mode("mini") is True
) # "mini" now resolves to gpt-5-mini which supports thinking
@patch("providers.openai_compatible.OpenAI")
def test_o3_pro_routes_to_responses_endpoint(self, mock_openai_class):
@@ -234,7 +282,7 @@ class TestOpenAIProvider:
mock_response.output.content = [MagicMock()]
mock_response.output.content[0].type = "output_text"
mock_response.output.content[0].text = "4"
mock_response.model = "o3-pro-2025-06-10"
mock_response.model = "o3-pro"
mock_response.id = "test-id"
mock_response.created_at = 1234567890
mock_response.usage = MagicMock()
@@ -252,13 +300,13 @@ class TestOpenAIProvider:
# Verify responses.create was called
mock_client.responses.create.assert_called_once()
call_args = mock_client.responses.create.call_args[1]
assert call_args["model"] == "o3-pro-2025-06-10"
assert call_args["model"] == "o3-pro"
assert call_args["input"][0]["role"] == "user"
assert "What is 2 + 2?" in call_args["input"][0]["content"][0]["text"]
# Verify the response
assert result.content == "4"
assert result.model_name == "o3-pro-2025-06-10"
assert result.model_name == "o3-pro"
assert result.metadata["endpoint"] == "responses"
@patch("providers.openai_compatible.OpenAI")