Files
my-pal-mcp-server/tests/test_prompt_regression.py
Fahad f44ca326ef Breaking change: openrouter_models.json -> custom_models.json
* Support for Custom URLs and custom models, including locally hosted models such as ollama
* Support for native + openrouter + local models (i.e. dozens of models) means you can start delegating sub-tasks to particular models or work to local models such as localizations or other boring work etc.
* Several tests added
* precommit to also include untracked (new) files
* Logfile auto rollover
* Improved logging
2025-06-13 15:22:09 +04:00

360 lines
15 KiB
Python

"""
Regression tests to ensure normal prompt handling still works after large prompt changes.
This test module verifies that all tools continue to work correctly with
normal-sized prompts after implementing the large prompt handling feature.
"""
import json
from unittest.mock import MagicMock, patch
import pytest
from tools.analyze import AnalyzeTool
from tools.chat import ChatTool
from tools.codereview import CodeReviewTool
from tools.debug import DebugIssueTool
from tools.precommit import Precommit
from tools.thinkdeep import ThinkDeepTool
class TestPromptRegression:
"""Regression test suite for normal prompt handling."""
@pytest.fixture
def mock_model_response(self):
"""Create a mock model response."""
from unittest.mock import Mock
def _create_response(text="Test response"):
# Return a Mock that acts like ModelResponse
return Mock(
content=text,
usage={"input_tokens": 10, "output_tokens": 20, "total_tokens": 30},
model_name="gemini-2.5-flash-preview-05-20",
metadata={"finish_reason": "STOP"},
)
return _create_response
@pytest.mark.asyncio
async def test_chat_normal_prompt(self, mock_model_response):
"""Test chat tool with normal prompt."""
tool = ChatTool()
with patch.object(tool, "get_model_provider") as mock_get_provider:
mock_provider = MagicMock()
mock_provider.get_provider_type.return_value = MagicMock(value="google")
mock_provider.supports_thinking_mode.return_value = False
mock_provider.generate_content.return_value = mock_model_response(
"This is a helpful response about Python."
)
mock_get_provider.return_value = mock_provider
result = await tool.execute({"prompt": "Explain Python decorators"})
assert len(result) == 1
output = json.loads(result[0].text)
assert output["status"] == "success"
assert "helpful response about Python" in output["content"]
# Verify provider was called
mock_provider.generate_content.assert_called_once()
@pytest.mark.asyncio
async def test_chat_with_files(self, mock_model_response):
"""Test chat tool with files parameter."""
tool = ChatTool()
with patch.object(tool, "get_model_provider") as mock_get_provider:
mock_provider = MagicMock()
mock_provider.get_provider_type.return_value = MagicMock(value="google")
mock_provider.supports_thinking_mode.return_value = False
mock_provider.generate_content.return_value = mock_model_response()
mock_get_provider.return_value = mock_provider
# Mock file reading through the centralized method
with patch.object(tool, "_prepare_file_content_for_prompt") as mock_prepare_files:
mock_prepare_files.return_value = "File content here"
result = await tool.execute({"prompt": "Analyze this code", "files": ["/path/to/file.py"]})
assert len(result) == 1
output = json.loads(result[0].text)
assert output["status"] == "success"
mock_prepare_files.assert_called_once_with(["/path/to/file.py"], None, "Context files")
@pytest.mark.asyncio
async def test_thinkdeep_normal_analysis(self, mock_model_response):
"""Test thinkdeep tool with normal analysis."""
tool = ThinkDeepTool()
with patch.object(tool, "get_model_provider") as mock_get_provider:
mock_provider = MagicMock()
mock_provider.get_provider_type.return_value = MagicMock(value="google")
mock_provider.supports_thinking_mode.return_value = False
mock_provider.generate_content.return_value = mock_model_response(
"Here's a deeper analysis with edge cases..."
)
mock_get_provider.return_value = mock_provider
result = await tool.execute(
{
"prompt": "I think we should use a cache for performance",
"problem_context": "Building a high-traffic API",
"focus_areas": ["scalability", "reliability"],
}
)
assert len(result) == 1
output = json.loads(result[0].text)
assert output["status"] == "success"
assert "Critical Evaluation Required" in output["content"]
assert "deeper analysis" in output["content"]
@pytest.mark.asyncio
async def test_codereview_normal_review(self, mock_model_response):
"""Test codereview tool with normal inputs."""
tool = CodeReviewTool()
with patch.object(tool, "get_model_provider") as mock_get_provider:
mock_provider = MagicMock()
mock_provider.get_provider_type.return_value = MagicMock(value="google")
mock_provider.supports_thinking_mode.return_value = False
mock_provider.generate_content.return_value = mock_model_response(
"Found 3 issues: 1) Missing error handling..."
)
mock_get_provider.return_value = mock_provider
# Mock file reading
with patch("tools.base.read_files") as mock_read_files:
mock_read_files.return_value = "def main(): pass"
result = await tool.execute(
{
"files": ["/path/to/code.py"],
"review_type": "security",
"focus_on": "Look for SQL injection vulnerabilities",
"prompt": "Test code review for validation purposes",
}
)
assert len(result) == 1
output = json.loads(result[0].text)
assert output["status"] == "success"
assert "Found 3 issues" in output["content"]
@pytest.mark.asyncio
async def test_review_changes_normal_request(self, mock_model_response):
"""Test review_changes tool with normal original_request."""
tool = Precommit()
with patch.object(tool, "get_model_provider") as mock_get_provider:
mock_provider = MagicMock()
mock_provider.get_provider_type.return_value = MagicMock(value="google")
mock_provider.supports_thinking_mode.return_value = False
mock_provider.generate_content.return_value = mock_model_response(
"Changes look good, implementing feature as requested..."
)
mock_get_provider.return_value = mock_provider
# Mock git operations
with patch("tools.precommit.find_git_repositories") as mock_find_repos:
with patch("tools.precommit.get_git_status") as mock_git_status:
mock_find_repos.return_value = ["/path/to/repo"]
mock_git_status.return_value = {
"branch": "main",
"ahead": 0,
"behind": 0,
"staged_files": ["file.py"],
"unstaged_files": [],
"untracked_files": [],
}
result = await tool.execute(
{
"path": "/path/to/repo",
"prompt": "Add user authentication feature with JWT tokens",
}
)
assert len(result) == 1
output = json.loads(result[0].text)
assert output["status"] == "success"
@pytest.mark.asyncio
async def test_debug_normal_error(self, mock_model_response):
"""Test debug tool with normal error description."""
tool = DebugIssueTool()
with patch.object(tool, "get_model_provider") as mock_get_provider:
mock_provider = MagicMock()
mock_provider.get_provider_type.return_value = MagicMock(value="google")
mock_provider.supports_thinking_mode.return_value = False
mock_provider.generate_content.return_value = mock_model_response(
"Root cause: The variable is undefined. Fix: Initialize it..."
)
mock_get_provider.return_value = mock_provider
result = await tool.execute(
{
"prompt": "TypeError: Cannot read property 'name' of undefined",
"error_context": "at line 42 in user.js\n console.log(user.name)",
"runtime_info": "Node.js v16.14.0",
}
)
assert len(result) == 1
output = json.loads(result[0].text)
assert output["status"] == "success"
assert "Next Steps:" in output["content"]
assert "Root cause" in output["content"]
@pytest.mark.asyncio
async def test_analyze_normal_question(self, mock_model_response):
"""Test analyze tool with normal question."""
tool = AnalyzeTool()
with patch.object(tool, "get_model_provider") as mock_get_provider:
mock_provider = MagicMock()
mock_provider.get_provider_type.return_value = MagicMock(value="google")
mock_provider.supports_thinking_mode.return_value = False
mock_provider.generate_content.return_value = mock_model_response(
"The code follows MVC pattern with clear separation..."
)
mock_get_provider.return_value = mock_provider
# Mock file reading
with patch("tools.base.read_files") as mock_read_files:
mock_read_files.return_value = "class UserController: ..."
result = await tool.execute(
{
"files": ["/path/to/project"],
"prompt": "What design patterns are used in this codebase?",
"analysis_type": "architecture",
}
)
assert len(result) == 1
output = json.loads(result[0].text)
assert output["status"] == "success"
assert "MVC pattern" in output["content"]
@pytest.mark.asyncio
async def test_empty_optional_fields(self, mock_model_response):
"""Test tools work with empty optional fields."""
tool = ChatTool()
with patch.object(tool, "get_model_provider") as mock_get_provider:
mock_provider = MagicMock()
mock_provider.get_provider_type.return_value = MagicMock(value="google")
mock_provider.supports_thinking_mode.return_value = False
mock_provider.generate_content.return_value = mock_model_response()
mock_get_provider.return_value = mock_provider
# Test with no files parameter
result = await tool.execute({"prompt": "Hello"})
assert len(result) == 1
output = json.loads(result[0].text)
assert output["status"] == "success"
@pytest.mark.asyncio
async def test_thinking_modes_work(self, mock_model_response):
"""Test that thinking modes are properly passed through."""
tool = ChatTool()
with patch.object(tool, "get_model_provider") as mock_get_provider:
mock_provider = MagicMock()
mock_provider.get_provider_type.return_value = MagicMock(value="google")
mock_provider.supports_thinking_mode.return_value = False
mock_provider.generate_content.return_value = mock_model_response()
mock_get_provider.return_value = mock_provider
result = await tool.execute({"prompt": "Test", "thinking_mode": "high", "temperature": 0.8})
assert len(result) == 1
output = json.loads(result[0].text)
assert output["status"] == "success"
# Verify generate_content was called with correct parameters
mock_provider.generate_content.assert_called_once()
call_kwargs = mock_provider.generate_content.call_args[1]
assert call_kwargs.get("temperature") == 0.8
# thinking_mode would be passed if the provider supports it
# In this test, we set supports_thinking_mode to False, so it won't be passed
@pytest.mark.asyncio
async def test_special_characters_in_prompts(self, mock_model_response):
"""Test prompts with special characters work correctly."""
tool = ChatTool()
with patch.object(tool, "get_model_provider") as mock_get_provider:
mock_provider = MagicMock()
mock_provider.get_provider_type.return_value = MagicMock(value="google")
mock_provider.supports_thinking_mode.return_value = False
mock_provider.generate_content.return_value = mock_model_response()
mock_get_provider.return_value = mock_provider
special_prompt = 'Test with "quotes" and\nnewlines\tand tabs'
result = await tool.execute({"prompt": special_prompt})
assert len(result) == 1
output = json.loads(result[0].text)
assert output["status"] == "success"
@pytest.mark.asyncio
async def test_mixed_file_paths(self, mock_model_response):
"""Test handling of various file path formats."""
tool = AnalyzeTool()
with patch.object(tool, "get_model_provider") as mock_get_provider:
mock_provider = MagicMock()
mock_provider.get_provider_type.return_value = MagicMock(value="google")
mock_provider.supports_thinking_mode.return_value = False
mock_provider.generate_content.return_value = mock_model_response()
mock_get_provider.return_value = mock_provider
with patch("tools.base.read_files") as mock_read_files:
mock_read_files.return_value = "Content"
result = await tool.execute(
{
"files": [
"/absolute/path/file.py",
"/Users/name/project/src/",
"/home/user/code.js",
],
"prompt": "Analyze these files",
}
)
assert len(result) == 1
output = json.loads(result[0].text)
assert output["status"] == "success"
mock_read_files.assert_called_once()
@pytest.mark.asyncio
async def test_unicode_content(self, mock_model_response):
"""Test handling of unicode content in prompts."""
tool = ChatTool()
with patch.object(tool, "get_model_provider") as mock_get_provider:
mock_provider = MagicMock()
mock_provider.get_provider_type.return_value = MagicMock(value="google")
mock_provider.supports_thinking_mode.return_value = False
mock_provider.generate_content.return_value = mock_model_response()
mock_get_provider.return_value = mock_provider
unicode_prompt = "Explain this: 你好世界 مرحبا بالعالم"
result = await tool.execute({"prompt": unicode_prompt})
assert len(result) == 1
output = json.loads(result[0].text)
assert output["status"] == "success"
if __name__ == "__main__":
pytest.main([__file__, "-v"])