New tool: testgen

Generates unit tests and encourages model to auto-detect framework and testing style from existing sample (if available)

simulator_tests/__init__.py

@@ -19,6 +19,7 @@ from .test_openrouter_fallback import OpenRouterFallbackTest
 from .test_openrouter_models import OpenRouterModelsTest
 from .test_per_tool_deduplication import PerToolDeduplicationTest
 from .test_redis_validation import RedisValidationTest
+from .test_testgen_validation import TestGenValidationTest
 from .test_token_allocation_validation import TokenAllocationValidationTest
 
 # Test registry for dynamic loading
@@ -36,6 +37,7 @@ TEST_REGISTRY = {
     "openrouter_fallback": OpenRouterFallbackTest,
     "openrouter_models": OpenRouterModelsTest,
     "token_allocation_validation": TokenAllocationValidationTest,
+    "testgen_validation": TestGenValidationTest,
     "conversation_chain_validation": ConversationChainValidationTest,
 }
 
@@ -54,6 +56,7 @@ __all__ = [
     "OpenRouterFallbackTest",
     "OpenRouterModelsTest",
     "TokenAllocationValidationTest",
+    "TestGenValidationTest",
     "ConversationChainValidationTest",
     "TEST_REGISTRY",
 ]

simulator_tests/test_testgen_validation.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""
+TestGen Tool Validation Test
+
+Tests the testgen tool by:
+- Creating a test code file with a specific function
+- Using testgen to generate tests with a specific function name
+- Validating that the output contains the expected test function
+- Confirming the format matches test generation patterns
+"""
+
+from .base_test import BaseSimulatorTest
+
+
+class TestGenValidationTest(BaseSimulatorTest):
+    """Test testgen tool validation with specific function name"""
+
+    @property
+    def test_name(self) -> str:
+        return "testgen_validation"
+
+    @property
+    def test_description(self) -> str:
+        return "TestGen tool validation with specific test function"
+
+    def run_test(self) -> bool:
+        """Test testgen tool with specific function name validation"""
+        try:
+            self.logger.info("Test: TestGen tool validation")
+
+            # Setup test files
+            self.setup_test_files()
+
+            # Create a specific code file for test generation
+            test_code_content = '''"""
+Sample authentication module for testing testgen
+"""
+
+class UserAuthenticator:
+    """Handles user authentication logic"""
+
+    def __init__(self):
+        self.failed_attempts = {}
+        self.max_attempts = 3
+
+    def validate_password(self, username, password):
+        """Validate user password with security checks"""
+        if not username or not password:
+            return False
+
+        if username in self.failed_attempts:
+            if self.failed_attempts[username] >= self.max_attempts:
+                return False  # Account locked
+
+        # Simple validation for demo
+        if len(password) < 8:
+            self._record_failed_attempt(username)
+            return False
+
+        if password == "password123":  # Demo valid password
+            self._reset_failed_attempts(username)
+            return True
+
+        self._record_failed_attempt(username)
+        return False
+
+    def _record_failed_attempt(self, username):
+        """Record a failed login attempt"""
+        self.failed_attempts[username] = self.failed_attempts.get(username, 0) + 1
+
+    def _reset_failed_attempts(self, username):
+        """Reset failed attempts after successful login"""
+        if username in self.failed_attempts:
+            del self.failed_attempts[username]
+'''
+
+            # Create the auth code file
+            auth_file = self.create_additional_test_file("user_auth.py", test_code_content)
+
+            # Test testgen tool with specific requirements
+            self.logger.info("  1.1: Generate tests with specific function name")
+            response, continuation_id = self.call_mcp_tool(
+                "testgen",
+                {
+                    "files": [auth_file],
+                    "prompt": "Generate comprehensive tests for the UserAuthenticator.validate_password method. Include tests for edge cases, security scenarios, and account locking. Use the specific test function name 'test_password_validation_edge_cases' for one of the test methods.",
+                    "model": "flash",
+                },
+            )
+
+            if not response:
+                self.logger.error("Failed to get testgen response")
+                return False
+
+            self.logger.info("  1.2: Validate response contains expected test function")
+
+            # Check that the response contains the specific test function name
+            if "test_password_validation_edge_cases" not in response:
+                self.logger.error("Response does not contain the requested test function name")
+                self.logger.debug(f"Response content: {response[:500]}...")
+                return False
+
+            # Check for common test patterns
+            test_patterns = [
+                "def test_",  # Test function definition
+                "assert",  # Assertion statements
+                "UserAuthenticator",  # Class being tested
+                "validate_password",  # Method being tested
+            ]
+
+            missing_patterns = []
+            for pattern in test_patterns:
+                if pattern not in response:
+                    missing_patterns.append(pattern)
+
+            if missing_patterns:
+                self.logger.error(f"Response missing expected test patterns: {missing_patterns}")
+                self.logger.debug(f"Response content: {response[:500]}...")
+                return False
+
+            self.logger.info("  ✅ TestGen tool validation successful")
+            self.logger.info("  ✅ Generated tests contain expected function name")
+            self.logger.info("  ✅ Generated tests follow proper test patterns")
+
+            return True
+
+        except Exception as e:
+            self.logger.error(f"TestGen validation test failed: {e}")
+            return False
+        finally:
+            self.cleanup_test_files()