4086306c58
Generates unit tests and encourages model to auto-detect framework and testing style from existing sample (if available)
132 lines
4.7 KiB
Python
132 lines
4.7 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
TestGen Tool Validation Test
|
|
|
|
Tests the testgen tool by:
|
|
- Creating a test code file with a specific function
|
|
- Using testgen to generate tests with a specific function name
|
|
- Validating that the output contains the expected test function
|
|
- Confirming the format matches test generation patterns
|
|
"""
|
|
|
|
from .base_test import BaseSimulatorTest
|
|
|
|
|
|
class TestGenValidationTest(BaseSimulatorTest):
|
|
"""Test testgen tool validation with specific function name"""
|
|
|
|
@property
|
|
def test_name(self) -> str:
|
|
return "testgen_validation"
|
|
|
|
@property
|
|
def test_description(self) -> str:
|
|
return "TestGen tool validation with specific test function"
|
|
|
|
def run_test(self) -> bool:
|
|
"""Test testgen tool with specific function name validation"""
|
|
try:
|
|
self.logger.info("Test: TestGen tool validation")
|
|
|
|
# Setup test files
|
|
self.setup_test_files()
|
|
|
|
# Create a specific code file for test generation
|
|
test_code_content = '''"""
|
|
Sample authentication module for testing testgen
|
|
"""
|
|
|
|
class UserAuthenticator:
|
|
"""Handles user authentication logic"""
|
|
|
|
def __init__(self):
|
|
self.failed_attempts = {}
|
|
self.max_attempts = 3
|
|
|
|
def validate_password(self, username, password):
|
|
"""Validate user password with security checks"""
|
|
if not username or not password:
|
|
return False
|
|
|
|
if username in self.failed_attempts:
|
|
if self.failed_attempts[username] >= self.max_attempts:
|
|
return False # Account locked
|
|
|
|
# Simple validation for demo
|
|
if len(password) < 8:
|
|
self._record_failed_attempt(username)
|
|
return False
|
|
|
|
if password == "password123": # Demo valid password
|
|
self._reset_failed_attempts(username)
|
|
return True
|
|
|
|
self._record_failed_attempt(username)
|
|
return False
|
|
|
|
def _record_failed_attempt(self, username):
|
|
"""Record a failed login attempt"""
|
|
self.failed_attempts[username] = self.failed_attempts.get(username, 0) + 1
|
|
|
|
def _reset_failed_attempts(self, username):
|
|
"""Reset failed attempts after successful login"""
|
|
if username in self.failed_attempts:
|
|
del self.failed_attempts[username]
|
|
'''
|
|
|
|
# Create the auth code file
|
|
auth_file = self.create_additional_test_file("user_auth.py", test_code_content)
|
|
|
|
# Test testgen tool with specific requirements
|
|
self.logger.info(" 1.1: Generate tests with specific function name")
|
|
response, continuation_id = self.call_mcp_tool(
|
|
"testgen",
|
|
{
|
|
"files": [auth_file],
|
|
"prompt": "Generate comprehensive tests for the UserAuthenticator.validate_password method. Include tests for edge cases, security scenarios, and account locking. Use the specific test function name 'test_password_validation_edge_cases' for one of the test methods.",
|
|
"model": "flash",
|
|
},
|
|
)
|
|
|
|
if not response:
|
|
self.logger.error("Failed to get testgen response")
|
|
return False
|
|
|
|
self.logger.info(" 1.2: Validate response contains expected test function")
|
|
|
|
# Check that the response contains the specific test function name
|
|
if "test_password_validation_edge_cases" not in response:
|
|
self.logger.error("Response does not contain the requested test function name")
|
|
self.logger.debug(f"Response content: {response[:500]}...")
|
|
return False
|
|
|
|
# Check for common test patterns
|
|
test_patterns = [
|
|
"def test_", # Test function definition
|
|
"assert", # Assertion statements
|
|
"UserAuthenticator", # Class being tested
|
|
"validate_password", # Method being tested
|
|
]
|
|
|
|
missing_patterns = []
|
|
for pattern in test_patterns:
|
|
if pattern not in response:
|
|
missing_patterns.append(pattern)
|
|
|
|
if missing_patterns:
|
|
self.logger.error(f"Response missing expected test patterns: {missing_patterns}")
|
|
self.logger.debug(f"Response content: {response[:500]}...")
|
|
return False
|
|
|
|
self.logger.info(" ✅ TestGen tool validation successful")
|
|
self.logger.info(" ✅ Generated tests contain expected function name")
|
|
self.logger.info(" ✅ Generated tests follow proper test patterns")
|
|
|
|
return True
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"TestGen validation test failed: {e}")
|
|
return False
|
|
finally:
|
|
self.cleanup_test_files()
|