feat: Add LOCAL variable support for responses with UTF-8 JSON encoding.

Description: This feature adds support for UTF-8 encoding in JSON responses, allowing for proper handling of special characters and emojis.

- Implement unit tests for UTF-8 encoding in various model providers including Gemini, OpenAI, and OpenAI Compatible.
- Validate UTF-8 support in token counting, content generation, and error handling.
- Introduce tests for JSON serialization ensuring proper handling of French characters and emojis.
- Create tests for language instruction generation based on locale settings.
- Validate UTF-8 handling in workflow tools including AnalyzeTool, CodereviewTool, and DebugIssueTool.
- Ensure that all tests check for correct UTF-8 character preservation and proper JSON formatting.
- Add integration tests to verify the interaction between locale settings and model responses.
This commit is contained in:
OhMyApps
2025-06-22 19:13:02 +02:00
parent 132c6ca025
commit e9c5662b3a
22 changed files with 1994 additions and 49 deletions

View File

@@ -112,11 +112,9 @@ class UserService:
result = await self.db.execute(
"SELECT * FROM users WHERE id = %s", (user_id,)
)
user_data = result.fetchone()
if user_data:
user_data = result.fetchone() if user_data:
# Cache for 1 hour - magic number
self.cache.setex(cache_key, 3600, json.dumps(user_data))
self.cache.setex(cache_key, 3600, json.dumps(user_data, ensure_ascii=False))
return user_data
@@ -273,10 +271,8 @@ class UserProfile(Base):
try:
return json.loads(self.preferences) if self.preferences else {}
except json.JSONDecodeError:
return {}
def set_preferences(self, prefs: dict):
self.preferences = json.dumps(prefs)
return {} def set_preferences(self, prefs: dict):
self.preferences = json.dumps(prefs, ensure_ascii=False)
class AuditLog(Base):
__tablename__ = "audit_logs"
@@ -298,7 +294,7 @@ class AuditLog(Base):
log = cls(
user_id=user_id,
action=action,
details=json.dumps(details) if details else None,
details=json.dumps(details, ensure_ascii=False) if details else None,
ip_address=ip_address,
user_agent=user_agent
)
@@ -692,9 +688,7 @@ class PerformanceTimer:
if not response_final_data.get("analysis_complete"):
self.logger.error("Expected analysis_complete=true for final step")
return False
# Check for expert analysis
return False # Check for expert analysis
if "expert_analysis" not in response_final_data:
self.logger.error("Missing expert_analysis in final response")
return False
@@ -702,7 +696,7 @@ class PerformanceTimer:
expert_analysis = response_final_data.get("expert_analysis", {})
# Check for expected analysis content (checking common patterns)
analysis_text = json.dumps(expert_analysis).lower()
analysis_text = json.dumps(expert_analysis, ensure_ascii=False).lower()
# Look for architectural analysis indicators
arch_indicators = ["architecture", "pattern", "coupling", "dependency", "scalability", "maintainability"]