feat: Add LOCAL variable support for responses with UTF-8 JSON encoding.

Description: This feature adds support for UTF-8 encoding in JSON responses, allowing for proper handling of special characters and emojis.

- Implement unit tests for UTF-8 encoding in various model providers including Gemini, OpenAI, and OpenAI Compatible.
- Validate UTF-8 support in token counting, content generation, and error handling.
- Introduce tests for JSON serialization ensuring proper handling of French characters and emojis.
- Create tests for language instruction generation based on locale settings.
- Validate UTF-8 handling in workflow tools including AnalyzeTool, CodereviewTool, and DebugIssueTool.
- Ensure that all tests check for correct UTF-8 character preservation and proper JSON formatting.
- Add integration tests to verify the interaction between locale settings and model responses.

simulator_tests/base_test.py

@@ -136,10 +136,12 @@ class Calculator:
                 "id": 2,
                 "method": "tools/call",
                 "params": {"name": tool_name, "arguments": params},
-            }
-
-            # Combine all messages
-            messages = [json.dumps(init_request), json.dumps(initialized_notification), json.dumps(tool_request)]
+            }            # Combine all messages
+            messages = [
+                json.dumps(init_request, ensure_ascii=False), 
+                json.dumps(initialized_notification, ensure_ascii=False), 
+                json.dumps(tool_request, ensure_ascii=False)
+            ]
 
             # Join with newlines as MCP expects
             input_data = "\n".join(messages) + "\n"

simulator_tests/test_analyze_validation.py

@@ -112,11 +112,9 @@ class UserService:
         result = await self.db.execute(
             "SELECT * FROM users WHERE id = %s", (user_id,)
         )
-        user_data = result.fetchone()
-
-        if user_data:
+        user_data = result.fetchone()        if user_data:
             # Cache for 1 hour - magic number
-            self.cache.setex(cache_key, 3600, json.dumps(user_data))
+            self.cache.setex(cache_key, 3600, json.dumps(user_data, ensure_ascii=False))
 
         return user_data
 
@@ -273,10 +271,8 @@ class UserProfile(Base):
         try:
             return json.loads(self.preferences) if self.preferences else {}
         except json.JSONDecodeError:
-            return {}
-
-    def set_preferences(self, prefs: dict):
-        self.preferences = json.dumps(prefs)
+            return {}    def set_preferences(self, prefs: dict):
+        self.preferences = json.dumps(prefs, ensure_ascii=False)
 
 class AuditLog(Base):
     __tablename__ = "audit_logs"
@@ -298,7 +294,7 @@ class AuditLog(Base):
         log = cls(
             user_id=user_id,
             action=action,
-            details=json.dumps(details) if details else None,
+            details=json.dumps(details, ensure_ascii=False) if details else None,
             ip_address=ip_address,
             user_agent=user_agent
         )
@@ -692,9 +688,7 @@ class PerformanceTimer:
 
             if not response_final_data.get("analysis_complete"):
                 self.logger.error("Expected analysis_complete=true for final step")
-                return False
-
-            # Check for expert analysis
+                return False            # Check for expert analysis
             if "expert_analysis" not in response_final_data:
                 self.logger.error("Missing expert_analysis in final response")
                 return False
@@ -702,7 +696,7 @@ class PerformanceTimer:
             expert_analysis = response_final_data.get("expert_analysis", {})
 
             # Check for expected analysis content (checking common patterns)
-            analysis_text = json.dumps(expert_analysis).lower()
+            analysis_text = json.dumps(expert_analysis, ensure_ascii=False).lower()
 
             # Look for architectural analysis indicators
             arch_indicators = ["architecture", "pattern", "coupling", "dependency", "scalability", "maintainability"]

simulator_tests/test_codereview_validation.py

@@ -514,7 +514,7 @@ class ConfigurationManager:
             expert_analysis = response_final_data.get("expert_analysis", {})
 
             # Check for expected analysis content (checking common patterns)
-            analysis_text = json.dumps(expert_analysis).lower()
+            analysis_text = json.dumps(expert_analysis, ensure_ascii=False).lower()
 
             # Look for code review identification
             review_indicators = ["security", "vulnerability", "performance", "critical", "api", "key"]

simulator_tests/test_debug_validation.py

@@ -385,7 +385,7 @@ RuntimeError: dictionary changed size during iteration
             expert_analysis = response_final_data.get("expert_analysis", {})
 
             # Check for expected analysis content (checking common patterns)
-            analysis_text = json.dumps(expert_analysis).lower()
+            analysis_text = json.dumps(expert_analysis, ensure_ascii=False).lower()
 
             # Look for bug identification
             bug_indicators = ["dictionary", "iteration", "modify", "runtime", "error", "del"]

simulator_tests/test_precommitworkflow_validation.py

@@ -430,7 +430,7 @@ REQUIREMENTS:
             expert_analysis = response_final_data.get("expert_analysis", {})
 
             # Check for expected analysis content (checking common patterns)
-            analysis_text = json.dumps(expert_analysis).lower()
+            analysis_text = json.dumps(expert_analysis, ensure_ascii=False).lower()
 
             # Look for security issue identification
             security_indicators = ["sql", "injection", "security", "hardcoded", "secret", "authentication"]

simulator_tests/test_refactor_validation.py

@@ -125,7 +125,7 @@ class DataProcessorManager:
         # Code smell: Duplicate date formatting logic
         if output_format == 'json':
             processed_data['processed_at'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-            result = json.dumps(processed_data)
+            result = json.dumps(processed_data, ensure_ascii=False)
         elif output_format == 'csv':
             processed_data['processed_at'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
             result = f"{processed_data['full_name']},{processed_data['email_domain']},{processed_data['age_category']}"
@@ -580,7 +580,7 @@ class UserData:
                     self.logger.error("Missing expert_analysis in final response")
                     return False
                 expert_analysis = response_final_data.get("expert_analysis", {})
-                analysis_content = json.dumps(expert_analysis).lower()
+                analysis_content = json.dumps(expert_analysis, ensure_ascii=False).lower()
             elif actual_status == "files_required_to_continue":
                 # For files_required_to_continue, analysis is in content field
                 if "content" not in response_final_data:
@@ -708,7 +708,7 @@ def format_output(data, format_type):
     \"\"\"Format output - duplicate logic\"\"\"
     if format_type == 'json':
         import json
-        return json.dumps(data)
+        return json.dumps(data, ensure_ascii=False)
     elif format_type == 'csv':
         return ','.join(str(v) for v in data.values())
     else:

simulator_tests/test_testgen_validation.py

@@ -346,7 +346,7 @@ class TestCalculatorBasic:
             expert_analysis = response_final_data.get("expert_analysis", {})
 
             # Check for expected analysis content
-            analysis_text = json.dumps(expert_analysis).lower()
+            analysis_text = json.dumps(expert_analysis, ensure_ascii=False).lower()
 
             # Look for test generation indicators
             test_indicators = ["test", "edge", "boundary", "error", "coverage", "pytest"]

simulator_tests/test_thinkdeep_validation.py

@@ -415,7 +415,7 @@ class ThinkDeepWorkflowValidationTest(ConversationBaseTest):
                         expert_analysis = {"analysis": expert_analysis}
 
             # Check for expected analysis content (checking common patterns)
-            analysis_text = json.dumps(expert_analysis).lower()
+            analysis_text = json.dumps(expert_analysis, ensure_ascii=False).lower()
 
             # Look for thinking analysis validation
             thinking_indicators = ["migration", "strategy", "microservices", "risk", "approach", "implementation"]