Fix conversation history duplication and optimize file embedding

This major refactoring addresses critical bugs in conversation history management
and significantly improves token efficiency through intelligent file embedding:

**Key Improvements:**
• Fixed conversation history duplication bug by centralizing reconstruction in server.py
• Added intelligent file filtering to prevent re-embedding files already in conversation history
• Centralized file processing logic in BaseTool._prepare_file_content_for_prompt()
• Enhanced log monitoring with better categorization and file embedding visibility
• Updated comprehensive test suite to verify new architecture and edge cases

**Architecture Changes:**
• Removed duplicate conversation history reconstruction from tools/base.py
• Conversation history now handled exclusively by server.py:reconstruct_thread_context
• All tools now use centralized file processing with automatic deduplication
• Improved token efficiency by embedding unique files only once per conversation

**Performance Benefits:**
• Reduced token usage through smart file filtering
• Eliminated redundant file embeddings in continued conversations
• Better observability with detailed debug logging for file operations

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

tests/test_conversation_memory.py

@@ -183,8 +183,13 @@ class TestConversationMemory:
         assert "Python is a programming language" in history
 
         # Test file tracking
-        assert "📁 Files referenced: /home/user/main.py, /home/user/docs/readme.md" in history
-        assert "📁 Files referenced: /home/user/examples/" in history
+        # Check that the new file embedding section is included
+        assert "=== FILES REFERENCED IN THIS CONVERSATION ===" in history
+        assert "The following files have been shared and analyzed during our conversation." in history
+
+        # Check that file context from previous turns is included (now shows files used per turn)
+        assert "📁 Files used in this turn: /home/user/main.py, /home/user/docs/readme.md" in history
+        assert "📁 Files used in this turn: /home/user/examples/" in history
 
         # Test follow-up attribution
         assert "[Gemini's Follow-up: Would you like examples?]" in history
@@ -598,9 +603,9 @@ class TestConversationFlow:
         assert "--- Turn 3 (Gemini using analyze) ---" in history
 
         # Verify all files are preserved in chronological order
-        turn_1_files = "📁 Files referenced: /project/src/main.py, /project/src/utils.py"
-        turn_2_files = "📁 Files referenced: /project/tests/, /project/test_main.py"
-        turn_3_files = "📁 Files referenced: /project/tests/test_utils.py, /project/coverage.html"
+        turn_1_files = "📁 Files used in this turn: /project/src/main.py, /project/src/utils.py"
+        turn_2_files = "📁 Files used in this turn: /project/tests/, /project/test_main.py"
+        turn_3_files = "📁 Files used in this turn: /project/tests/test_utils.py, /project/coverage.html"
 
         assert turn_1_files in history
         assert turn_2_files in history
@@ -718,6 +723,63 @@ class TestConversationFlow:
         assert len(retrieved_context.turns) == 1
         assert retrieved_context.turns[0].follow_up_question == "Want to explore scalability?"
 
+    def test_token_limit_optimization_in_conversation_history(self):
+        """Test that build_conversation_history efficiently handles token limits"""
+        import os
+        import tempfile
+
+        from utils.conversation_memory import build_conversation_history
+
+        # Create test files with known content sizes
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Create small and large test files
+            small_file = os.path.join(temp_dir, "small.py")
+            large_file = os.path.join(temp_dir, "large.py")
+
+            small_content = "# Small file\nprint('hello')\n"
+            large_content = "# Large file\n" + "x = 1\n" * 10000  # Very large file
+
+            with open(small_file, "w") as f:
+                f.write(small_content)
+            with open(large_file, "w") as f:
+                f.write(large_content)
+
+            # Create context with files that would exceed token limit
+            context = ThreadContext(
+                thread_id="test-token-limit",
+                created_at="2023-01-01T00:00:00Z",
+                last_updated_at="2023-01-01T00:01:00Z",
+                tool_name="analyze",
+                turns=[
+                    ConversationTurn(
+                        role="user",
+                        content="Analyze these files",
+                        timestamp="2023-01-01T00:00:30Z",
+                        files=[small_file, large_file],  # Large file should be truncated
+                    )
+                ],
+                initial_context={"prompt": "Analyze code"},
+            )
+
+            # Build conversation history (should handle token limits gracefully)
+            history = build_conversation_history(context)
+
+            # Verify the history was built successfully
+            assert "=== CONVERSATION HISTORY ===" in history
+            assert "=== FILES REFERENCED IN THIS CONVERSATION ===" in history
+
+            # The small file should be included, but large file might be truncated
+            # At minimum, verify no crashes and history is generated
+            assert len(history) > 0
+
+            # If truncation occurred, there should be a note about it
+            if "additional file(s) were truncated due to token limit" in history:
+                assert small_file in history or large_file in history
+            else:
+                # Both files fit within limit
+                assert small_file in history
+                assert large_file in history
+
 
 if __name__ == "__main__":
     pytest.main([__file__])