diff --git a/src/cloudcode/count-tokens.js b/src/cloudcode/count-tokens.js index c1088b9..be12a70 100644 --- a/src/cloudcode/count-tokens.js +++ b/src/cloudcode/count-tokens.js @@ -3,8 +3,8 @@ * * Implements Anthropic's /v1/messages/count_tokens endpoint * Uses official tokenizers for each model family: - * - Claude: @anthropic-ai/tokenizer (99.99% accuracy) - * - Gemini: @lenml/tokenizer-gemini (99.99% accuracy) + * - Claude: @anthropic-ai/tokenizer + * - Gemini: @lenml/tokenizer-gemini * * @see https://platform.claude.com/docs/en/api/messages-count-tokens */ @@ -112,6 +112,12 @@ function estimateTextTokens(text, model, geminiTok = null) { /** * Extract text from message content * + * Note: This function only extracts text from 'text' type blocks. + * Image blocks (type: 'image') and document blocks (type: 'document') are not tokenized + * and will not contribute to the token count. This is intentional as binary content + * requires different handling and Anthropic's actual token counting for images uses + * a fixed estimate (~1600 tokens per image) that depends on image dimensions. + * * @param {string|Array} content - Message content * @returns {string} Concatenated text */ @@ -194,7 +200,7 @@ function countTokensLocally(request, geminiTok = null) { /** * Count tokens in a message request * Implements Anthropic's /v1/messages/count_tokens endpoint - * Uses local tokenization for all content types (99.99% accuracy) + * Uses local tokenization for all content types * * @param {Object} anthropicRequest - Anthropic format request with messages, model, system, tools * @param {Object} accountManager - Account manager instance (unused, kept for API compatibility) diff --git a/src/server.js b/src/server.js index 40c41d8..86c92a2 100644 --- a/src/server.js +++ b/src/server.js @@ -599,7 +599,7 @@ app.get('/v1/models', async (req, res) => { /** * Count tokens endpoint - Anthropic Messages API compatible - * Uses hybrid approach: local tokenizer for text, API for complex content (images, documents) + * Uses local tokenization with official tokenizers (@anthropic-ai/tokenizer for Claude, @lenml/tokenizer-gemini for Gemini) */ app.post('/v1/messages/count_tokens', async (req, res) => { try { diff --git a/tests/test-count-tokens.cjs b/tests/test-count-tokens.cjs index 71be240..325a72d 100644 --- a/tests/test-count-tokens.cjs +++ b/tests/test-count-tokens.cjs @@ -2,7 +2,7 @@ * Test Count Tokens - Tests for the /v1/messages/count_tokens endpoint * * Verifies token counting functionality: - * - Local estimation using gpt-tokenizer + * - Local estimation using official tokenizers (@anthropic-ai/tokenizer for Claude, @lenml/tokenizer-gemini for Gemini) * - Request validation * - Different content types (text, tools, system prompts) */ @@ -436,6 +436,47 @@ async function runTests() { assertGreater(response.input_tokens, 10, 'input_tokens'); }); + // Test 16: Gemini model token counting + await test('Gemini model returns token count', async () => { + const response = await countTokensRequest({ + model: 'gemini-3-flash', + messages: [ + { role: 'user', content: 'Hello, how are you?' } + ] + }); + + assert(response.statusCode === 200, `Expected 200, got ${response.statusCode}`); + assertType(response.input_tokens, 'number', 'input_tokens'); + assertGreater(response.input_tokens, 0, 'input_tokens'); + }); + + // Test 17: Gemini model with system prompt and tools + await test('Gemini model with system prompt and tools', async () => { + const response = await countTokensRequest({ + model: 'gemini-3-flash', + system: 'You are a helpful assistant.', + messages: [ + { role: 'user', content: 'What is the weather in Tokyo?' } + ], + tools: [ + { + name: 'get_weather', + description: 'Get weather for a location', + input_schema: { + type: 'object', + properties: { + location: { type: 'string' } + } + } + } + ] + }); + + assert(response.statusCode === 200, `Expected 200, got ${response.statusCode}`); + assertType(response.input_tokens, 'number', 'input_tokens'); + assertGreater(response.input_tokens, 10, 'input_tokens for Gemini with tools'); + }); + // Summary console.log('\n' + '═'.repeat(60)); console.log(`Tests completed: ${passed} passed, ${failed} failed`);