Use consistent terminology
This commit is contained in:
@@ -23,7 +23,7 @@ import os
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from .token_utils import MAX_CONTEXT_TOKENS, estimate_tokens
|
||||
from .token_utils import DEFAULT_CONTEXT_WINDOW, estimate_tokens
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -508,14 +508,14 @@ def read_files(
|
||||
Args:
|
||||
file_paths: List of file or directory paths (absolute paths required)
|
||||
code: Optional direct code to include (prioritized over files)
|
||||
max_tokens: Maximum tokens to use (defaults to MAX_CONTEXT_TOKENS)
|
||||
max_tokens: Maximum tokens to use (defaults to DEFAULT_CONTEXT_WINDOW)
|
||||
reserve_tokens: Tokens to reserve for prompt and response (default 50K)
|
||||
|
||||
Returns:
|
||||
str: All file contents formatted for AI consumption
|
||||
"""
|
||||
if max_tokens is None:
|
||||
max_tokens = MAX_CONTEXT_TOKENS
|
||||
max_tokens = DEFAULT_CONTEXT_WINDOW
|
||||
|
||||
logger.debug(f"[FILES] read_files called with {len(file_paths)} paths")
|
||||
logger.debug(
|
||||
|
||||
@@ -72,7 +72,7 @@ class ModelContext:
|
||||
Returns:
|
||||
TokenAllocation with calculated budgets
|
||||
"""
|
||||
total_tokens = self.capabilities.max_tokens
|
||||
total_tokens = self.capabilities.context_window
|
||||
|
||||
# Dynamic allocation based on model capacity
|
||||
if total_tokens < 300_000:
|
||||
|
||||
@@ -9,7 +9,8 @@ approximate. For production systems requiring precise token counts,
|
||||
consider using the actual tokenizer for the specific model.
|
||||
"""
|
||||
|
||||
from config import MAX_CONTEXT_TOKENS
|
||||
# Default fallback for token limit (conservative estimate)
|
||||
DEFAULT_CONTEXT_WINDOW = 200_000 # Conservative fallback for unknown models
|
||||
|
||||
|
||||
def estimate_tokens(text: str) -> int:
|
||||
@@ -32,9 +33,9 @@ def estimate_tokens(text: str) -> int:
|
||||
return len(text) // 4
|
||||
|
||||
|
||||
def check_token_limit(text: str) -> tuple[bool, int]:
|
||||
def check_token_limit(text: str, context_window: int = DEFAULT_CONTEXT_WINDOW) -> tuple[bool, int]:
|
||||
"""
|
||||
Check if text exceeds the maximum token limit for Gemini models.
|
||||
Check if text exceeds the specified token limit.
|
||||
|
||||
This function is used to validate that prepared prompts will fit
|
||||
within the model's context window, preventing API errors and ensuring
|
||||
@@ -42,11 +43,12 @@ def check_token_limit(text: str) -> tuple[bool, int]:
|
||||
|
||||
Args:
|
||||
text: The text to check
|
||||
context_window: The model's context window size (defaults to conservative fallback)
|
||||
|
||||
Returns:
|
||||
Tuple[bool, int]: (is_within_limit, estimated_tokens)
|
||||
- is_within_limit: True if the text fits within MAX_CONTEXT_TOKENS
|
||||
- is_within_limit: True if the text fits within context_window
|
||||
- estimated_tokens: The estimated token count
|
||||
"""
|
||||
estimated = estimate_tokens(text)
|
||||
return estimated <= MAX_CONTEXT_TOKENS, estimated
|
||||
return estimated <= context_window, estimated
|
||||
|
||||
Reference in New Issue
Block a user