Use consistent terminology

This commit is contained in:
Fahad
2025-06-13 09:06:12 +04:00
parent e2762c4ed0
commit b16f85979b
13 changed files with 38 additions and 52 deletions

View File

@@ -23,7 +23,7 @@ import os
from pathlib import Path
from typing import Optional
from .token_utils import MAX_CONTEXT_TOKENS, estimate_tokens
from .token_utils import DEFAULT_CONTEXT_WINDOW, estimate_tokens
logger = logging.getLogger(__name__)
@@ -508,14 +508,14 @@ def read_files(
Args:
file_paths: List of file or directory paths (absolute paths required)
code: Optional direct code to include (prioritized over files)
max_tokens: Maximum tokens to use (defaults to MAX_CONTEXT_TOKENS)
max_tokens: Maximum tokens to use (defaults to DEFAULT_CONTEXT_WINDOW)
reserve_tokens: Tokens to reserve for prompt and response (default 50K)
Returns:
str: All file contents formatted for AI consumption
"""
if max_tokens is None:
max_tokens = MAX_CONTEXT_TOKENS
max_tokens = DEFAULT_CONTEXT_WINDOW
logger.debug(f"[FILES] read_files called with {len(file_paths)} paths")
logger.debug(

View File

@@ -72,7 +72,7 @@ class ModelContext:
Returns:
TokenAllocation with calculated budgets
"""
total_tokens = self.capabilities.max_tokens
total_tokens = self.capabilities.context_window
# Dynamic allocation based on model capacity
if total_tokens < 300_000:

View File

@@ -9,7 +9,8 @@ approximate. For production systems requiring precise token counts,
consider using the actual tokenizer for the specific model.
"""
from config import MAX_CONTEXT_TOKENS
# Default fallback for token limit (conservative estimate)
DEFAULT_CONTEXT_WINDOW = 200_000 # Conservative fallback for unknown models
def estimate_tokens(text: str) -> int:
@@ -32,9 +33,9 @@ def estimate_tokens(text: str) -> int:
return len(text) // 4
def check_token_limit(text: str) -> tuple[bool, int]:
def check_token_limit(text: str, context_window: int = DEFAULT_CONTEXT_WINDOW) -> tuple[bool, int]:
"""
Check if text exceeds the maximum token limit for Gemini models.
Check if text exceeds the specified token limit.
This function is used to validate that prepared prompts will fit
within the model's context window, preventing API errors and ensuring
@@ -42,11 +43,12 @@ def check_token_limit(text: str) -> tuple[bool, int]:
Args:
text: The text to check
context_window: The model's context window size (defaults to conservative fallback)
Returns:
Tuple[bool, int]: (is_within_limit, estimated_tokens)
- is_within_limit: True if the text fits within MAX_CONTEXT_TOKENS
- is_within_limit: True if the text fits within context_window
- estimated_tokens: The estimated token count
"""
estimated = estimate_tokens(text)
return estimated <= MAX_CONTEXT_TOKENS, estimated
return estimated <= context_window, estimated