feat: added intelligence_score to the model capabilities schema; a 1-20 number that can be specified to influence the sort order of models presented to the CLI in auto selection mode
fix: model definition re-introduced into the schema but intelligently and only a summary is generated per tool. Required to ensure CLI calls and uses the correct model fix: removed `model` param from some tools where this wasn't needed fix: fixed adherence to `*_ALLOWED_MODELS` by advertising only the allowed models to the CLI fix: removed duplicates across providers when passing canonical names back to the CLI; the first enabled provider wins
This commit is contained in:
Fahad
@@ -291,13 +291,161 @@ class BaseTool(ABC):
|
||||
def _format_available_models_list(self) -> str:
|
||||
"""Return a human-friendly list of available models or guidance when none found."""
|
||||
|
||||
available_models = self._get_available_models()
|
||||
if not available_models:
|
||||
summaries, total, has_restrictions = self._get_ranked_model_summaries()
|
||||
if not summaries:
|
||||
return (
|
||||
"No models detected. Configure provider credentials or set DEFAULT_MODEL to a valid option. "
|
||||
"If the user requested a specific model, respond with this notice instead of substituting another model."
|
||||
)
|
||||
return ", ".join(available_models)
|
||||
display = "; ".join(summaries)
|
||||
remainder = total - len(summaries)
|
||||
if remainder > 0:
|
||||
display = f"{display}; +{remainder} more (use the `listmodels` tool for the full roster)"
|
||||
return display
|
||||
|
||||
@staticmethod
|
||||
def _format_context_window(tokens: int) -> Optional[str]:
|
||||
"""Convert a raw context window into a short display string."""
|
||||
|
||||
if not tokens or tokens <= 0:
|
||||
return None
|
||||
|
||||
if tokens >= 1_000_000:
|
||||
if tokens % 1_000_000 == 0:
|
||||
return f"{tokens // 1_000_000}M ctx"
|
||||
return f"{tokens / 1_000_000:.1f}M ctx"
|
||||
|
||||
if tokens >= 1_000:
|
||||
if tokens % 1_000 == 0:
|
||||
return f"{tokens // 1_000}K ctx"
|
||||
return f"{tokens / 1_000:.1f}K ctx"
|
||||
|
||||
return f"{tokens} ctx"
|
||||
|
||||
def _collect_ranked_capabilities(self) -> list[tuple[int, str, Any]]:
|
||||
"""Gather available model capabilities sorted by capability rank."""
|
||||
|
||||
from providers.registry import ModelProviderRegistry
|
||||
|
||||
ranked: list[tuple[int, str, Any]] = []
|
||||
available = ModelProviderRegistry.get_available_models(respect_restrictions=True)
|
||||
|
||||
for model_name, provider_type in available.items():
|
||||
provider = ModelProviderRegistry.get_provider(provider_type)
|
||||
if not provider:
|
||||
continue
|
||||
|
||||
try:
|
||||
capabilities = provider.get_capabilities(model_name)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
rank = capabilities.get_effective_capability_rank()
|
||||
ranked.append((rank, model_name, capabilities))
|
||||
|
||||
ranked.sort(key=lambda item: (-item[0], item[1]))
|
||||
return ranked
|
||||
|
||||
@staticmethod
|
||||
def _normalize_model_identifier(name: str) -> str:
|
||||
"""Normalize model names for deduplication across providers."""
|
||||
|
||||
normalized = name.lower()
|
||||
if ":" in normalized:
|
||||
normalized = normalized.split(":", 1)[0]
|
||||
if "/" in normalized:
|
||||
normalized = normalized.split("/", 1)[-1]
|
||||
return normalized
|
||||
|
||||
def _get_ranked_model_summaries(self, limit: int = 5) -> tuple[list[str], int, bool]:
|
||||
"""Return formatted, ranked model summaries and restriction status."""
|
||||
|
||||
ranked = self._collect_ranked_capabilities()
|
||||
|
||||
# Build allowlist map (provider -> lowercase names) when restrictions are active
|
||||
allowed_map: dict[Any, set[str]] = {}
|
||||
try:
|
||||
from utils.model_restrictions import get_restriction_service
|
||||
|
||||
restriction_service = get_restriction_service()
|
||||
if restriction_service:
|
||||
from providers.shared import ProviderType
|
||||
|
||||
for provider_type in ProviderType:
|
||||
allowed = restriction_service.get_allowed_models(provider_type)
|
||||
if allowed:
|
||||
allowed_map[provider_type] = {name.lower() for name in allowed if name}
|
||||
except Exception:
|
||||
allowed_map = {}
|
||||
|
||||
filtered: list[tuple[int, str, Any]] = []
|
||||
seen_normalized: set[str] = set()
|
||||
|
||||
for rank, model_name, capabilities in ranked:
|
||||
canonical_name = getattr(capabilities, "model_name", model_name)
|
||||
canonical_lower = canonical_name.lower()
|
||||
alias_lower = model_name.lower()
|
||||
provider_type = getattr(capabilities, "provider", None)
|
||||
|
||||
if allowed_map:
|
||||
if provider_type not in allowed_map:
|
||||
continue
|
||||
allowed_set = allowed_map[provider_type]
|
||||
if canonical_lower not in allowed_set and alias_lower not in allowed_set:
|
||||
continue
|
||||
|
||||
normalized = self._normalize_model_identifier(canonical_name)
|
||||
if normalized in seen_normalized:
|
||||
continue
|
||||
|
||||
seen_normalized.add(normalized)
|
||||
filtered.append((rank, canonical_name, capabilities))
|
||||
|
||||
summaries: list[str] = []
|
||||
for rank, canonical_name, capabilities in filtered[:limit]:
|
||||
details: list[str] = []
|
||||
|
||||
context_str = self._format_context_window(getattr(capabilities, "context_window", 0))
|
||||
if context_str:
|
||||
details.append(context_str)
|
||||
|
||||
if getattr(capabilities, "supports_extended_thinking", False):
|
||||
details.append("thinking")
|
||||
|
||||
base = f"{canonical_name} (score {rank}"
|
||||
if details:
|
||||
base = f"{base}, {', '.join(details)}"
|
||||
summaries.append(f"{base})")
|
||||
|
||||
return summaries, len(filtered), bool(allowed_map)
|
||||
|
||||
def _get_restriction_note(self) -> Optional[str]:
|
||||
"""Return a string describing active per-provider allowlists, if any."""
|
||||
|
||||
env_labels = {
|
||||
"OPENAI_ALLOWED_MODELS": "OpenAI",
|
||||
"GOOGLE_ALLOWED_MODELS": "Google",
|
||||
"XAI_ALLOWED_MODELS": "X.AI",
|
||||
"OPENROUTER_ALLOWED_MODELS": "OpenRouter",
|
||||
"DIAL_ALLOWED_MODELS": "DIAL",
|
||||
}
|
||||
|
||||
notes: list[str] = []
|
||||
for env_var, label in env_labels.items():
|
||||
raw = os.getenv(env_var)
|
||||
if not raw:
|
||||
continue
|
||||
|
||||
models = sorted({token.strip() for token in raw.split(",") if token.strip()})
|
||||
if not models:
|
||||
continue
|
||||
|
||||
notes.append(f"{label}: {', '.join(models)}")
|
||||
|
||||
if not notes:
|
||||
return None
|
||||
|
||||
return "Policy allows only → " + "; ".join(notes)
|
||||
|
||||
def _build_model_unavailable_message(self, model_name: str) -> str:
|
||||
"""Compose a consistent error message for unavailable model scenarios."""
|
||||
@@ -344,8 +492,23 @@ class BaseTool(ABC):
|
||||
if self.is_effective_auto_mode():
|
||||
description = (
|
||||
"Currently in auto model selection mode. CRITICAL: When the user names a model, you MUST use that exact name unless the server rejects it. "
|
||||
"If no model is provided, you may call the `listmodels` tool to review options and select an appropriate match."
|
||||
"If no model is provided, you may use the `listmodels` tool to review options and select an appropriate match."
|
||||
)
|
||||
summaries, total, restricted = self._get_ranked_model_summaries()
|
||||
remainder = max(0, total - len(summaries))
|
||||
if summaries:
|
||||
top_line = "; ".join(summaries)
|
||||
if remainder > 0:
|
||||
label = "Allowed models" if restricted else "Top models"
|
||||
top_line = f"{label}: {top_line}; +{remainder} more via `listmodels`."
|
||||
else:
|
||||
label = "Allowed models" if restricted else "Top models"
|
||||
top_line = f"{label}: {top_line}."
|
||||
description = f"{description} {top_line}"
|
||||
|
||||
restriction_note = self._get_restriction_note()
|
||||
if restriction_note and (remainder > 0 or not summaries):
|
||||
description = f"{description} {restriction_note}."
|
||||
return {
|
||||
"type": "string",
|
||||
"description": description,
|
||||
@@ -353,8 +516,23 @@ class BaseTool(ABC):
|
||||
|
||||
description = (
|
||||
f"The default model is '{DEFAULT_MODEL}'. Override only when the user explicitly requests a different model, and use that exact name. "
|
||||
"If the requested model fails validation, surface the server error instead of substituting another model. When unsure, call the `listmodels` tool for details."
|
||||
"If the requested model fails validation, surface the server error instead of substituting another model. When unsure, use the `listmodels` tool for details."
|
||||
)
|
||||
summaries, total, restricted = self._get_ranked_model_summaries()
|
||||
remainder = max(0, total - len(summaries))
|
||||
if summaries:
|
||||
top_line = "; ".join(summaries)
|
||||
if remainder > 0:
|
||||
label = "Allowed models" if restricted else "Preferred alternatives"
|
||||
top_line = f"{label}: {top_line}; +{remainder} more via `listmodels`."
|
||||
else:
|
||||
label = "Allowed models" if restricted else "Preferred alternatives"
|
||||
top_line = f"{label}: {top_line}."
|
||||
description = f"{description} {top_line}"
|
||||
|
||||
restriction_note = self._get_restriction_note()
|
||||
if restriction_note and (remainder > 0 or not summaries):
|
||||
description = f"{description} {restriction_note}."
|
||||
|
||||
return {
|
||||
"type": "string",
|
||||
@@ -1242,31 +1420,6 @@ When recommending searches, be specific about what information you need and why
|
||||
import base64
|
||||
from pathlib import Path
|
||||
|
||||
# Handle legacy calls (positional model_name string)
|
||||
if isinstance(model_context, str):
|
||||
# Legacy call: _validate_image_limits(images, "model-name")
|
||||
logger.warning(
|
||||
"Legacy _validate_image_limits call with model_name string. Use model_context object instead."
|
||||
)
|
||||
try:
|
||||
from utils.model_context import ModelContext
|
||||
|
||||
model_context = ModelContext(model_context)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to create model context from legacy model_name: {e}")
|
||||
# Generic error response for any unavailable model
|
||||
return {
|
||||
"status": "error",
|
||||
"content": self._build_model_unavailable_message(str(model_context)),
|
||||
"content_type": "text",
|
||||
"metadata": {
|
||||
"error_type": "validation_error",
|
||||
"model_name": model_context,
|
||||
"supports_images": None, # Unknown since model doesn't exist
|
||||
"image_count": len(images) if images else 0,
|
||||
},
|
||||
}
|
||||
|
||||
if not model_context:
|
||||
# Get from tool's stored context as fallback
|
||||
model_context = getattr(self, "_model_context", None)
|
||||
|
||||
Reference in New Issue
Block a user