Schema now lists all models including locally available models

New tool to list all models `listmodels`
Integration test to for all the different combinations of API keys
Tweaks to codereview prompt for a better quality input from Claude
Fixed missing 'low' severity in codereview

tests/test_model_enumeration.py

@@ -0,0 +1,282 @@
+"""
+Integration tests for model enumeration across all provider combinations.
+
+These tests ensure that the _get_available_models() method correctly returns
+all expected models based on which providers are configured via environment variables.
+"""
+
+import importlib
+import os
+
+import pytest
+
+from providers.registry import ModelProviderRegistry
+from tools.analyze import AnalyzeTool
+
+
+@pytest.mark.no_mock_provider
+class TestModelEnumeration:
+    """Test model enumeration with various provider configurations"""
+
+    def setup_method(self):
+        """Set up clean state before each test."""
+        # Save original environment state
+        self._original_env = {
+            "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL", ""),
+            "GEMINI_API_KEY": os.environ.get("GEMINI_API_KEY", ""),
+            "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY", ""),
+            "XAI_API_KEY": os.environ.get("XAI_API_KEY", ""),
+            "OPENROUTER_API_KEY": os.environ.get("OPENROUTER_API_KEY", ""),
+            "CUSTOM_API_URL": os.environ.get("CUSTOM_API_URL", ""),
+        }
+
+        # Clear provider registry
+        ModelProviderRegistry._instance = None
+
+    def teardown_method(self):
+        """Clean up after each test."""
+        # Restore original environment
+        for key, value in self._original_env.items():
+            if value:
+                os.environ[key] = value
+            elif key in os.environ:
+                del os.environ[key]
+
+        # Reload config
+        import config
+
+        importlib.reload(config)
+
+        # Clear provider registry
+        ModelProviderRegistry._instance = None
+
+    def _setup_environment(self, provider_config):
+        """Helper to set up environment variables for testing."""
+        # Clear all provider-related env vars first
+        for key in ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "CUSTOM_API_URL"]:
+            if key in os.environ:
+                del os.environ[key]
+
+        # Set new values
+        for key, value in provider_config.items():
+            if value is not None:
+                os.environ[key] = value
+
+        # Always set auto mode for these tests
+        os.environ["DEFAULT_MODEL"] = "auto"
+
+        # Reload config to pick up changes
+        import config
+
+        importlib.reload(config)
+
+        # Reload tools.base to ensure fresh state
+        import tools.base
+
+        importlib.reload(tools.base)
+
+    def test_native_models_always_included(self):
+        """Test that native models from MODEL_CAPABILITIES_DESC are always included."""
+        self._setup_environment({})  # No providers configured
+
+        tool = AnalyzeTool()
+        models = tool._get_available_models()
+
+        # All native models should be present
+        native_models = [
+            "flash",
+            "pro",  # Gemini aliases
+            "o3",
+            "o3-mini",
+            "o3-pro",
+            "o4-mini",
+            "o4-mini-high",  # OpenAI models
+            "grok",
+            "grok-3",
+            "grok-3-fast",
+            "grok3",
+            "grokfast",  # X.AI models
+            "gemini-2.5-flash-preview-05-20",
+            "gemini-2.5-pro-preview-06-05",  # Full Gemini names
+        ]
+
+        for model in native_models:
+            assert model in models, f"Native model {model} should always be in enum"
+
+    def test_openrouter_models_with_api_key(self):
+        """Test that OpenRouter models are included when API key is configured."""
+        self._setup_environment({"OPENROUTER_API_KEY": "test-key"})
+
+        tool = AnalyzeTool()
+        models = tool._get_available_models()
+
+        # Check for some known OpenRouter model aliases
+        openrouter_models = ["opus", "sonnet", "haiku", "mistral-large", "deepseek"]
+        found_count = sum(1 for m in openrouter_models if m in models)
+
+        assert found_count >= 3, f"Expected at least 3 OpenRouter models, found {found_count}"
+        assert len(models) > 20, f"With OpenRouter, should have many models, got {len(models)}"
+
+    def test_openrouter_models_without_api_key(self):
+        """Test that OpenRouter models are NOT included when API key is not configured."""
+        self._setup_environment({})  # No OpenRouter key
+
+        tool = AnalyzeTool()
+        models = tool._get_available_models()
+
+        # OpenRouter-specific models should NOT be present
+        openrouter_only_models = ["opus", "sonnet", "haiku"]
+        found_count = sum(1 for m in openrouter_only_models if m in models)
+
+        assert found_count == 0, "OpenRouter models should not be included without API key"
+
+    def test_custom_models_with_custom_url(self):
+        """Test that custom models are included when CUSTOM_API_URL is configured."""
+        self._setup_environment({"CUSTOM_API_URL": "http://localhost:11434"})
+
+        tool = AnalyzeTool()
+        models = tool._get_available_models()
+
+        # Check for custom models (marked with is_custom=true)
+        custom_models = ["local-llama", "llama3.2"]
+        found_count = sum(1 for m in custom_models if m in models)
+
+        assert found_count >= 1, f"Expected at least 1 custom model, found {found_count}"
+
+    def test_custom_models_without_custom_url(self):
+        """Test that custom models are NOT included when CUSTOM_API_URL is not configured."""
+        self._setup_environment({})  # No custom URL
+
+        tool = AnalyzeTool()
+        models = tool._get_available_models()
+
+        # Custom-only models should NOT be present
+        custom_only_models = ["local-llama", "llama3.2"]
+        found_count = sum(1 for m in custom_only_models if m in models)
+
+        assert found_count == 0, "Custom models should not be included without CUSTOM_API_URL"
+
+    def test_all_providers_combined(self):
+        """Test that all models are included when all providers are configured."""
+        self._setup_environment(
+            {
+                "GEMINI_API_KEY": "test-key",
+                "OPENAI_API_KEY": "test-key",
+                "XAI_API_KEY": "test-key",
+                "OPENROUTER_API_KEY": "test-key",
+                "CUSTOM_API_URL": "http://localhost:11434",
+            }
+        )
+
+        tool = AnalyzeTool()
+        models = tool._get_available_models()
+
+        # Should have all types of models
+        assert "flash" in models  # Gemini
+        assert "o3" in models  # OpenAI
+        assert "grok" in models  # X.AI
+        assert "opus" in models or "sonnet" in models  # OpenRouter
+        assert "local-llama" in models or "llama3.2" in models  # Custom
+
+        # Should have many models total
+        assert len(models) > 50, f"With all providers, should have 50+ models, got {len(models)}"
+
+        # No duplicates
+        assert len(models) == len(set(models)), "Should have no duplicate models"
+
+    def test_mixed_provider_combinations(self):
+        """Test various mixed provider configurations."""
+        test_cases = [
+            # (provider_config, expected_model_samples, min_count)
+            (
+                {"GEMINI_API_KEY": "test", "OPENROUTER_API_KEY": "test"},
+                ["flash", "pro", "opus"],  # Gemini + OpenRouter models
+                30,
+            ),
+            (
+                {"OPENAI_API_KEY": "test", "CUSTOM_API_URL": "http://localhost"},
+                ["o3", "o4-mini", "local-llama"],  # OpenAI + Custom models
+                18,  # 14 native + ~4 custom models
+            ),
+            (
+                {"XAI_API_KEY": "test", "OPENROUTER_API_KEY": "test"},
+                ["grok", "grok-3", "opus"],  # X.AI + OpenRouter models
+                30,
+            ),
+        ]
+
+        for provider_config, expected_samples, min_count in test_cases:
+            self._setup_environment(provider_config)
+
+            tool = AnalyzeTool()
+            models = tool._get_available_models()
+
+            # Check expected models are present
+            for model in expected_samples:
+                if model in ["local-llama", "llama3.2"]:  # Custom models might not all be present
+                    continue
+                assert model in models, f"Expected {model} with config {provider_config}"
+
+            # Check minimum count
+            assert (
+                len(models) >= min_count
+            ), f"Expected at least {min_count} models with {provider_config}, got {len(models)}"
+
+    def test_no_duplicates_with_overlapping_providers(self):
+        """Test that models aren't duplicated when multiple providers offer the same model."""
+        self._setup_environment(
+            {
+                "OPENAI_API_KEY": "test",
+                "OPENROUTER_API_KEY": "test",  # OpenRouter also offers OpenAI models
+            }
+        )
+
+        tool = AnalyzeTool()
+        models = tool._get_available_models()
+
+        # Count occurrences of each model
+        model_counts = {}
+        for model in models:
+            model_counts[model] = model_counts.get(model, 0) + 1
+
+        # Check no duplicates
+        duplicates = {m: count for m, count in model_counts.items() if count > 1}
+        assert len(duplicates) == 0, f"Found duplicate models: {duplicates}"
+
+    def test_schema_enum_matches_get_available_models(self):
+        """Test that the schema enum matches what _get_available_models returns."""
+        self._setup_environment({"OPENROUTER_API_KEY": "test", "CUSTOM_API_URL": "http://localhost:11434"})
+
+        tool = AnalyzeTool()
+
+        # Get models from both methods
+        available_models = tool._get_available_models()
+        schema = tool.get_input_schema()
+        schema_enum = schema["properties"]["model"]["enum"]
+
+        # They should match exactly
+        assert set(available_models) == set(schema_enum), "Schema enum should match _get_available_models output"
+        assert len(available_models) == len(schema_enum), "Should have same number of models (no duplicates)"
+
+    @pytest.mark.parametrize(
+        "model_name,should_exist",
+        [
+            ("flash", True),  # Native Gemini
+            ("o3", True),  # Native OpenAI
+            ("grok", True),  # Native X.AI
+            ("gemini-2.5-flash-preview-05-20", True),  # Full native name
+            ("o4-mini-high", True),  # Native OpenAI variant
+            ("grok-3-fast", True),  # Native X.AI variant
+        ],
+    )
+    def test_specific_native_models_always_present(self, model_name, should_exist):
+        """Test that specific native models are always present regardless of configuration."""
+        self._setup_environment({})  # No providers
+
+        tool = AnalyzeTool()
+        models = tool._get_available_models()
+
+        if should_exist:
+            assert model_name in models, f"Native model {model_name} should always be present"
+        else:
+            assert model_name not in models, f"Model {model_name} should not be present"