Improved challenge prompt and instructions

This commit is contained in:
Fahad
2025-06-29 17:52:00 +04:00
parent 26170efa65
commit a54343dc79
3 changed files with 30 additions and 26 deletions

View File

@@ -14,7 +14,7 @@ import os
# These values are used in server responses and for tracking releases # These values are used in server responses and for tracking releases
# IMPORTANT: This is the single source of truth for version and author info # IMPORTANT: This is the single source of truth for version and author info
# Semantic versioning: MAJOR.MINOR.PATCH # Semantic versioning: MAJOR.MINOR.PATCH
__version__ = "5.8.0" __version__ = "5.8.1"
# Last update date in ISO format # Last update date in ISO format
__updated__ = "2025-06-30" __updated__ = "2025-06-30"
# Primary maintainer # Primary maintainer

View File

@@ -25,8 +25,8 @@ class TestChallengeTool:
"""Test that tool metadata matches requirements""" """Test that tool metadata matches requirements"""
assert self.tool.get_name() == "challenge" assert self.tool.get_name() == "challenge"
assert "CRITICAL CHALLENGE PROMPT" in self.tool.get_description() assert "CRITICAL CHALLENGE PROMPT" in self.tool.get_description()
assert "challenge it thoughtfully" in self.tool.get_description() assert "critical examination" in self.tool.get_description()
assert "agreeing by default" in self.tool.get_description() assert "default agreement" in self.tool.get_description()
assert self.tool.get_default_temperature() == 0.2 # TEMPERATURE_ANALYTICAL assert self.tool.get_default_temperature() == 0.2 # TEMPERATURE_ANALYTICAL
def test_requires_model(self): def test_requires_model(self):
@@ -100,11 +100,11 @@ class TestChallengeTool:
# Check that the challenge prompt contains critical thinking instructions # Check that the challenge prompt contains critical thinking instructions
challenge_prompt = response_data["challenge_prompt"] challenge_prompt = response_data["challenge_prompt"]
assert "CHALLENGE THIS STATEMENT - Do not automatically agree" in challenge_prompt assert "CRITICAL REASSESSMENT Do not automatically agree" in challenge_prompt
assert "Is this actually correct? Check carefully" in challenge_prompt assert "Carefully evaluate the statement above" in challenge_prompt
assert response_data["original_statement"] in challenge_prompt assert response_data["original_statement"] in challenge_prompt
assert "you must say so" in challenge_prompt assert "flaws, gaps, or misleading points" in challenge_prompt
assert "Provide your honest assessment, not automatic agreement" in challenge_prompt assert "thoughtful analysis" in challenge_prompt
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_execute_error_handling(self): async def test_execute_error_handling(self):
@@ -124,11 +124,11 @@ class TestChallengeTool:
wrapped = self.tool._wrap_prompt_for_challenge(original_prompt) wrapped = self.tool._wrap_prompt_for_challenge(original_prompt)
# Check structure # Check structure
assert "CHALLENGE THIS STATEMENT - Do not automatically agree" in wrapped assert "CRITICAL REASSESSMENT Do not automatically agree" in wrapped
assert "Is this actually correct? Check carefully" in wrapped assert "Carefully evaluate the statement above" in wrapped
assert f'"{original_prompt}"' in wrapped assert f'"{original_prompt}"' in wrapped
assert "you must say so" in wrapped assert "flaws, gaps, or misleading points" in wrapped
assert "Provide your honest assessment, not automatic agreement" in wrapped assert "thoughtful analysis" in wrapped
def test_multiple_prompts(self): def test_multiple_prompts(self):
"""Test that tool handles various types of prompts correctly""" """Test that tool handles various types of prompts correctly"""
@@ -146,7 +146,7 @@ class TestChallengeTool:
# Each wrapped prompt should contain the original # Each wrapped prompt should contain the original
assert prompt in wrapped assert prompt in wrapped
assert "CHALLENGE THIS STATEMENT" in wrapped assert "CRITICAL REASSESSMENT" in wrapped
def test_tool_fields(self): def test_tool_fields(self):
"""Test tool-specific field definitions""" """Test tool-specific field definitions"""
@@ -155,7 +155,7 @@ class TestChallengeTool:
assert "prompt" in fields assert "prompt" in fields
assert fields["prompt"]["type"] == "string" assert fields["prompt"]["type"] == "string"
assert "statement" in fields["prompt"]["description"] assert "statement" in fields["prompt"]["description"]
assert "challenge" in fields["prompt"]["description"] assert "critically evaluate" in fields["prompt"]["description"]
def test_required_fields_list(self): def test_required_fields_list(self):
"""Test required fields list""" """Test required fields list"""

View File

@@ -23,8 +23,10 @@ from .simple.base import SimpleTool
# Field descriptions for the Challenge tool # Field descriptions for the Challenge tool
CHALLENGE_FIELD_DESCRIPTIONS = { CHALLENGE_FIELD_DESCRIPTIONS = {
"prompt": ( "prompt": (
"The statement, question, or assertion the user wants to challenge critically. " "The original statement, question, or assertion the user intends to critically evaluate. "
"This may be a claim, suggestion, or idea that requires thoughtful reconsideration, not automatic agreement." "This may be a claim, suggestion, or idea that requires thoughtful analysis, not automatic agreement. "
"MANDATORY: Do NOT rephrase or paraphrase. Submit the statement EXACTLY AS PROVIDED, as it will be used "
"verbatim in a new prompt."
), ),
} }
@@ -53,11 +55,11 @@ class ChallengeTool(SimpleTool):
def get_description(self) -> str: def get_description(self) -> str:
return ( return (
"CRITICAL CHALLENGE PROMPT Use this to frame your statement in a way that prompts " "CRITICAL CHALLENGE PROMPT Use this to explicitly frame your input for critical examination. "
"the CLI agent to challenge it thoughtfully instead of agreeing by default. Ideal for " "Instead of default agreement, the CLI agent will be instructed to analyze, question, and, if needed, "
"challenging assumptions, validating ideas, and seeking honest, analytical feedback as part of an ongoing " "disagree with your statement. Ideal for validating assumptions, pressure-testing ideas, and eliciting "
"task. The tool wraps your input with instructions explicitly telling the agent to think critically " "honest feedback during complex workflows. The tool ensures your input is wrapped with clear guidance "
"and disagree if warranted." "to encourage thoughtful scrutiny."
) )
def get_system_prompt(self) -> str: def get_system_prompt(self) -> str:
@@ -133,8 +135,9 @@ class ChallengeTool(SimpleTool):
"challenge_prompt": wrapped_prompt, "challenge_prompt": wrapped_prompt,
"instructions": ( "instructions": (
"Present the challenge_prompt to yourself and follow its instructions. " "Present the challenge_prompt to yourself and follow its instructions. "
"Challenge the statement critically before forming your response. " "Reassess the statement carefully and critically before responding. "
"If you disagree after careful reconsideration, explain why." "If, after reflection, you find reasons to disagree or qualify it, explain your reasoning. "
"Likewise, if you find reasons to agree, articulate them clearly and justify your agreement."
), ),
} }
@@ -165,11 +168,12 @@ class ChallengeTool(SimpleTool):
The statement wrapped in challenge instructions The statement wrapped in challenge instructions
""" """
return ( return (
f"CHALLENGE THIS STATEMENT - Do not automatically agree:\n\n" f"CRITICAL REASSESSMENT Do not automatically agree:\n\n"
f'"{prompt}"\n\n' f'"{prompt}"\n\n'
f"Is this actually correct? Check carefully. If it's wrong, incomplete, misleading or incorrect, " f"Carefully evaluate the statement above. Is it accurate, complete, and well-reasoned? "
f"you must say so. Provide your honest assessment, not automatic agreement. If you " f"Investigate if needed before replying, and stay focused. If you identify flaws, gaps, or misleading "
f"feel there is merit in what the user is saying, explain WHY you agree." f"points, explain them clearly. Likewise, if you find the reasoning sound, explain why it holds up. "
f"Respond with thoughtful analysis—stay to the point and avoid reflexive agreement."
) )
# Required method implementations from SimpleTool # Required method implementations from SimpleTool