Improved challenge prompt and instructions
This commit is contained in:
@@ -14,7 +14,7 @@ import os
|
||||
# These values are used in server responses and for tracking releases
|
||||
# IMPORTANT: This is the single source of truth for version and author info
|
||||
# Semantic versioning: MAJOR.MINOR.PATCH
|
||||
__version__ = "5.8.0"
|
||||
__version__ = "5.8.1"
|
||||
# Last update date in ISO format
|
||||
__updated__ = "2025-06-30"
|
||||
# Primary maintainer
|
||||
|
||||
@@ -25,8 +25,8 @@ class TestChallengeTool:
|
||||
"""Test that tool metadata matches requirements"""
|
||||
assert self.tool.get_name() == "challenge"
|
||||
assert "CRITICAL CHALLENGE PROMPT" in self.tool.get_description()
|
||||
assert "challenge it thoughtfully" in self.tool.get_description()
|
||||
assert "agreeing by default" in self.tool.get_description()
|
||||
assert "critical examination" in self.tool.get_description()
|
||||
assert "default agreement" in self.tool.get_description()
|
||||
assert self.tool.get_default_temperature() == 0.2 # TEMPERATURE_ANALYTICAL
|
||||
|
||||
def test_requires_model(self):
|
||||
@@ -100,11 +100,11 @@ class TestChallengeTool:
|
||||
|
||||
# Check that the challenge prompt contains critical thinking instructions
|
||||
challenge_prompt = response_data["challenge_prompt"]
|
||||
assert "CHALLENGE THIS STATEMENT - Do not automatically agree" in challenge_prompt
|
||||
assert "Is this actually correct? Check carefully" in challenge_prompt
|
||||
assert "CRITICAL REASSESSMENT – Do not automatically agree" in challenge_prompt
|
||||
assert "Carefully evaluate the statement above" in challenge_prompt
|
||||
assert response_data["original_statement"] in challenge_prompt
|
||||
assert "you must say so" in challenge_prompt
|
||||
assert "Provide your honest assessment, not automatic agreement" in challenge_prompt
|
||||
assert "flaws, gaps, or misleading points" in challenge_prompt
|
||||
assert "thoughtful analysis" in challenge_prompt
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_error_handling(self):
|
||||
@@ -124,11 +124,11 @@ class TestChallengeTool:
|
||||
wrapped = self.tool._wrap_prompt_for_challenge(original_prompt)
|
||||
|
||||
# Check structure
|
||||
assert "CHALLENGE THIS STATEMENT - Do not automatically agree" in wrapped
|
||||
assert "Is this actually correct? Check carefully" in wrapped
|
||||
assert "CRITICAL REASSESSMENT – Do not automatically agree" in wrapped
|
||||
assert "Carefully evaluate the statement above" in wrapped
|
||||
assert f'"{original_prompt}"' in wrapped
|
||||
assert "you must say so" in wrapped
|
||||
assert "Provide your honest assessment, not automatic agreement" in wrapped
|
||||
assert "flaws, gaps, or misleading points" in wrapped
|
||||
assert "thoughtful analysis" in wrapped
|
||||
|
||||
def test_multiple_prompts(self):
|
||||
"""Test that tool handles various types of prompts correctly"""
|
||||
@@ -146,7 +146,7 @@ class TestChallengeTool:
|
||||
|
||||
# Each wrapped prompt should contain the original
|
||||
assert prompt in wrapped
|
||||
assert "CHALLENGE THIS STATEMENT" in wrapped
|
||||
assert "CRITICAL REASSESSMENT" in wrapped
|
||||
|
||||
def test_tool_fields(self):
|
||||
"""Test tool-specific field definitions"""
|
||||
@@ -155,7 +155,7 @@ class TestChallengeTool:
|
||||
assert "prompt" in fields
|
||||
assert fields["prompt"]["type"] == "string"
|
||||
assert "statement" in fields["prompt"]["description"]
|
||||
assert "challenge" in fields["prompt"]["description"]
|
||||
assert "critically evaluate" in fields["prompt"]["description"]
|
||||
|
||||
def test_required_fields_list(self):
|
||||
"""Test required fields list"""
|
||||
|
||||
@@ -23,8 +23,10 @@ from .simple.base import SimpleTool
|
||||
# Field descriptions for the Challenge tool
|
||||
CHALLENGE_FIELD_DESCRIPTIONS = {
|
||||
"prompt": (
|
||||
"The statement, question, or assertion the user wants to challenge critically. "
|
||||
"This may be a claim, suggestion, or idea that requires thoughtful reconsideration, not automatic agreement."
|
||||
"The original statement, question, or assertion the user intends to critically evaluate. "
|
||||
"This may be a claim, suggestion, or idea that requires thoughtful analysis, not automatic agreement. "
|
||||
"MANDATORY: Do NOT rephrase or paraphrase. Submit the statement EXACTLY AS PROVIDED, as it will be used "
|
||||
"verbatim in a new prompt."
|
||||
),
|
||||
}
|
||||
|
||||
@@ -53,11 +55,11 @@ class ChallengeTool(SimpleTool):
|
||||
|
||||
def get_description(self) -> str:
|
||||
return (
|
||||
"CRITICAL CHALLENGE PROMPT – Use this to frame your statement in a way that prompts "
|
||||
"the CLI agent to challenge it thoughtfully instead of agreeing by default. Ideal for "
|
||||
"challenging assumptions, validating ideas, and seeking honest, analytical feedback as part of an ongoing "
|
||||
"task. The tool wraps your input with instructions explicitly telling the agent to think critically "
|
||||
"and disagree if warranted."
|
||||
"CRITICAL CHALLENGE PROMPT – Use this to explicitly frame your input for critical examination. "
|
||||
"Instead of default agreement, the CLI agent will be instructed to analyze, question, and, if needed, "
|
||||
"disagree with your statement. Ideal for validating assumptions, pressure-testing ideas, and eliciting "
|
||||
"honest feedback during complex workflows. The tool ensures your input is wrapped with clear guidance "
|
||||
"to encourage thoughtful scrutiny."
|
||||
)
|
||||
|
||||
def get_system_prompt(self) -> str:
|
||||
@@ -133,8 +135,9 @@ class ChallengeTool(SimpleTool):
|
||||
"challenge_prompt": wrapped_prompt,
|
||||
"instructions": (
|
||||
"Present the challenge_prompt to yourself and follow its instructions. "
|
||||
"Challenge the statement critically before forming your response. "
|
||||
"If you disagree after careful reconsideration, explain why."
|
||||
"Reassess the statement carefully and critically before responding. "
|
||||
"If, after reflection, you find reasons to disagree or qualify it, explain your reasoning. "
|
||||
"Likewise, if you find reasons to agree, articulate them clearly and justify your agreement."
|
||||
),
|
||||
}
|
||||
|
||||
@@ -165,11 +168,12 @@ class ChallengeTool(SimpleTool):
|
||||
The statement wrapped in challenge instructions
|
||||
"""
|
||||
return (
|
||||
f"CHALLENGE THIS STATEMENT - Do not automatically agree:\n\n"
|
||||
f"CRITICAL REASSESSMENT – Do not automatically agree:\n\n"
|
||||
f'"{prompt}"\n\n'
|
||||
f"Is this actually correct? Check carefully. If it's wrong, incomplete, misleading or incorrect, "
|
||||
f"you must say so. Provide your honest assessment, not automatic agreement. If you "
|
||||
f"feel there is merit in what the user is saying, explain WHY you agree."
|
||||
f"Carefully evaluate the statement above. Is it accurate, complete, and well-reasoned? "
|
||||
f"Investigate if needed before replying, and stay focused. If you identify flaws, gaps, or misleading "
|
||||
f"points, explain them clearly. Likewise, if you find the reasoning sound, explain why it holds up. "
|
||||
f"Respond with thoughtful analysis—stay to the point and avoid reflexive agreement."
|
||||
)
|
||||
|
||||
# Required method implementations from SimpleTool
|
||||
|
||||
Reference in New Issue
Block a user