Improved challenge prompt and instructions
This commit is contained in:
@@ -14,7 +14,7 @@ import os
|
|||||||
# These values are used in server responses and for tracking releases
|
# These values are used in server responses and for tracking releases
|
||||||
# IMPORTANT: This is the single source of truth for version and author info
|
# IMPORTANT: This is the single source of truth for version and author info
|
||||||
# Semantic versioning: MAJOR.MINOR.PATCH
|
# Semantic versioning: MAJOR.MINOR.PATCH
|
||||||
__version__ = "5.8.0"
|
__version__ = "5.8.1"
|
||||||
# Last update date in ISO format
|
# Last update date in ISO format
|
||||||
__updated__ = "2025-06-30"
|
__updated__ = "2025-06-30"
|
||||||
# Primary maintainer
|
# Primary maintainer
|
||||||
|
|||||||
@@ -25,8 +25,8 @@ class TestChallengeTool:
|
|||||||
"""Test that tool metadata matches requirements"""
|
"""Test that tool metadata matches requirements"""
|
||||||
assert self.tool.get_name() == "challenge"
|
assert self.tool.get_name() == "challenge"
|
||||||
assert "CRITICAL CHALLENGE PROMPT" in self.tool.get_description()
|
assert "CRITICAL CHALLENGE PROMPT" in self.tool.get_description()
|
||||||
assert "challenge it thoughtfully" in self.tool.get_description()
|
assert "critical examination" in self.tool.get_description()
|
||||||
assert "agreeing by default" in self.tool.get_description()
|
assert "default agreement" in self.tool.get_description()
|
||||||
assert self.tool.get_default_temperature() == 0.2 # TEMPERATURE_ANALYTICAL
|
assert self.tool.get_default_temperature() == 0.2 # TEMPERATURE_ANALYTICAL
|
||||||
|
|
||||||
def test_requires_model(self):
|
def test_requires_model(self):
|
||||||
@@ -100,11 +100,11 @@ class TestChallengeTool:
|
|||||||
|
|
||||||
# Check that the challenge prompt contains critical thinking instructions
|
# Check that the challenge prompt contains critical thinking instructions
|
||||||
challenge_prompt = response_data["challenge_prompt"]
|
challenge_prompt = response_data["challenge_prompt"]
|
||||||
assert "CHALLENGE THIS STATEMENT - Do not automatically agree" in challenge_prompt
|
assert "CRITICAL REASSESSMENT – Do not automatically agree" in challenge_prompt
|
||||||
assert "Is this actually correct? Check carefully" in challenge_prompt
|
assert "Carefully evaluate the statement above" in challenge_prompt
|
||||||
assert response_data["original_statement"] in challenge_prompt
|
assert response_data["original_statement"] in challenge_prompt
|
||||||
assert "you must say so" in challenge_prompt
|
assert "flaws, gaps, or misleading points" in challenge_prompt
|
||||||
assert "Provide your honest assessment, not automatic agreement" in challenge_prompt
|
assert "thoughtful analysis" in challenge_prompt
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_execute_error_handling(self):
|
async def test_execute_error_handling(self):
|
||||||
@@ -124,11 +124,11 @@ class TestChallengeTool:
|
|||||||
wrapped = self.tool._wrap_prompt_for_challenge(original_prompt)
|
wrapped = self.tool._wrap_prompt_for_challenge(original_prompt)
|
||||||
|
|
||||||
# Check structure
|
# Check structure
|
||||||
assert "CHALLENGE THIS STATEMENT - Do not automatically agree" in wrapped
|
assert "CRITICAL REASSESSMENT – Do not automatically agree" in wrapped
|
||||||
assert "Is this actually correct? Check carefully" in wrapped
|
assert "Carefully evaluate the statement above" in wrapped
|
||||||
assert f'"{original_prompt}"' in wrapped
|
assert f'"{original_prompt}"' in wrapped
|
||||||
assert "you must say so" in wrapped
|
assert "flaws, gaps, or misleading points" in wrapped
|
||||||
assert "Provide your honest assessment, not automatic agreement" in wrapped
|
assert "thoughtful analysis" in wrapped
|
||||||
|
|
||||||
def test_multiple_prompts(self):
|
def test_multiple_prompts(self):
|
||||||
"""Test that tool handles various types of prompts correctly"""
|
"""Test that tool handles various types of prompts correctly"""
|
||||||
@@ -146,7 +146,7 @@ class TestChallengeTool:
|
|||||||
|
|
||||||
# Each wrapped prompt should contain the original
|
# Each wrapped prompt should contain the original
|
||||||
assert prompt in wrapped
|
assert prompt in wrapped
|
||||||
assert "CHALLENGE THIS STATEMENT" in wrapped
|
assert "CRITICAL REASSESSMENT" in wrapped
|
||||||
|
|
||||||
def test_tool_fields(self):
|
def test_tool_fields(self):
|
||||||
"""Test tool-specific field definitions"""
|
"""Test tool-specific field definitions"""
|
||||||
@@ -155,7 +155,7 @@ class TestChallengeTool:
|
|||||||
assert "prompt" in fields
|
assert "prompt" in fields
|
||||||
assert fields["prompt"]["type"] == "string"
|
assert fields["prompt"]["type"] == "string"
|
||||||
assert "statement" in fields["prompt"]["description"]
|
assert "statement" in fields["prompt"]["description"]
|
||||||
assert "challenge" in fields["prompt"]["description"]
|
assert "critically evaluate" in fields["prompt"]["description"]
|
||||||
|
|
||||||
def test_required_fields_list(self):
|
def test_required_fields_list(self):
|
||||||
"""Test required fields list"""
|
"""Test required fields list"""
|
||||||
|
|||||||
@@ -23,8 +23,10 @@ from .simple.base import SimpleTool
|
|||||||
# Field descriptions for the Challenge tool
|
# Field descriptions for the Challenge tool
|
||||||
CHALLENGE_FIELD_DESCRIPTIONS = {
|
CHALLENGE_FIELD_DESCRIPTIONS = {
|
||||||
"prompt": (
|
"prompt": (
|
||||||
"The statement, question, or assertion the user wants to challenge critically. "
|
"The original statement, question, or assertion the user intends to critically evaluate. "
|
||||||
"This may be a claim, suggestion, or idea that requires thoughtful reconsideration, not automatic agreement."
|
"This may be a claim, suggestion, or idea that requires thoughtful analysis, not automatic agreement. "
|
||||||
|
"MANDATORY: Do NOT rephrase or paraphrase. Submit the statement EXACTLY AS PROVIDED, as it will be used "
|
||||||
|
"verbatim in a new prompt."
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,11 +55,11 @@ class ChallengeTool(SimpleTool):
|
|||||||
|
|
||||||
def get_description(self) -> str:
|
def get_description(self) -> str:
|
||||||
return (
|
return (
|
||||||
"CRITICAL CHALLENGE PROMPT – Use this to frame your statement in a way that prompts "
|
"CRITICAL CHALLENGE PROMPT – Use this to explicitly frame your input for critical examination. "
|
||||||
"the CLI agent to challenge it thoughtfully instead of agreeing by default. Ideal for "
|
"Instead of default agreement, the CLI agent will be instructed to analyze, question, and, if needed, "
|
||||||
"challenging assumptions, validating ideas, and seeking honest, analytical feedback as part of an ongoing "
|
"disagree with your statement. Ideal for validating assumptions, pressure-testing ideas, and eliciting "
|
||||||
"task. The tool wraps your input with instructions explicitly telling the agent to think critically "
|
"honest feedback during complex workflows. The tool ensures your input is wrapped with clear guidance "
|
||||||
"and disagree if warranted."
|
"to encourage thoughtful scrutiny."
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_system_prompt(self) -> str:
|
def get_system_prompt(self) -> str:
|
||||||
@@ -133,8 +135,9 @@ class ChallengeTool(SimpleTool):
|
|||||||
"challenge_prompt": wrapped_prompt,
|
"challenge_prompt": wrapped_prompt,
|
||||||
"instructions": (
|
"instructions": (
|
||||||
"Present the challenge_prompt to yourself and follow its instructions. "
|
"Present the challenge_prompt to yourself and follow its instructions. "
|
||||||
"Challenge the statement critically before forming your response. "
|
"Reassess the statement carefully and critically before responding. "
|
||||||
"If you disagree after careful reconsideration, explain why."
|
"If, after reflection, you find reasons to disagree or qualify it, explain your reasoning. "
|
||||||
|
"Likewise, if you find reasons to agree, articulate them clearly and justify your agreement."
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -165,11 +168,12 @@ class ChallengeTool(SimpleTool):
|
|||||||
The statement wrapped in challenge instructions
|
The statement wrapped in challenge instructions
|
||||||
"""
|
"""
|
||||||
return (
|
return (
|
||||||
f"CHALLENGE THIS STATEMENT - Do not automatically agree:\n\n"
|
f"CRITICAL REASSESSMENT – Do not automatically agree:\n\n"
|
||||||
f'"{prompt}"\n\n'
|
f'"{prompt}"\n\n'
|
||||||
f"Is this actually correct? Check carefully. If it's wrong, incomplete, misleading or incorrect, "
|
f"Carefully evaluate the statement above. Is it accurate, complete, and well-reasoned? "
|
||||||
f"you must say so. Provide your honest assessment, not automatic agreement. If you "
|
f"Investigate if needed before replying, and stay focused. If you identify flaws, gaps, or misleading "
|
||||||
f"feel there is merit in what the user is saying, explain WHY you agree."
|
f"points, explain them clearly. Likewise, if you find the reasoning sound, explain why it holds up. "
|
||||||
|
f"Respond with thoughtful analysis—stay to the point and avoid reflexive agreement."
|
||||||
)
|
)
|
||||||
|
|
||||||
# Required method implementations from SimpleTool
|
# Required method implementations from SimpleTool
|
||||||
|
|||||||
Reference in New Issue
Block a user