From 68901e1e401c558d1c8d9c772c9c6623508938c9 Mon Sep 17 00:00:00 2001 From: John Reynolds Date: Thu, 5 Oct 2023 15:47:24 -0700 Subject: [PATCH] Update output_parser.py (#10430) - Description: Updated output parser for mrkl to remove any hallucination actions after the final answer; this was encountered when using Anthropic claude v2 for planning; reopening PR with updated unit tests - Issue: #10278 - Dependencies: N/A - Twitter handle: @johnreynolds --- .../langchain/agents/mrkl/output_parser.py | 13 ++++++++++-- .../agents/test_mrkl_output_parser.py | 20 ++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/libs/langchain/langchain/agents/mrkl/output_parser.py b/libs/langchain/langchain/agents/mrkl/output_parser.py index 37c95f46f8..60b1f58b46 100644 --- a/libs/langchain/langchain/agents/mrkl/output_parser.py +++ b/libs/langchain/langchain/agents/mrkl/output_parser.py @@ -29,11 +29,20 @@ class MRKLOutputParser(AgentOutputParser): r"Action\s*\d*\s*:[\s]*(.*?)[\s]*Action\s*\d*\s*Input\s*\d*\s*:[\s]*(.*)" ) action_match = re.search(regex, text, re.DOTALL) - if action_match: - if includes_answer: + if action_match and includes_answer: + if text.find(FINAL_ANSWER_ACTION) < text.find(action_match.group(0)): + # if final answer is before the hallucination, return final answer + start_index = text.find(FINAL_ANSWER_ACTION) + len(FINAL_ANSWER_ACTION) + end_index = text.find("\n\n", start_index) + return AgentFinish( + {"output": text[start_index:end_index].strip()}, text[:end_index] + ) + else: raise OutputParserException( f"{FINAL_ANSWER_AND_PARSABLE_ACTION_ERROR_MESSAGE}: {text}" ) + + if action_match: action = action_match.group(1).strip() action_input = action_match.group(2) tool_input = action_input.strip(" ") diff --git a/libs/langchain/tests/unit_tests/agents/test_mrkl_output_parser.py b/libs/langchain/tests/unit_tests/agents/test_mrkl_output_parser.py index 08a7f128a4..8774451df1 100644 --- a/libs/langchain/tests/unit_tests/agents/test_mrkl_output_parser.py +++ b/libs/langchain/tests/unit_tests/agents/test_mrkl_output_parser.py @@ -52,11 +52,25 @@ def test_missing_action_input() -> None: ) -def test_final_answer_and_parsable_action() -> None: - llm_output = """Final Answer: The best pizza to eat is margaritta - I can use the `foo` tool to achieve the goal. +def test_final_answer_before_parsable_action() -> None: + llm_output = """Final Answer: The best pizza to eat is margaritta + + Action: foo + Action Input: bar + """ + agent_finish: AgentFinish = mrkl_output_parser.parse(llm_output) # type: ignore + assert ( + agent_finish.return_values.get("output") + == "The best pizza to eat is margaritta" + ) + + +def test_final_answer_after_parsable_action() -> None: + llm_output = """ + Observation: I can use the `foo` tool to achieve the goal. Action: foo Action Input: bar + Final Answer: The best pizza to eat is margaritta """ with pytest.raises(OutputParserException) as exception_info: mrkl_output_parser.parse(llm_output)