remove prep (#2136)

agents should be stateless or async stuff may not work
doc
Harrison Chase 1 year ago committed by GitHub
parent a0453ebcf5
commit f5a4bf0ce4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -52,15 +52,19 @@
"\n", "\n",
"\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n", "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
"\u001b[32;1m\u001b[1;3m\n", "\u001b[32;1m\u001b[1;3m\n",
"Thought 1: I need to search David Chanoff and find the U.S. Navy admiral he collaborated\n", "Thought: I need to search David Chanoff and find the U.S. Navy admiral he collaborated with. Then I need to find which President the admiral served under.\n",
"with.\n", "\n",
"Action 1: Search[David Chanoff]\u001b[0m\n", "Action: Search[David Chanoff]\n",
"Observation 1: \u001b[36;1m\u001b[1;3mDavid Chanoff is a noted author of non-fiction work. His work has typically involved collaborations with the principal protagonist of the work concerned. His collaborators have included; Augustus A. White, Joycelyn Elders, Đoàn Văn Toại, William J. Crowe, Ariel Sharon, Kenneth Good and Felix Zandman. He has also written about a wide range of subjects including literary history, education and foreign for The Washington Post, The New Republic and The New York Times Magazine. He has published more than twelve books.\u001b[0m\n", "\u001b[0m\n",
"Thought 2:\u001b[32;1m\u001b[1;3m The U.S. Navy admiral David Chanoff collaborated with is William J. Crowe.\n", "Observation: \u001b[36;1m\u001b[1;3mDavid Chanoff is a noted author of non-fiction work. His work has typically involved collaborations with the principal protagonist of the work concerned. His collaborators have included; Augustus A. White, Joycelyn Elders, Đoàn Văn Toại, William J. Crowe, Ariel Sharon, Kenneth Good and Felix Zandman. He has also written about a wide range of subjects including literary history, education and foreign for The Washington Post, The New Republic and The New York Times Magazine. He has published more than twelve books.\u001b[0m\n",
"Action 2: Search[William J. Crowe]\u001b[0m\n", "Thought:\u001b[32;1m\u001b[1;3m The U.S. Navy admiral David Chanoff collaborated with is William J. Crowe. I need to find which President he served under.\n",
"Observation 2: \u001b[36;1m\u001b[1;3mWilliam James Crowe Jr. (January 2, 1925 October 18, 2007) was a United States Navy admiral and diplomat who served as the 11th chairman of the Joint Chiefs of Staff under Presidents Ronald Reagan and George H. W. Bush, and as the ambassador to the United Kingdom and Chair of the Intelligence Oversight Board under President Bill Clinton.\u001b[0m\n", "\n",
"Thought 3:\u001b[32;1m\u001b[1;3m The President William J. Crowe served as the ambassador to the United Kingdom under is Bill Clinton.\n", "Action: Search[William J. Crowe]\n",
"Action 3: Finish[Bill Clinton]\u001b[0m\n", "\u001b[0m\n",
"Observation: \u001b[36;1m\u001b[1;3mWilliam James Crowe Jr. (January 2, 1925 October 18, 2007) was a United States Navy admiral and diplomat who served as the 11th chairman of the Joint Chiefs of Staff under Presidents Ronald Reagan and George H. W. Bush, and as the ambassador to the United Kingdom and Chair of the Intelligence Oversight Board under President Bill Clinton.\u001b[0m\n",
"Thought:\u001b[32;1m\u001b[1;3m William J. Crowe served as the ambassador to the United Kingdom under President Bill Clinton, so the answer is Bill Clinton.\n",
"\n",
"Action: Finish[Bill Clinton]\u001b[0m\n",
"\n", "\n",
"\u001b[1m> Finished chain.\u001b[0m\n" "\u001b[1m> Finished chain.\u001b[0m\n"
] ]
@ -84,7 +88,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "cc4a6efe", "id": "09604a7f",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [] "source": []

@ -135,10 +135,6 @@ class Agent(BaseModel):
full_inputs = {**kwargs, **new_inputs} full_inputs = {**kwargs, **new_inputs}
return full_inputs return full_inputs
def prepare_for_new_call(self) -> None:
"""Prepare the agent for new call, if needed."""
pass
@property @property
def finish_tool_name(self) -> str: def finish_tool_name(self) -> str:
"""Name of the tool to use to finish the chain.""" """Name of the tool to use to finish the chain."""
@ -493,8 +489,6 @@ class AgentExecutor(Chain, BaseModel):
def _call(self, inputs: Dict[str, str]) -> Dict[str, Any]: def _call(self, inputs: Dict[str, str]) -> Dict[str, Any]:
"""Run text through and get agent response.""" """Run text through and get agent response."""
# Do any preparation necessary when receiving a new input.
self.agent.prepare_for_new_call()
# Construct a mapping of tool name to tool for easy lookup # Construct a mapping of tool name to tool for easy lookup
name_to_tool_map = {tool.name: tool for tool in self.tools} name_to_tool_map = {tool.name: tool for tool in self.tools}
# We construct a mapping from each tool to a color, used for logging. # We construct a mapping from each tool to a color, used for logging.
@ -525,8 +519,6 @@ class AgentExecutor(Chain, BaseModel):
async def _acall(self, inputs: Dict[str, str]) -> Dict[str, str]: async def _acall(self, inputs: Dict[str, str]) -> Dict[str, str]:
"""Run text through and get agent response.""" """Run text through and get agent response."""
# Do any preparation necessary when receiving a new input.
self.agent.prepare_for_new_call()
# Construct a mapping of tool name to tool for easy lookup # Construct a mapping of tool name to tool for easy lookup
name_to_tool_map = {tool.name: tool for tool in self.tools} name_to_tool_map = {tool.name: tool for tool in self.tools}
# We construct a mapping from each tool to a color, used for logging. # We construct a mapping from each tool to a color, used for logging.

@ -28,8 +28,6 @@ class ReActDocstoreAgent(Agent, BaseModel):
"""Return default prompt.""" """Return default prompt."""
return WIKI_PROMPT return WIKI_PROMPT
i: int = 1
@classmethod @classmethod
def _validate_tools(cls, tools: Sequence[BaseTool]) -> None: def _validate_tools(cls, tools: Sequence[BaseTool]) -> None:
if len(tools) != 2: if len(tools) != 2:
@ -40,18 +38,14 @@ class ReActDocstoreAgent(Agent, BaseModel):
f"Tool names should be Lookup and Search, got {tool_names}" f"Tool names should be Lookup and Search, got {tool_names}"
) )
def _prepare_for_new_call(self) -> None:
self.i = 1
def _fix_text(self, text: str) -> str: def _fix_text(self, text: str) -> str:
return text + f"\nAction {self.i}:" return text + "\nAction:"
def _extract_tool_and_input(self, text: str) -> Optional[Tuple[str, str]]: def _extract_tool_and_input(self, text: str) -> Optional[Tuple[str, str]]:
action_prefix = f"Action {self.i}: " action_prefix = "Action: "
if not text.split("\n")[-1].startswith(action_prefix): if not text.strip().split("\n")[-1].startswith(action_prefix):
return None return None
self.i += 1 action_block = text.strip().split("\n")[-1]
action_block = text.split("\n")[-1]
action_str = action_block[len(action_prefix) :] action_str = action_block[len(action_prefix) :]
# Parse out the action and the directive. # Parse out the action and the directive.
@ -68,16 +62,16 @@ class ReActDocstoreAgent(Agent, BaseModel):
@property @property
def observation_prefix(self) -> str: def observation_prefix(self) -> str:
"""Prefix to append the observation with.""" """Prefix to append the observation with."""
return f"Observation {self.i - 1}: " return "Observation: "
@property @property
def _stop(self) -> List[str]: def _stop(self) -> List[str]:
return [f"\nObservation {self.i}:"] return ["\nObservation:"]
@property @property
def llm_prefix(self) -> str: def llm_prefix(self) -> str:
"""Prefix to append the LLM call with.""" """Prefix to append the LLM call with."""
return f"Thought {self.i}:" return "Thought:"
class DocstoreExplorer: class DocstoreExplorer:

@ -16,9 +16,9 @@ should have stuff on it?
You don't like doors? Why not try going east, that entranceway is unguarded. You don't like doors? Why not try going east, that entranceway is unguarded.
Thought 1: I need to travel east Thought: I need to travel east
Action 1: Play[go east] Action: Play[go east]
Observation 1: -= Office =- Observation: -= Office =-
You arrive in an office. An ordinary one. You arrive in an office. An ordinary one.
You can make out a locker. The locker contains a binder. You see a case. The You can make out a locker. The locker contains a binder. You see a case. The
@ -29,18 +29,18 @@ be empty. If you haven't noticed it already, there seems to be something there
by the wall, it's a table. Unfortunately, there isn't a thing on it. Hm. Oh well by the wall, it's a table. Unfortunately, there isn't a thing on it. Hm. Oh well
There is an exit to the west. Don't worry, it is unguarded. There is an exit to the west. Don't worry, it is unguarded.
Thought 2: I need to take the binder from the locker Thought: I need to take the binder from the locker
Action 2: Play[take binder] Action: Play[take binder]
Observation 2: You take the binder from the locker. Observation: You take the binder from the locker.
Thought 3: I need to place the binder on the mantelpiece Thought: I need to place the binder on the mantelpiece
Action 3: Play[put binder on mantelpiece] Action: Play[put binder on mantelpiece]
Observation 3: You put the binder on the mantelpiece. Observation: You put the binder on the mantelpiece.
Your score has just gone up by one point. Your score has just gone up by one point.
*** The End *** *** The End ***
Thought 4: The End has occurred Thought: The End has occurred
Action 4: Finish[yes] Action: Finish[yes]
""" """
] ]

@ -2,110 +2,66 @@
from langchain.prompts.prompt import PromptTemplate from langchain.prompts.prompt import PromptTemplate
EXAMPLES = [ EXAMPLES = [
"""Question: What is the elevation range for the area that the eastern sector of the """Question: What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into?
Colorado orogeny extends into? Thought: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado orogeny extends into, then find the elevation range of the area.
Thought 1: I need to search Colorado orogeny, find the area that the eastern sector Action: Search[Colorado orogeny]
of the Colorado orogeny extends into, then find the elevation range of the Observation: The Colorado orogeny was an episode of mountain building (an orogeny) in Colorado and surrounding areas.
area. Thought: It does not mention the eastern sector. So I need to look up eastern sector.
Action 1: Search[Colorado orogeny] Action: Lookup[eastern sector]
Observation 1: The Colorado orogeny was an episode of mountain building (an orogeny) in Observation: (Result 1 / 1) The eastern sector extends into the High Plains and is called the Central Plains orogeny.
Colorado and surrounding areas. Thought: The eastern sector of Colorado orogeny extends into the High Plains. So I need to search High Plains and find its elevation range.
Thought 2: It does not mention the eastern sector. So I need to look up eastern Action: Search[High Plains]
sector. Observation: High Plains refers to one of two distinct land regions
Action 2: Lookup[eastern sector] Thought: I need to instead search High Plains (United States).
Observation 2: (Result 1 / 1) The eastern sector extends into the High Plains and is called Action: Search[High Plains (United States)]
the Central Plains orogeny. Observation: The High Plains are a subregion of the Great Plains. From east to west, the High Plains rise in elevation from around 1,800 to 7,000 ft (550 to 2,130 m).[3]
Thought 3: The eastern sector of Colorado orogeny extends into the High Plains. So I Thought: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft.
need to search High Plains and find its elevation range. Action: Finish[1,800 to 7,000 ft]""",
Action 3: Search[High Plains] """Question: Musician and satirist Allie Goertz wrote a song about the "The Simpsons" character Milhouse, who Matt Groening named after who?
Observation 3: High Plains refers to one of two distinct land regions Thought: The question simplifies to "The Simpsons" character Milhouse is named after who. I only need to search Milhouse and find who it is named after.
Thought 4: I need to instead search High Plains (United States). Action: Search[Milhouse]
Action 4: Search[High Plains (United States)] Observation: Milhouse Mussolini Van Houten is a recurring character in the Fox animated television series The Simpsons voiced by Pamela Hayden and created by Matt Groening.
Observation 4: The High Plains are a subregion of the Great Plains. From east to west, the Thought: The paragraph does not tell who Milhouse is named after, maybe I can look up "named after".
High Plains rise in elevation from around 1,800 to 7,000 ft (550 to 2,130 Action: Lookup[named after]
m).[3] Observation: (Result 1 / 1) Milhouse was named after U.S. president Richard Nixon, whose middle name was Milhous.
Thought 5: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer Thought: Milhouse was named after U.S. president Richard Nixon, so the answer is Richard Nixon.
is 1,800 to 7,000 ft. Action: Finish[Richard Nixon]""",
Action 5: Finish[1,800 to 7,000 ft]""", """Question: Which documentary is about Finnish rock groups, Adam Clayton Powell or The Saimaa Gesture?
"""Question: Musician and satirist Allie Goertz wrote a song about the "The Simpsons" Thought: I need to search Adam Clayton Powell and The Saimaa Gesture, and find which documentary is about Finnish rock groups.
character Milhouse, who Matt Groening named after who? Action: Search[Adam Clayton Powell]
Thought 1: The question simplifies to "The Simpsons" character Milhouse is named after Observation: Could not find [Adam Clayton Powell]. Similar: [Adam Clayton Powell III, Seventh Avenue (Manhattan), Adam Clayton Powell Jr. State Office Building, Isabel Washington Powell, Adam Powell, Adam Clayton Powell (film), Giancarlo Esposito].
who. I only need to search Milhouse and find who it is named after. Thought: To find the documentary, I can search Adam Clayton Powell (film).
Action 1: Search[Milhouse] Action: Search[Adam Clayton Powell (film)]
Observation 1: Milhouse Mussolini Van Houten is a recurring character in the Fox animated Observation: Adam Clayton Powell is a 1989 American documentary film directed by Richard Kilberg. The film is about the rise and fall of influential African-American politician Adam Clayton Powell Jr.[3][4] It was later aired as part of the PBS series The American Experience.
television series The Simpsons voiced by Pamela Hayden and created by Matt Thought: Adam Clayton Powell (film) is a documentary about an African-American politician, not Finnish rock groups. So the documentary about Finnish rock groups must instead be The Saimaa Gesture.
Groening. Action: Finish[The Saimaa Gesture]""",
Thought 2: The paragraph does not tell who Milhouse is named after, maybe I can look up
"named after".
Action 2: Lookup[named after]
Observation 2: (Result 1 / 1) Milhouse was named after U.S. president Richard Nixon, whose
middle name was Milhous.
Thought 3: Milhouse was named after U.S. president Richard Nixon, so the answer is
Richard Nixon.
Action 3: Finish[Richard Nixon]""",
"""Question: Which documentary is about Finnish rock groups, Adam Clayton Powell or The
Saimaa Gesture?
Thought 1: I need to search Adam Clayton Powell and The Saimaa Gesture, and find which
documentary is about Finnish rock groups.
Action 1: Search[Adam Clayton Powell]
Observation 1 Could not find [Adam Clayton Powell]. Similar: [Adam Clayton Powell
III, Seventh Avenue (Manhattan), Adam Clayton Powell Jr. State Office
Building, Isabel Washington Powell, Adam Powell, Adam Clayton Powell
(film), Giancarlo Esposito].
Thought 2: To find the documentary, I can search Adam Clayton Powell (film).
Action 2: Search[Adam Clayton Powell (film)]
Observation 2: Adam Clayton Powell is a 1989 American documentary film directed by
Richard Kilberg. The film is about the rise and fall of influential
African-American politician Adam Clayton Powell Jr.[3][4] It was later aired
as part of the PBS series The American Experience.
Thought 3: Adam Clayton Powell (film) is a documentary about an African-American
politician, not Finnish rock groups. So the documentary about Finnish rock
groups must instead be The Saimaa Gesture.
Action 3: Finish[The Saimaa Gesture]""",
"""Question: What profession does Nicholas Ray and Elia Kazan have in common? """Question: What profession does Nicholas Ray and Elia Kazan have in common?
Thought 1: I need to search Nicholas Ray and Elia Kazan, find their professions, then Thought: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common.
find the profession they have in common. Action: Search[Nicholas Ray]
Action 1: Search[Nicholas Ray] Observation: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 - June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause.
Observation 1: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 - June 16, Thought: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions.
1979) was an American film director, screenwriter, and actor best known for Action: Search[Elia Kazan]
the 1955 film Rebel Without a Cause. Observation: Elia Kazan was an American film and theatre director, producer, screenwriter and actor.
Thought 2: Professions of Nicholas Ray are director, screenwriter, and actor. I need Thought: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor.
to search Elia Kazan next and find his professions. Action: Finish[director, screenwriter, actor]""",
Action 2: Search[Elia Kazan]
Observation 2: Elia Kazan was an American film and theatre director, producer, screenwriter
and actor.
Thought 3: Professions of Elia Kazan are director, producer, screenwriter, and actor.
So profession Nicholas Ray and Elia Kazan have in common is director,
screenwriter, and actor.
Action 3: Finish[director, screenwriter, actor]""",
"""Question: Which magazine was started first Arthurs Magazine or First for Women? """Question: Which magazine was started first Arthurs Magazine or First for Women?
Thought 1: I need to search Arthurs Magazine and First for Women, and find which was Thought: I need to search Arthurs Magazine and First for Women, and find which was started first.
started first. Action: Search[Arthurs Magazine]
Action 1: Search[Arthurs Magazine] Observation: Arthurs Magazine (1844-1846) was an American literary periodical published in Philadelphia in the 19th century.
Observation 1: Arthurs Magazine (1844-1846) was an American literary periodical published Thought: Arthurs Magazine was started in 1844. I need to search First for Women next.
in Philadelphia in the 19th century. Action: Search[First for Women]
Thought 2: Arthurs Magazine was started in 1844. I need to search First for Women Observation: First for Women is a womans magazine published by Bauer Media Group in the USA.[1] The magazine was started in 1989.
next. Thought: First for Women was started in 1989. 1844 (Arthurs Magazine) < 1989 (First for Women), so Arthurs Magazine was started first.
Action 2: Search[First for Women] Action: Finish[Arthurs Magazine]""",
Observation 2: First for Women is a womans magazine published by Bauer Media Group in the
USA.[1] The magazine was started in 1989.
Thought 3: First for Women was started in 1989. 1844 (Arthurs Magazine) < 1989 (First
for Women), so Arthurs Magazine was started first.
Action 3: Finish[Arthurs Magazine]""",
"""Question: Were Pavel Urysohn and Leonid Levin known for the same type of work? """Question: Were Pavel Urysohn and Leonid Levin known for the same type of work?
Thought 1: I need to search Pavel Urysohn and Leonid Levin, find their types of work, Thought: I need to search Pavel Urysohn and Leonid Levin, find their types of work, then find if they are the same.
then find if they are the same. Action: Search[Pavel Urysohn]
Action 1: Search[Pavel Urysohn] Observation: Pavel Samuilovich Urysohn (February 3, 1898 - August 17, 1924) was a Soviet mathematician who is best known for his contributions in dimension theory.
Observation 1: Pavel Samuilovich Urysohn (February 3, 1898 - August 17, 1924) was a Soviet Thought: Pavel Urysohn is a mathematician. I need to search Leonid Levin next and find its type of work.
mathematician who is best known for his contributions in dimension theory. Action: Search[Leonid Levin]
Thought 2: Pavel Urysohn is a mathematician. I need to search Leonid Levin next and Observation: Leonid Anatolievich Levin is a Soviet-American mathematician and computer scientist.
find its type of work. Thought: Leonid Levin is a mathematician and computer scientist. So Pavel Urysohn and Leonid Levin have the same type of work.
Action 2: Search[Leonid Levin] Action: Finish[yes]""",
Observation 2: Leonid Anatolievich Levin is a Soviet-American mathematician and computer
scientist.
Thought 3: Leonid Levin is a mathematician and computer scientist. So Pavel Urysohn
and Leonid Levin have the same type of work.
Action 3: Finish[yes]""",
] ]
SUFFIX = """\nQuestion: {input} SUFFIX = """\nQuestion: {input}
{agent_scratchpad}""" {agent_scratchpad}"""

@ -55,7 +55,7 @@ class FakeDocstore(Docstore):
def test_predict_until_observation_normal() -> None: def test_predict_until_observation_normal() -> None:
"""Test predict_until_observation when observation is made normally.""" """Test predict_until_observation when observation is made normally."""
outputs = ["foo\nAction 1: Search[foo]"] outputs = ["foo\nAction: Search[foo]"]
fake_llm = FakeListLLM(responses=outputs) fake_llm = FakeListLLM(responses=outputs)
tools = [ tools = [
Tool(name="Search", func=lambda x: x, description="foo"), Tool(name="Search", func=lambda x: x, description="foo"),
@ -77,16 +77,16 @@ def test_predict_until_observation_repeat() -> None:
] ]
agent = ReActDocstoreAgent.from_llm_and_tools(fake_llm, tools) agent = ReActDocstoreAgent.from_llm_and_tools(fake_llm, tools)
output = agent.plan([], input="") output = agent.plan([], input="")
expected_output = AgentAction("Search", "foo", "foo\nAction 1: Search[foo]") expected_output = AgentAction("Search", "foo", "foo\nAction: Search[foo]")
assert output == expected_output assert output == expected_output
def test_react_chain() -> None: def test_react_chain() -> None:
"""Test react chain.""" """Test react chain."""
responses = [ responses = [
"I should probably search\nAction 1: Search[langchain]", "I should probably search\nAction: Search[langchain]",
"I should probably lookup\nAction 2: Lookup[made]", "I should probably lookup\nAction: Lookup[made]",
"Ah okay now I know the answer\nAction 3: Finish[2022]", "Ah okay now I know the answer\nAction: Finish[2022]",
] ]
fake_llm = FakeListLLM(responses=responses) fake_llm = FakeListLLM(responses=responses)
react_chain = ReActChain(llm=fake_llm, docstore=FakeDocstore()) react_chain = ReActChain(llm=fake_llm, docstore=FakeDocstore())
@ -98,8 +98,8 @@ def test_react_chain_bad_action() -> None:
"""Test react chain when bad action given.""" """Test react chain when bad action given."""
bad_action_name = "BadAction" bad_action_name = "BadAction"
responses = [ responses = [
f"I'm turning evil\nAction 1: {bad_action_name}[langchain]", f"I'm turning evil\nAction: {bad_action_name}[langchain]",
"Oh well\nAction 2: Finish[curses foiled again]", "Oh well\nAction: Finish[curses foiled again]",
] ]
fake_llm = FakeListLLM(responses=responses) fake_llm = FakeListLLM(responses=responses)
react_chain = ReActChain(llm=fake_llm, docstore=FakeDocstore()) react_chain = ReActChain(llm=fake_llm, docstore=FakeDocstore())

Loading…
Cancel
Save