Harrison/return intermediate steps (#428)

harrison/combine-docs-parse
Harrison Chase 1 year ago committed by GitHub
parent 7fc4b4b3e1
commit b7566b5ec3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -183,6 +183,51 @@
"chain({\"input_documents\": docs, \"question\": query}, return_only_outputs=True)"
]
},
{
"cell_type": "markdown",
"id": "ae2f6d97",
"metadata": {},
"source": [
"**Intermediate Steps**\n",
"\n",
"We can also return the intermediate steps for `map_reduce` chains, should we want to inspect them. This is done with the `return_map_steps` variable."
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "15af265f",
"metadata": {},
"outputs": [],
"source": [
"chain = load_qa_with_sources_chain(OpenAI(temperature=0), chain_type=\"map_reduce\", return_map_steps=True)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "21b136e5",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'map_steps': [{'text': ' \"Tonight, Id like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service.\"'},\n",
" {'text': ' None'},\n",
" {'text': ' None'},\n",
" {'text': ' None'}],\n",
" 'output_text': ' The president thanked Justice Breyer for his service.\\nSOURCES: 30-pl'}"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"chain({\"input_documents\": docs, \"question\": query}, return_only_outputs=True)"
]
},
{
"cell_type": "markdown",
"id": "5bf0e1ab",
@ -225,10 +270,55 @@
"chain({\"input_documents\": docs, \"question\": query}, return_only_outputs=True)"
]
},
{
"cell_type": "markdown",
"id": "ac357530",
"metadata": {},
"source": [
"**Intermediate Steps**\n",
"\n",
"We can also return the intermediate steps for `refine` chains, should we want to inspect them. This is done with the `return_refine_steps` variable."
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "3396a773",
"metadata": {},
"outputs": [],
"source": [
"chain = load_qa_with_sources_chain(OpenAI(temperature=0), chain_type=\"refine\", return_refine_steps=True)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "be5739ef",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'refine_steps': ['\\nThe president said that he was honoring Justice Breyer for his dedication to serving the country and that he was a retiring Justice of the United States Supreme Court.',\n",
" \"\\n\\nThe president said that he was honoring Justice Breyer for his dedication to serving the country, his career as a top litigator in private practice, a former federal public defender, and his family of public school educators and police officers. He also noted Justice Breyer's consensus-building skills and the broad range of support he has received from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. He also highlighted the importance of advancing liberty and justice by securing the border and fixing the immigration system, noting the new technology and joint patrols with Mexico and Guatemala to catch more human traffickers, as well as the dedicated immigration judges and commitments to support partners in South and Central America to host more refugees and secure their own borders. \\nSource: 31\",\n",
" \"\\n\\nThe president said that he was honoring Justice Breyer for his dedication to serving the country, his career as a top litigator in private practice, a former federal public defender, and his family of public school educators and police officers. He also noted Justice Breyer's consensus-building skills and the broad range of support he has received from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. He also highlighted the importance of advancing liberty and justice by securing the border and fixing the immigration system, noting the new technology and joint patrols with Mexico and Guatemala to catch more human traffickers, as well as the dedicated immigration judges and commitments to support partners in South and Central America to host more refugees and secure their own borders. Additionally, he mentioned the need for the bipartisan Equality Act to be passed and signed into law, and the importance of strengthening the Violence Against Women Act. He also offered a Unity Agenda for the Nation, which includes beating the opioid epidemic. \\nSource: 31, 33\",\n",
" \"\\n\\nThe president said that he was honoring Justice Breyer for his dedication to serving the country, his career as a top litigator in private practice, a former federal public defender, and his family of public school educators and police officers. He also noted Justice Breyer's consensus-building skills and the broad range of support he has received from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. He also highlighted the importance of advancing liberty and justice by securing the border and fixing the immigration system, noting the new technology and joint patrols with Mexico and Guatemala to catch more human traffickers, as well as the dedicated immigration judges and commitments to support partners in South and Central America to host more refugees and secure their own borders. Additionally, he mentioned the need for the bipartisan Equality Act to be passed and signed into law, and the importance of strengthening the Violence Against Women Act. He also offered a Unity Agenda for the Nation, which includes beating the opioid epidemic, and announced that the Justice Department will name a chief prosecutor for pandemic fraud. Source: 31, 33, 20\"],\n",
" 'output_text': \"\\n\\nThe president said that he was honoring Justice Breyer for his dedication to serving the country, his career as a top litigator in private practice, a former federal public defender, and his family of public school educators and police officers. He also noted Justice Breyer's consensus-building skills and the broad range of support he has received from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. He also highlighted the importance of advancing liberty and justice by securing the border and fixing the immigration system, noting the new technology and joint patrols with Mexico and Guatemala to catch more human traffickers, as well as the dedicated immigration judges and commitments to support partners in South and Central America to host more refugees and secure their own borders. Additionally, he mentioned the need for the bipartisan Equality Act to be passed and signed into law, and the importance of strengthening the Violence Against Women Act. He also offered a Unity Agenda for the Nation, which includes beating the opioid epidemic, and announced that the Justice Department will name a chief prosecutor for pandemic fraud. Source: 31, 33, 20\"}"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"chain({\"input_documents\": docs, \"question\": query}, return_only_outputs=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "929620d0",
"id": "7355fedd",
"metadata": {},
"outputs": [],
"source": []

@ -173,6 +173,51 @@
"chain({\"input_documents\": docs, \"question\": query}, return_only_outputs=True)"
]
},
{
"cell_type": "markdown",
"id": "31478d32",
"metadata": {},
"source": [
"**Intermediate Steps**\n",
"\n",
"We can also return the intermediate steps for `map_reduce` chains, should we want to inspect them. This is done with the `return_map_steps` variable."
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "452c8680",
"metadata": {},
"outputs": [],
"source": [
"chain = load_qa_chain(OpenAI(temperature=0), chain_type=\"map_reduce\", return_map_steps=True)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "90b47a75",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'map_steps': [{'text': ' \"Tonight, Id like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service.\"'},\n",
" {'text': ' None'},\n",
" {'text': ' None'},\n",
" {'text': ' None'}],\n",
" 'output_text': ' The president said, \"Justice Breyer, thank you for your service.\"'}"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"chain({\"input_documents\": docs, \"question\": query}, return_only_outputs=True)"
]
},
{
"cell_type": "markdown",
"id": "6ea50ad0",
@ -215,10 +260,55 @@
"chain({\"input_documents\": docs, \"question\": query}, return_only_outputs=True)"
]
},
{
"cell_type": "markdown",
"id": "f95dfb2e",
"metadata": {},
"source": [
"**Intermediate Steps**\n",
"\n",
"We can also return the intermediate steps for `refine` chains, should we want to inspect them. This is done with the `return_refine_steps` variable."
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "a5c64200",
"metadata": {},
"outputs": [],
"source": [
"chain = load_qa_chain(OpenAI(temperature=0), chain_type=\"refine\", return_refine_steps=True)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "817546ac",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'refine_steps': ['\\nThe president said that he wanted to honor Justice Breyer for his dedication to serving the country and his legacy of excellence.',\n",
" '\\n\\nThe president said that he wanted to honor Justice Breyer for his dedication to serving the country, his legacy of excellence, and his commitment to advancing liberty and justice.',\n",
" '\\n\\nThe president said that he wanted to honor Justice Breyer for his dedication to serving the country, his legacy of excellence, and his commitment to advancing liberty and justice, as well as for his commitment to protecting the rights of LGBTQ+ Americans and his support for the bipartisan Equality Act.',\n",
" '\\n\\nThe president said that he wanted to honor Justice Breyer for his dedication to serving the country, his legacy of excellence, and his commitment to advancing liberty and justice, as well as for his commitment to protecting the rights of LGBTQ+ Americans and his support for the bipartisan Equality Act. He also mentioned his plan to lower costs to give families a fair shot, lower the deficit, and go after criminals who stole pandemic relief funds. He also announced that the Justice Department will name a chief prosecutor for pandemic fraud.'],\n",
" 'output_text': '\\n\\nThe president said that he wanted to honor Justice Breyer for his dedication to serving the country, his legacy of excellence, and his commitment to advancing liberty and justice, as well as for his commitment to protecting the rights of LGBTQ+ Americans and his support for the bipartisan Equality Act. He also mentioned his plan to lower costs to give families a fair shot, lower the deficit, and go after criminals who stole pandemic relief funds. He also announced that the Justice Department will name a chief prosecutor for pandemic fraud.'}"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"chain({\"input_documents\": docs, \"question\": query}, return_only_outputs=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "49e9c6d7",
"id": "97d335c6",
"metadata": {},
"outputs": [],
"source": []

@ -160,6 +160,50 @@
"chain.run(docs)"
]
},
{
"cell_type": "markdown",
"id": "d0c2a6d3",
"metadata": {},
"source": [
"**Intermediate Steps**\n",
"\n",
"We can also return the intermediate steps for `map_reduce` chains, should we want to inspect them. This is done with the `return_map_steps` variable."
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "d9cfc24e",
"metadata": {},
"outputs": [],
"source": [
"chain = load_summarize_chain(OpenAI(temperature=0), chain_type=\"map_reduce\", return_map_steps=True)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "c7dff5e8",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'map_steps': [{'text': \" In response to Russia's aggression in Ukraine, the United States has united with other freedom-loving nations to impose economic sanctions and hold Putin accountable. The U.S. Department of Justice is also assembling a task force to go after the crimes of Russian oligarchs and seize their ill-gotten gains.\"},\n",
" {'text': ' The United States and its European allies are taking action to punish Russia for its invasion of Ukraine, including seizing assets, closing off airspace, and providing economic and military assistance to Ukraine. The US is also mobilizing forces to protect NATO countries and has released 30 million barrels of oil from its Strategic Petroleum Reserve to help blunt gas prices. The world is uniting in support of Ukraine and democracy, and the US stands with its Ukrainian allies.'},\n",
" {'text': \" President Biden and Vice President Harris ran for office with a new economic vision for America, and have since passed the American Rescue Plan and the Bipartisan Infrastructure Law to help struggling families and rebuild America's infrastructure. This includes creating jobs, modernizing roads, airports, ports, and waterways, replacing lead pipes, providing affordable high-speed internet, and investing in American products to support American jobs.\"}],\n",
" 'output_text': \" In response to Russia's aggression in Ukraine, the United States and its allies have imposed economic sanctions and are taking other measures to hold Putin accountable. The US is also providing economic and military assistance to Ukraine, protecting NATO countries, and passing legislation to help struggling families and rebuild America's infrastructure. The world is uniting in support of Ukraine and democracy, and the US stands with its Ukrainian allies.\"}"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"chain({\"input_documents\": docs}, return_only_outputs=True)"
]
},
{
"cell_type": "markdown",
"id": "f61350f9",
@ -201,10 +245,54 @@
"chain.run(docs)"
]
},
{
"cell_type": "markdown",
"id": "84e9567e",
"metadata": {},
"source": [
"**Intermediate Steps**\n",
"\n",
"We can also return the intermediate steps for `refine` chains, should we want to inspect them. This is done with the `return_refine_steps` variable."
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "cd49ac4d",
"metadata": {},
"outputs": [],
"source": [
"chain = load_summarize_chain(OpenAI(temperature=0), chain_type=\"refine\", return_refine_steps=True)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "6a74029d",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'refine_steps': [\" In response to Russia's aggression in Ukraine, the United States has united with other freedom-loving nations to impose economic sanctions and hold Putin accountable. The U.S. Department of Justice is also assembling a task force to go after the crimes of Russian oligarchs and seize their ill-gotten gains.\",\n",
" \"\\n\\nIn response to Russia's aggression in Ukraine, the United States has united with other freedom-loving nations to impose economic sanctions and hold Putin accountable. The U.S. Department of Justice is also assembling a task force to go after the crimes of Russian oligarchs and seize their ill-gotten gains. We are joining with our European allies to find and seize the assets of Russian oligarchs, including yachts, luxury apartments, and private jets. The U.S. is also closing off American airspace to all Russian flights, further isolating Russia and adding an additional squeeze on their economy. The U.S. and its allies are providing support to the Ukrainians in their fight for freedom, including military, economic, and humanitarian assistance. The U.S. is also mobilizing ground forces, air squadrons, and ship deployments to protect NATO countries. The U.S. and its allies are also releasing 60 million barrels of oil from reserves around the world, with the U.S. contributing 30 million barrels from its own Strategic Petroleum Reserve. Putin's war on Ukraine has left Russia weaker and the rest of the world stronger, with the world uniting in support of democracy and peace.\",\n",
" \"\\n\\nIn response to Russia's aggression in Ukraine, the United States has united with other freedom-loving nations to impose economic sanctions and hold Putin accountable. The U.S. Department of Justice is also assembling a task force to go after the crimes of Russian oligarchs and seize their ill-gotten gains. We are joining with our European allies to find and seize the assets of Russian oligarchs, including yachts, luxury apartments, and private jets. The U.S. is also closing off American airspace to all Russian flights, further isolating Russia and adding an additional squeeze on their economy. The U.S. and its allies are providing support to the Ukrainians in their fight for freedom, including military, economic, and humanitarian assistance. The U.S. is also mobilizing ground forces, air squadrons, and ship deployments to protect NATO countries. The U.S. and its allies are also releasing 60 million barrels of oil from reserves around the world, with the U.S. contributing 30 million barrels from its own Strategic Petroleum Reserve. In addition, the U.S. has passed the American Rescue Plan to provide immediate economic relief for tens of millions of Americans, and the Bipartisan Infrastructure Law to rebuild America and create jobs. This includes investing\"],\n",
" 'output_text': \"\\n\\nIn response to Russia's aggression in Ukraine, the United States has united with other freedom-loving nations to impose economic sanctions and hold Putin accountable. The U.S. Department of Justice is also assembling a task force to go after the crimes of Russian oligarchs and seize their ill-gotten gains. We are joining with our European allies to find and seize the assets of Russian oligarchs, including yachts, luxury apartments, and private jets. The U.S. is also closing off American airspace to all Russian flights, further isolating Russia and adding an additional squeeze on their economy. The U.S. and its allies are providing support to the Ukrainians in their fight for freedom, including military, economic, and humanitarian assistance. The U.S. is also mobilizing ground forces, air squadrons, and ship deployments to protect NATO countries. The U.S. and its allies are also releasing 60 million barrels of oil from reserves around the world, with the U.S. contributing 30 million barrels from its own Strategic Petroleum Reserve. In addition, the U.S. has passed the American Rescue Plan to provide immediate economic relief for tens of millions of Americans, and the Bipartisan Infrastructure Law to rebuild America and create jobs. This includes investing\"}"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"chain({\"input_documents\": docs}, return_only_outputs=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0da92750",
"id": "db1ed69d",
"metadata": {},
"outputs": [],
"source": []

@ -1,7 +1,7 @@
"""Base interface for chains combining documents."""
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Tuple
from pydantic import BaseModel
@ -39,12 +39,13 @@ class BaseCombineDocumentsChain(Chain, BaseModel, ABC):
return None
@abstractmethod
def combine_docs(self, docs: List[Document], **kwargs: Any) -> str:
def combine_docs(self, docs: List[Document], **kwargs: Any) -> Tuple[str, dict]:
"""Combine documents into a single string."""
def _call(self, inputs: Dict[str, Any]) -> Dict[str, str]:
docs = inputs[self.input_key]
# Other keys are assumed to be needed for LLM prediction
other_keys = {k: v for k, v in inputs.items() if k != self.input_key}
output = self.combine_docs(docs, **other_keys)
return {self.output_key: output}
output, extra_return_dict = self.combine_docs(docs, **other_keys)
extra_return_dict[self.output_key] = output
return extra_return_dict

@ -2,7 +2,7 @@
from __future__ import annotations
from typing import Any, Callable, Dict, List, Optional
from typing import Any, Callable, Dict, List, Optional, Tuple
from pydantic import BaseModel, Extra, root_validator
@ -65,6 +65,19 @@ class MapReduceDocumentsChain(BaseCombineDocumentsChain, BaseModel):
document_variable_name: str
"""The variable name in the llm_chain to put the documents in.
If only one variable in the llm_chain, this need not be provided."""
return_map_steps: bool = False
"""Return the results of the map steps in the output."""
@property
def output_keys(self) -> List[str]:
"""Expect input key.
:meta private:
"""
_output_keys = super().output_keys
if self.return_map_steps:
_output_keys = _output_keys + ["map_steps"]
return _output_keys
class Config:
"""Configuration for this pydantic object."""
@ -102,7 +115,7 @@ class MapReduceDocumentsChain(BaseCombineDocumentsChain, BaseModel):
def combine_docs(
self, docs: List[Document], token_max: int = 3000, **kwargs: Any
) -> str:
) -> Tuple[str, dict]:
"""Combine documents in a map reduce manner.
Combine by mapping first chain over all documents, then reducing the results.
@ -133,5 +146,9 @@ class MapReduceDocumentsChain(BaseCombineDocumentsChain, BaseModel):
num_tokens = self.combine_document_chain.prompt_length(
result_docs, **kwargs
)
output = self.combine_document_chain.combine_docs(result_docs, **kwargs)
return output
if self.return_map_steps:
extra_return_dict = {"map_steps": results}
else:
extra_return_dict = {}
output, _ = self.combine_document_chain.combine_docs(result_docs, **kwargs)
return output, extra_return_dict

@ -2,7 +2,7 @@
from __future__ import annotations
from typing import Any, Dict, List
from typing import Any, Dict, List, Tuple
from pydantic import BaseModel, Extra, Field, root_validator
@ -33,6 +33,19 @@ class RefineDocumentsChain(BaseCombineDocumentsChain, BaseModel):
default_factory=_get_default_document_prompt
)
"""Prompt to use to format each document."""
return_refine_steps: bool = False
"""Return the results of the refine steps in the output."""
@property
def output_keys(self) -> List[str]:
"""Expect input key.
:meta private:
"""
_output_keys = super().output_keys
if self.return_refine_steps:
_output_keys = _output_keys + ["refine_steps"]
return _output_keys
class Config:
"""Configuration for this pydantic object."""
@ -61,7 +74,7 @@ class RefineDocumentsChain(BaseCombineDocumentsChain, BaseModel):
)
return values
def combine_docs(self, docs: List[Document], **kwargs: Any) -> str:
def combine_docs(self, docs: List[Document], **kwargs: Any) -> Tuple[str, dict]:
"""Combine by mapping first chain over all, then stuffing into final chain."""
base_info = {"page_content": docs[0].page_content}
base_info.update(docs[0].metadata)
@ -71,6 +84,7 @@ class RefineDocumentsChain(BaseCombineDocumentsChain, BaseModel):
}
inputs = {**base_inputs, **kwargs}
res = self.initial_llm_chain.predict(**inputs)
refine_steps = [res]
for doc in docs[1:]:
base_info = {"page_content": doc.page_content}
base_info.update(doc.metadata)
@ -85,4 +99,9 @@ class RefineDocumentsChain(BaseCombineDocumentsChain, BaseModel):
}
inputs = {**base_inputs, **kwargs}
res = self.refine_llm_chain.predict(**inputs)
return res
refine_steps.append(res)
if self.return_refine_steps:
extra_return_dict = {"refine_steps": refine_steps}
else:
extra_return_dict = {}
return res, extra_return_dict

@ -1,6 +1,6 @@
"""Chain that combines documents by stuffing into context."""
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Tuple
from pydantic import BaseModel, Extra, Field, root_validator
@ -78,8 +78,8 @@ class StuffDocumentsChain(BaseCombineDocumentsChain, BaseModel):
prompt = self.llm_chain.prompt.format(**inputs)
return self.llm_chain.llm.get_num_tokens(prompt)
def combine_docs(self, docs: List[Document], **kwargs: Any) -> str:
def combine_docs(self, docs: List[Document], **kwargs: Any) -> Tuple[str, dict]:
"""Stuff all documents into one prompt and pass to LLM."""
inputs = self._get_inputs(docs, **kwargs)
# Call predict on the LLM.
return self.llm_chain.predict(**inputs)
return self.llm_chain.predict(**inputs), {}

@ -70,5 +70,5 @@ class MapReduceChain(Chain, BaseModel):
# Split the larger text into smaller chunks.
texts = self.text_splitter.split_text(inputs[self.input_key])
docs = [Document(page_content=text) for text in texts]
outputs = self.combine_documents_chain.combine_docs(docs)
outputs, _ = self.combine_documents_chain.combine_docs(docs)
return {self.output_key: outputs}

@ -106,7 +106,7 @@ class BaseQAWithSourcesChain(Chain, BaseModel, ABC):
def _call(self, inputs: Dict[str, Any]) -> Dict[str, str]:
docs = self._get_docs(inputs)
answer = self.combine_document_chain.combine_docs(docs, **inputs)
answer, _ = self.combine_document_chain.combine_docs(docs, **inputs)
if "\nSOURCES: " in answer:
answer, sources = answer.split("\nSOURCES: ")
else:

@ -101,5 +101,5 @@ class VectorDBQA(Chain, BaseModel):
def _call(self, inputs: Dict[str, str]) -> Dict[str, str]:
question = inputs[self.input_key]
docs = self.vectorstore.similarity_search(question, k=self.k)
answer = self.combine_documents_chain.combine_docs(docs, question=question)
answer, _ = self.combine_documents_chain.combine_docs(docs, question=question)
return {self.output_key: answer}

Loading…
Cancel
Save