From 634ccb8ccd6ad3c181179252a972c353e97f940e Mon Sep 17 00:00:00 2001 From: Mohammad Mohtashim <45242107+keenborder786@users.noreply.github.com> Date: Tue, 17 Oct 2023 06:46:19 +0500 Subject: [PATCH] test_stream_log_retriever Unit Test + Tool names fix (#11808) ## Description | Tool | Original Tool Name | |-----------------------------|---------------------------| | open-meteo-api | Open Meteo API | | news-api | News API | | tmdb-api | TMDB API | | podcast-api | Podcast API | | golden_query | Golden Query | | dall-e-image-generator | Dall-E Image Generator | | twilio | Text Message | | searx_search_results | Searx Search Results | | dataforseo | DataForSeo Results JSON | When using these tools through `load_tools`, I encountered the following validation error: ```console openai.error.InvalidRequestError: 'TMDB API' does not match '^[a-zA-Z0-9_-]{1,64}$' - 'functions.0.name' ``` In order to avoid this error, I replaced spaces with hyphens in the tool names: | Tool | Corrected Tool Name | |-----------------------------|---------------------------| | open-meteo-api | Open-Meteo-API | | news-api | News-API | | tmdb-api | TMDB-API | | podcast-api | Podcast-API | | golden_query | Golden-Query | | dall-e-image-generator | Dall-E-Image-Generator | | twilio | Text-Message | | searx_search_results | Searx-Search-Results | | dataforseo | DataForSeo-Results-JSON | This correction resolved the validation error. Additionally, a unit test, `tests/unit_tests/schema/runnable/test_runnable.py::test_stream_log_retriever`, was failing at random. Upon further investigation, I confirmed that the failure was not related to the above-mentioned changes. The `stream_log` variable was generating the order of logs in two ways at random The reason for this behavior is unclear, but in the assertion, I included both possible orders to account for this variability. --- libs/langchain/langchain/agents/load_tools.py | 12 +- .../tools/dataforseo_api_search/tool.py | 2 +- .../langchain/tools/golden_query/tool.py | 2 +- .../langchain/tools/searx_search/tool.py | 2 +- .../schema/runnable/test_runnable.py | 414 ++++++++++++------ 5 files changed, 287 insertions(+), 145 deletions(-) diff --git a/libs/langchain/langchain/agents/load_tools.py b/libs/langchain/langchain/agents/load_tools.py index 0ff690605c..71166969d1 100644 --- a/libs/langchain/langchain/agents/load_tools.py +++ b/libs/langchain/langchain/agents/load_tools.py @@ -120,7 +120,7 @@ def _get_llm_math(llm: BaseLanguageModel) -> BaseTool: def _get_open_meteo_api(llm: BaseLanguageModel) -> BaseTool: chain = APIChain.from_llm_and_api_docs(llm, open_meteo_docs.OPEN_METEO_DOCS) return Tool( - name="Open Meteo API", + name="Open-Meteo-API", description="Useful for when you want to get weather information from the OpenMeteo API. The input should be a question in natural language that this API can answer.", func=chain.run, ) @@ -138,7 +138,7 @@ def _get_news_api(llm: BaseLanguageModel, **kwargs: Any) -> BaseTool: llm, news_docs.NEWS_DOCS, headers={"X-Api-Key": news_api_key} ) return Tool( - name="News API", + name="News-API", description="Use this when you want to get information about the top headlines of current news stories. The input should be a question in natural language that this API can answer.", func=chain.run, ) @@ -152,7 +152,7 @@ def _get_tmdb_api(llm: BaseLanguageModel, **kwargs: Any) -> BaseTool: headers={"Authorization": f"Bearer {tmdb_bearer_token}"}, ) return Tool( - name="TMDB API", + name="TMDB-API", description="Useful for when you want to get information from The Movie Database. The input should be a question in natural language that this API can answer.", func=chain.run, ) @@ -166,7 +166,7 @@ def _get_podcast_api(llm: BaseLanguageModel, **kwargs: Any) -> BaseTool: headers={"X-ListenAPI-Key": listen_api_key}, ) return Tool( - name="Podcast API", + name="Podcast-API", description="Use the Listen Notes Podcast API to search all podcasts or episodes. The input should be a question in natural language that this API can answer.", func=chain.run, ) @@ -235,7 +235,7 @@ def _get_serpapi(**kwargs: Any) -> BaseTool: def _get_dalle_image_generator(**kwargs: Any) -> Tool: return Tool( - "Dall-E Image Generator", + "Dall-E-Image-Generator", DallEAPIWrapper(**kwargs).run, "A wrapper around OpenAI DALL-E API. Useful for when you need to generate images from a text description. Input should be an image description.", ) @@ -243,7 +243,7 @@ def _get_dalle_image_generator(**kwargs: Any) -> Tool: def _get_twilio(**kwargs: Any) -> BaseTool: return Tool( - name="Text Message", + name="Text-Message", description="Useful for when you need to send a text message to a provided phone number.", func=TwilioAPIWrapper(**kwargs).run, ) diff --git a/libs/langchain/langchain/tools/dataforseo_api_search/tool.py b/libs/langchain/langchain/tools/dataforseo_api_search/tool.py index 6b2cf5a3e9..6d6509999d 100644 --- a/libs/langchain/langchain/tools/dataforseo_api_search/tool.py +++ b/libs/langchain/langchain/tools/dataforseo_api_search/tool.py @@ -43,7 +43,7 @@ class DataForSeoAPISearchResults(BaseTool): """Tool that queries the DataForSeo Google Search API and get back json.""" - name: str = "DataForSeo Results JSON" + name: str = "DataForSeo-Results-JSON" description: str = ( "A comprehensive Google Search API provided by DataForSeo." "This tool is useful for obtaining real-time data on current events " diff --git a/libs/langchain/langchain/tools/golden_query/tool.py b/libs/langchain/langchain/tools/golden_query/tool.py index c84bec9188..81bc95d103 100644 --- a/libs/langchain/langchain/tools/golden_query/tool.py +++ b/libs/langchain/langchain/tools/golden_query/tool.py @@ -10,7 +10,7 @@ from langchain.utilities.golden_query import GoldenQueryAPIWrapper class GoldenQueryRun(BaseTool): """Tool that adds the capability to query using the Golden API and get back JSON.""" - name: str = "Golden Query" + name: str = "Golden-Query" description: str = ( "A wrapper around Golden Query API." " Useful for getting entities that match" diff --git a/libs/langchain/langchain/tools/searx_search/tool.py b/libs/langchain/langchain/tools/searx_search/tool.py index 7dc1dcbd0c..dfe99306d3 100644 --- a/libs/langchain/langchain/tools/searx_search/tool.py +++ b/libs/langchain/langchain/tools/searx_search/tool.py @@ -42,7 +42,7 @@ class SearxSearchRun(BaseTool): class SearxSearchResults(BaseTool): """Tool that queries a Searx instance and gets back json.""" - name: str = "Searx Search Results" + name: str = "Searx-Search-Results" description: str = ( "A meta search engine." "Useful for when you need to answer questions about current events." diff --git a/libs/langchain/tests/unit_tests/schema/runnable/test_runnable.py b/libs/langchain/tests/unit_tests/schema/runnable/test_runnable.py index 2ec0ed3719..87a558db64 100644 --- a/libs/langchain/tests/unit_tests/schema/runnable/test_runnable.py +++ b/libs/langchain/tests/unit_tests/schema/runnable/test_runnable.py @@ -1709,149 +1709,291 @@ async def test_stream_log_retriever() -> None: ): del op["value"]["id"] - assert stream_log[:-9] == [ - RunLogPatch( - { - "op": "replace", - "path": "", - "value": { - "logs": {}, - "final_output": None, - "streamed_output": [], + assert stream_log[:-9] in [ + [ + RunLogPatch( + { + "op": "replace", + "path": "", + "value": { + "logs": {}, + "final_output": None, + "streamed_output": [], + }, + } + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/RunnableParallel", + "value": { + "end_time": None, + "final_output": None, + "metadata": {}, + "name": "RunnableParallel", + "start_time": "2023-01-01T00:00:00.000", + "streamed_output_str": [], + "tags": ["seq:step:1"], + "type": "chain", + }, + } + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/RunnableLambda", + "value": { + "end_time": None, + "final_output": None, + "metadata": {}, + "name": "RunnableLambda", + "start_time": "2023-01-01T00:00:00.000", + "streamed_output_str": [], + "tags": ["map:key:question"], + "type": "chain", + }, + } + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/RunnableLambda/final_output", + "value": {"output": "What is your name?"}, }, - } - ), - RunLogPatch( - { - "op": "add", - "path": "/logs/RunnableParallel", - "value": { - "end_time": None, - "final_output": None, - "metadata": {}, - "name": "RunnableParallel", - "start_time": "2023-01-01T00:00:00.000", - "streamed_output_str": [], - "tags": ["seq:step:1"], - "type": "chain", + { + "op": "add", + "path": "/logs/RunnableLambda/end_time", + "value": "2023-01-01T00:00:00.000", }, - } - ), - RunLogPatch( - { - "op": "add", - "path": "/logs/RunnableLambda", - "value": { - "end_time": None, - "final_output": None, - "metadata": {}, - "name": "RunnableLambda", - "start_time": "2023-01-01T00:00:00.000", - "streamed_output_str": [], - "tags": ["map:key:question"], - "type": "chain", + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/Retriever", + "value": { + "end_time": None, + "final_output": None, + "metadata": {}, + "name": "Retriever", + "start_time": "2023-01-01T00:00:00.000", + "streamed_output_str": [], + "tags": ["map:key:documents"], + "type": "retriever", + }, + } + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/Retriever/final_output", + "value": { + "documents": [ + Document(page_content="foo"), + Document(page_content="bar"), + ] + }, }, - } - ), - RunLogPatch( - { - "op": "add", - "path": "/logs/RunnableLambda/final_output", - "value": {"output": "What is your name?"}, - }, - { - "op": "add", - "path": "/logs/RunnableLambda/end_time", - "value": "2023-01-01T00:00:00.000", - }, - ), - RunLogPatch( - { - "op": "add", - "path": "/logs/Retriever", - "value": { - "end_time": None, - "final_output": None, - "metadata": {}, - "name": "Retriever", - "start_time": "2023-01-01T00:00:00.000", - "streamed_output_str": [], - "tags": ["map:key:documents"], - "type": "retriever", + { + "op": "add", + "path": "/logs/Retriever/end_time", + "value": "2023-01-01T00:00:00.000", }, - } - ), - RunLogPatch( - { - "op": "add", - "path": "/logs/Retriever/final_output", - "value": { - "documents": [ - Document(page_content="foo"), - Document(page_content="bar"), - ] + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/RunnableParallel/final_output", + "value": { + "documents": [ + Document(page_content="foo"), + Document(page_content="bar"), + ], + "question": "What is your name?", + }, }, - }, - { - "op": "add", - "path": "/logs/Retriever/end_time", - "value": "2023-01-01T00:00:00.000", - }, - ), - RunLogPatch( - { - "op": "add", - "path": "/logs/RunnableParallel/final_output", - "value": { - "documents": [ - Document(page_content="foo"), - Document(page_content="bar"), - ], - "question": "What is your name?", + { + "op": "add", + "path": "/logs/RunnableParallel/end_time", + "value": "2023-01-01T00:00:00.000", }, - }, - { - "op": "add", - "path": "/logs/RunnableParallel/end_time", - "value": "2023-01-01T00:00:00.000", - }, - ), - RunLogPatch( - { - "op": "add", - "path": "/logs/ChatPromptTemplate", - "value": { - "end_time": None, - "final_output": None, - "metadata": {}, - "name": "ChatPromptTemplate", - "start_time": "2023-01-01T00:00:00.000", - "streamed_output_str": [], - "tags": ["seq:step:2"], - "type": "prompt", + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/ChatPromptTemplate", + "value": { + "end_time": None, + "final_output": None, + "metadata": {}, + "name": "ChatPromptTemplate", + "start_time": "2023-01-01T00:00:00.000", + "streamed_output_str": [], + "tags": ["seq:step:2"], + "type": "prompt", + }, + } + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/ChatPromptTemplate/final_output", + "value": ChatPromptValue( + messages=[ + SystemMessage(content="You are a nice assistant."), + HumanMessage( + content="[Document(page_content='foo'), Document(page_content='bar')]" # noqa: E501 + ), + HumanMessage(content="What is your name?"), + ] + ), }, - } - ), - RunLogPatch( - { - "op": "add", - "path": "/logs/ChatPromptTemplate/final_output", - "value": ChatPromptValue( - messages=[ - SystemMessage(content="You are a nice assistant."), - HumanMessage( - content="[Document(page_content='foo'), Document(page_content='bar')]" # noqa: E501 - ), - HumanMessage(content="What is your name?"), - ] - ), - }, - { - "op": "add", - "path": "/logs/ChatPromptTemplate/end_time", - "value": "2023-01-01T00:00:00.000", - }, - ), + { + "op": "add", + "path": "/logs/ChatPromptTemplate/end_time", + "value": "2023-01-01T00:00:00.000", + }, + ), + ], + [ + RunLogPatch( + { + "op": "replace", + "path": "", + "value": {"final_output": None, "logs": {}, "streamed_output": []}, + } + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/RunnableParallel", + "value": { + "end_time": None, + "final_output": None, + "metadata": {}, + "name": "RunnableParallel", + "start_time": "2023-01-01T00:00:00.000", + "streamed_output_str": [], + "tags": ["seq:step:1"], + "type": "chain", + }, + } + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/Retriever", + "value": { + "end_time": None, + "final_output": None, + "metadata": {}, + "name": "Retriever", + "start_time": "2023-01-01T00:00:00.000", + "streamed_output_str": [], + "tags": ["map:key:documents"], + "type": "retriever", + }, + } + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/RunnableLambda", + "value": { + "end_time": None, + "final_output": None, + "metadata": {}, + "name": "RunnableLambda", + "start_time": "2023-01-01T00:00:00.000", + "streamed_output_str": [], + "tags": ["map:key:question"], + "type": "chain", + }, + } + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/RunnableLambda/final_output", + "value": {"output": "What is your name?"}, + }, + { + "op": "add", + "path": "/logs/RunnableLambda/end_time", + "value": "2023-01-01T00:00:00.000", + }, + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/Retriever/final_output", + "value": { + "documents": [ + Document(page_content="foo"), + Document(page_content="bar"), + ] + }, + }, + { + "op": "add", + "path": "/logs/Retriever/end_time", + "value": "2023-01-01T00:00:00.000", + }, + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/RunnableParallel/final_output", + "value": { + "documents": [ + Document(page_content="foo"), + Document(page_content="bar"), + ], + "question": "What is your name?", + }, + }, + { + "op": "add", + "path": "/logs/RunnableParallel/end_time", + "value": "2023-01-01T00:00:00.000", + }, + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/ChatPromptTemplate", + "value": { + "end_time": None, + "final_output": None, + "metadata": {}, + "name": "ChatPromptTemplate", + "start_time": "2023-01-01T00:00:00.000", + "streamed_output_str": [], + "tags": ["seq:step:2"], + "type": "prompt", + }, + } + ), + RunLogPatch( + { + "op": "add", + "path": "/logs/ChatPromptTemplate/final_output", + "value": ChatPromptValue( + messages=[ + SystemMessage(content="You are a nice assistant."), + HumanMessage( + content="[Document(page_content='foo'), Document(page_content='bar')]" # noqa: E501 + ), + HumanMessage(content="What is your name?"), + ] + ), + }, + { + "op": "add", + "path": "/logs/ChatPromptTemplate/end_time", + "value": "2023-01-01T00:00:00.000", + }, + ), + ], ] assert sorted(cast(RunLog, add(stream_log)).state["logs"]) == [