From b051bba1a9f3f2c6020d7c8dbcc792d14b3cbe17 Mon Sep 17 00:00:00 2001 From: Scott Nath Date: Sun, 3 Mar 2024 17:30:05 -0500 Subject: [PATCH] community: Add you.com tool, add async to retriever, add async testing, add You tool doc (#18032) - **Description:** finishes adding the you.com functionality including: - add async functions to utility and retriever - add the You.com Tool - add async testing for utility, retriever, and tool - add a tool integration notebook page - **Dependencies:** any dependencies required for this change - **Twitter handle:** @scottnath --- docs/docs/integrations/tools/you.ipynb | 265 ++++++++++++++++++ .../langchain_community/retrievers/you.py | 17 +- .../langchain_community/tools/__init__.py | 9 + .../langchain_community/tools/you/__init__.py | 8 + .../langchain_community/tools/you/tool.py | 43 +++ .../langchain_community/utilities/you.py | 77 ++--- .../tests/unit_tests/retrievers/test_you.py | 39 +++ .../tests/unit_tests/tools/test_imports.py | 1 + .../tests/unit_tests/tools/test_public_api.py | 1 + .../tests/unit_tests/tools/test_you.py | 87 ++++++ .../tests/unit_tests/utilities/test_you.py | 58 +++- 11 files changed, 566 insertions(+), 39 deletions(-) create mode 100644 docs/docs/integrations/tools/you.ipynb create mode 100644 libs/community/langchain_community/tools/you/__init__.py create mode 100644 libs/community/langchain_community/tools/you/tool.py create mode 100644 libs/community/tests/unit_tests/tools/test_you.py diff --git a/docs/docs/integrations/tools/you.ipynb b/docs/docs/integrations/tools/you.ipynb new file mode 100644 index 0000000000..fbdade0d7b --- /dev/null +++ b/docs/docs/integrations/tools/you.ipynb @@ -0,0 +1,265 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "818fc023", + "metadata": {}, + "source": [ + "# You.com Search\n", + "\n", + "The [you.com API](https://api.you.com) is a suite of tools designed to help developers ground the output of LLMs in the most recent, most accurate, most relevant information that may not have been included in their training dataset." + ] + }, + { + "cell_type": "markdown", + "id": "02335552", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "markdown", + "id": "c5c53f28", + "metadata": {}, + "source": [ + "The tool lives in the `langchain-community` package.\n", + "\n", + "You also need to set your you.com API key." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d091ccb", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install --upgrade --quiet langchain-community" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f43dd867", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ[\"YDC_API_KEY\"] = \"\"\n", + "\n", + "# For use in Chaining section\n", + "os.environ[\"OPENAI_API_KEY\"] = \"\"\n", + "\n", + "## ALTERNATIVE: load YDC_API_KEY from a .env file\n", + "\n", + "# !pip install --quiet -U python-dotenv\n", + "# import dotenv\n", + "# dotenv.load_dotenv()" + ] + }, + { + "cell_type": "markdown", + "id": "17e70216", + "metadata": {}, + "source": [ + "It's also helpful (but not needed) to set up [LangSmith](https://smith.langchain.com/) for best-in-class observability" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "ce49fa4c", + "metadata": {}, + "outputs": [], + "source": [ + "# os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n", + "# os.environ[\"LANGCHAIN_API_KEY\"] = getpass.getpass()" + ] + }, + { + "cell_type": "markdown", + "id": "190ed085", + "metadata": {}, + "source": [ + "## Tool Usage" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "1367af5c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "YouSearchTool(api_wrapper=YouSearchAPIWrapper(ydc_api_key='054da371-e73b-47c1-a6d9-3b0cddf0fa3e<__>1Obt7EETU8N2v5f4MxaH0Zhx', num_web_results=1, safesearch=None, country=None, k=None, n_snippets_per_hit=None, endpoint_type='search', n_hits=None))" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from langchain_community.tools.you import YouSearchTool\n", + "from langchain_community.utilities.you import YouSearchAPIWrapper\n", + "\n", + "api_wrapper = YouSearchAPIWrapper(num_web_results=1)\n", + "tool = YouSearchTool(api_wrapper=api_wrapper)\n", + "\n", + "tool" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "a90d61d4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "7\n", + "page_content='10 Day Weather-Manhattan, NY\\nToday43°/39°1%\\nToday\\nSun 31 | Day\\nGenerally cloudy. High 43F. Winds W at 10 to 15 mph.\\n- Humidity54%\\n- UV Index0 of 11\\n- Sunrise7:19 am\\n- Sunset4:38 pm\\nSun 31 | Night\\nCloudy. Low 39F. Winds light and variable.\\n- Humidity70%\\n- UV Index0 of 11\\n- Moonrise9:13 pmWaning Gibbous\\n- Moonset10:28 am\\nMon 0145°/33°7%\\nMon 01\\nMon 01 | Day\\nConsiderable cloudiness. High around 45F. Winds light and variable.\\n- Humidity71%\\n- UV Index1 of 11\\n- Sunrise7:19 am\\n- Sunset4:39 pm\\nMon 01 | Night\\nA few clouds. Low 33F. Winds NNW at 5 to 10 mph.\\n- Humidity64%\\n- UV Index0 of 11\\n- Moonrise10:14 pmWaning Gibbous\\n- Moonset10:49 am\\nTue 0246°/35°4%\\nTue 02\\nTue 02 | Day\\nMainly sunny. High 46F. Winds NW at 5 to 10 mph.\\n- Humidity52%\\n- UV Index2 of 11\\n- Sunrise7:19 am\\n- Sunset4:40 pm\\nTue 02 | Night\\nA few clouds overnight. Low around 35F. Winds W at 5 to 10 mph.\\n- Humidity64%\\n- UV Index0 of 11\\n- Moonrise11:13 pmWaning Gibbous\\n- Moonset11:08 am\\nWed 0346°/38°4%\\nWed 03\\nWed 03 | Day' metadata={'url': 'https://weather.com/weather/tenday/l/New+York+NY+USNY0996:1:US', 'thumbnail_url': None, 'title': '10-Day Weather Forecast for Manhattan, NY - The Weather Channel ...', 'description': 'Some sun in the morning with increasing clouds during the afternoon. High around 45F. Winds SSE at 5 to 10 mph. ... Cloudy with showers. Low near 40F. Winds SSE at 5 to 10 mph. Chance of rain 60%. ... A steady rain in the morning. Showers continuing in the afternoon.'}\n", + "page_content='Radar\\nLatest News\\nOur Changing World\\nYour Privacy\\nTo personalize your product experience, we collect data from your device. We also may use or disclose to specific data vendors your precise geolocation data to provide the Services. To learn more please refer to our Privacy Policy.\\nChoose how my information is shared' metadata={'url': 'https://weather.com/weather/tenday/l/New+York+NY+USNY0996:1:US', 'thumbnail_url': None, 'title': '10-Day Weather Forecast for Manhattan, NY - The Weather Channel ...', 'description': 'Some sun in the morning with increasing clouds during the afternoon. High around 45F. Winds SSE at 5 to 10 mph. ... Cloudy with showers. Low near 40F. Winds SSE at 5 to 10 mph. Chance of rain 60%. ... A steady rain in the morning. Showers continuing in the afternoon.'}\n", + "page_content='- Humidity82%\\n- UV Index1 of 11\\n- Sunrise7:18 am\\n- Sunset4:34 pm\\nTue 26 | Night\\nCloudy with light rain developing after midnight. Low 47F. Winds light and variable. Chance of rain 80%.\\n- Humidity90%\\n- UV Index0 of 11\\n- Moonrise4:00 pmFull Moon\\n- Moonset7:17 am\\nWed 2754°/49°93%\\nWed 27\\nWed 27 | Day\\nRain. High 54F. Winds E at 5 to 10 mph. Chance of rain 90%. Rainfall near a half an inch.\\n- Humidity93%\\n- UV Index1 of 11\\n- Sunrise7:18 am\\n- Sunset4:35 pm\\nWed 27 | Night\\nSteady light rain in the evening. Showers continuing late. Low 49F. Winds light and variable. Chance of rain 70%.\\n- Humidity91%\\n- UV Index0 of 11\\n- Moonrise4:59 pmFull Moon\\n- Moonset8:12 am\\nThu 2853°/42°19%\\nThu 28\\nThu 28 | Day\\nCloudy skies early will become partly cloudy later in the day. Slight chance of a rain shower. High 53F. Winds WSW at 5 to 10 mph.\\n- Humidity77%\\n- UV Index1 of 11\\n- Sunrise7:18 am\\n- Sunset4:36 pm\\nThu 28 | Night\\nPartly cloudy skies. Low 42F. Winds W at 5 to 10 mph.\\n- Humidity71%\\n- UV Index0 of 11' metadata={'url': 'https://weather.com/weather/tenday/l/New+York+NY+USNY0996:1:US', 'thumbnail_url': None, 'title': '10-Day Weather Forecast for Manhattan, NY - The Weather Channel ...', 'description': 'Some sun in the morning with increasing clouds during the afternoon. High around 45F. Winds SSE at 5 to 10 mph. ... Cloudy with showers. Low near 40F. Winds SSE at 5 to 10 mph. Chance of rain 60%. ... A steady rain in the morning. Showers continuing in the afternoon.'}\n", + "page_content='- Moonrise2:20 amWaning Crescent\\n- Moonset12:33 pm\\nSun 0740°/29°19%\\nSun 07\\nSun 07 | Day\\nIntervals of clouds and sunshine. High around 40F. Winds NW at 5 to 10 mph.\\n- Humidity57%\\n- UV Index2 of 11\\n- Sunrise7:19 am\\n- Sunset4:44 pm\\nSun 07 | Night\\nA few clouds from time to time. Low 29F. Winds NNW at 5 to 10 mph.\\n- Humidity60%\\n- UV Index0 of 11\\n- Moonrise3:28 amWaning Crescent\\n- Moonset1:04 pm\\nMon 0840°/32°35%\\nMon 08\\nMon 08 | Day\\nPartly cloudy early followed mostly cloudy skies and a few snow showers later in the day. High near 40F. Winds N at 5 to 10 mph. Chance of snow 40%.\\n- UV Index1 of 11\\n- Sunrise7:19 am\\n- Sunset4:45 pm\\nMon 08 | Night\\nVariable clouds with snow showers or flurries. Low 32F. Winds NNE at 5 to 10 mph. Chance of snow 60%. Snow accumulations less than one inch.\\n- UV Index0 of 11\\n- Moonrise4:40 amWaning Crescent\\n- Moonset1:43 pm\\nLatest News\\nOur Changing World\\nYour Privacy' metadata={'url': 'https://weather.com/weather/tenday/l/New+York+NY+USNY0996:1:US', 'thumbnail_url': None, 'title': '10-Day Weather Forecast for Manhattan, NY - The Weather Channel ...', 'description': 'Some sun in the morning with increasing clouds during the afternoon. High around 45F. Winds SSE at 5 to 10 mph. ... Cloudy with showers. Low near 40F. Winds SSE at 5 to 10 mph. Chance of rain 60%. ... A steady rain in the morning. Showers continuing in the afternoon.'}\n", + "page_content='- Humidity91%\\n- UV Index0 of 11\\n- Moonrise5:50 amWaning Crescent\\n- Moonset2:35 pm\\nWed 1056°/39°34%\\nWed 10\\nWed 10 | Day\\nA shower or two possible early with partly cloudy skies in the afternoon. Morning high of 56F with temps falling to near 45. Winds SW at 15 to 25 mph. Chance of rain 30%.\\n- Humidity66%\\n- UV Index1 of 11\\n- Sunrise7:19 am\\n- Sunset4:47 pm\\nWed 10 | Night\\nA few clouds from time to time. Low 39F. Winds WSW at 10 to 20 mph.\\n- Humidity64%\\n- UV Index0 of 11\\n- Moonrise6:56 amWaning Crescent\\n- Moonset3:38 pm\\nThu 1147°/38°5%\\nThu 11\\nThu 11 | Day\\nPartly cloudy. High 47F. Winds WSW at 5 to 10 mph.\\n- Humidity62%\\n- UV Index2 of 11\\n- Sunrise7:19 am\\n- Sunset4:48 pm\\nThu 11 | Night\\nMostly clear skies. Low 38F. Winds W at 5 to 10 mph.\\n- Humidity66%\\n- UV Index0 of 11\\n- Moonrise7:52 amNew Moon\\n- Moonset4:53 pm\\nFri 1248°/42°19%\\nFri 12\\nFri 12 | Day\\nIntervals of clouds and sunshine. High 48F. Winds WSW at 5 to 10 mph.\\n- Humidity62%\\n- UV Index2 of 11\\n- Sunrise7:18 am\\n- Sunset4:49 pm' metadata={'url': 'https://weather.com/weather/tenday/l/New+York+NY+USNY0996:1:US', 'thumbnail_url': None, 'title': '10-Day Weather Forecast for Manhattan, NY - The Weather Channel ...', 'description': 'Some sun in the morning with increasing clouds during the afternoon. High around 45F. Winds SSE at 5 to 10 mph. ... Cloudy with showers. Low near 40F. Winds SSE at 5 to 10 mph. Chance of rain 60%. ... A steady rain in the morning. Showers continuing in the afternoon.'}\n", + "page_content='Sat 1346°/36°53%\\nSat 13\\nSat 13 | Day\\nCloudy with showers. High 46F. Winds WSW at 10 to 15 mph. Chance of rain 50%.\\n- Humidity73%\\n- UV Index1 of 11\\n- Sunrise7:18 am\\n- Sunset4:50 pm\\nSat 13 | Night\\nRain showers early transitioning to snow showers late. Low 36F. Winds W at 10 to 15 mph. Chance of precip 50%.\\n- Humidity70%\\n- UV Index0 of 11\\n- Moonrise9:14 amWaxing Crescent\\n- Moonset7:33 pm\\nSun 1442°/34°37%\\nSun 14\\nSun 14 | Day\\nSnow showers early will transition to a few showers later. High 42F. Winds WSW at 10 to 15 mph. Chance of rain 40%.\\n- Humidity63%\\n- UV Index1 of 11\\n- Sunrise7:18 am\\n- Sunset4:51 pm\\nSun 14 | Night\\nVariable clouds with snow showers. Low 34F. Winds W at 10 to 15 mph. Chance of snow 60%. Snow accumulations less than one inch.\\n- UV Index0 of 11\\n- Moonrise9:44 amWaxing Crescent\\n- Moonset8:52 pm\\nMon 1540°/31°51%\\nMon 15\\nMon 15 | Day' metadata={'url': 'https://weather.com/weather/tenday/l/New+York+NY+USNY0996:1:US', 'thumbnail_url': None, 'title': '10-Day Weather Forecast for Manhattan, NY - The Weather Channel ...', 'description': 'Some sun in the morning with increasing clouds during the afternoon. High around 45F. Winds SSE at 5 to 10 mph. ... Cloudy with showers. Low near 40F. Winds SSE at 5 to 10 mph. Chance of rain 60%. ... A steady rain in the morning. Showers continuing in the afternoon.'}\n", + "page_content='- Humidity70%\\n- UV Index1 of 11\\n- Sunrise7:18 am\\n- Sunset4:34 pm\\nMon 25 | Night\\nOvercast with showers at times. Low 43F. Winds light and variable. Chance of rain 40%.\\n- Humidity80%\\n- UV Index0 of 11\\n- Moonrise3:08 pmWaxing Gibbous\\n- Moonset6:14 am\\nTue 2653°/45°58%\\nTue 26\\nTue 26 | Day\\nOvercast with rain showers at times. High 53F. Winds E at 5 to 10 mph. Chance of rain 60%.\\n- Humidity79%\\n- UV Index1 of 11\\n- Sunrise7:18 am\\n- Sunset4:34 pm\\nTue 26 | Night\\nShowers early then scattered thunderstorms developing late. Low near 45F. Winds ESE at 5 to 10 mph. Chance of rain 60%.\\n- Humidity93%\\n- UV Index0 of 11\\n- Moonrise4:00 pmFull Moon\\n- Moonset7:17 am\\nWed 2751°/41°58%\\nWed 27\\nWed 27 | Day\\nCloudy with showers. High 51F. Winds WSW at 5 to 10 mph. Chance of rain 60%.\\n- Humidity79%\\n- UV Index1 of 11\\n- Sunrise7:18 am\\n- Sunset4:35 pm\\nWed 27 | Night\\nCloudy with showers. Low 41F. Winds NW at 5 to 10 mph. Chance of rain 60%.\\n- Humidity72%\\n- UV Index0 of 11\\n- Moonrise4:59 pmFull Moon\\n- Moonset8:13 am' metadata={'url': 'https://weather.com/weather/tenday/l/New+York+NY+USNY0996:1:US', 'thumbnail_url': None, 'title': '10-Day Weather Forecast for Manhattan, NY - The Weather Channel ...', 'description': 'Some sun in the morning with increasing clouds during the afternoon. High around 45F. Winds SSE at 5 to 10 mph. ... Cloudy with showers. Low near 40F. Winds SSE at 5 to 10 mph. Chance of rain 60%. ... A steady rain in the morning. Showers continuing in the afternoon.'}\n" + ] + } + ], + "source": [ + "# .invoke wraps utility.results\n", + "response = tool.invoke(\"What is the weather in NY\")\n", + "\n", + "# .invoke should have a Document for each `snippet`\n", + "print(len(response))\n", + "\n", + "for item in response:\n", + " print(item)" + ] + }, + { + "cell_type": "markdown", + "id": "aac4a1f9", + "metadata": {}, + "source": [ + "## Chaining\n", + "\n", + "We show here how to use it as part of an [agent](/docs/modules/agents). We use the OpenAI Functions Agent, so we will need to setup and install the required dependencies for that. We will also use [LangSmith Hub](https://smith.langchain.com/hub) to pull the prompt from, so we will need to install that." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21559e10", + "metadata": {}, + "outputs": [], + "source": [ + "# you need a model to use in the chain\n", + "!pip install --upgrade --quiet langchain langchain-openai langchainhub langchain-community" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "7947986a-ea7b-417c-97df-1ecf98e95793", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain import hub\n", + "from langchain.agents import AgentExecutor, create_openai_functions_agent\n", + "from langchain_openai import ChatOpenAI\n", + "\n", + "instructions = \"\"\"You are an assistant.\"\"\"\n", + "base_prompt = hub.pull(\"langchain-ai/openai-functions-template\")\n", + "prompt = base_prompt.partial(instructions=instructions)\n", + "llm = ChatOpenAI(temperature=0)\n", + "you_tool = YouSearchTool(api_wrapper=YouSearchAPIWrapper(num_web_results=1))\n", + "tools = [you_tool]\n", + "agent = create_openai_functions_agent(llm, tools, prompt)\n", + "agent_executor = AgentExecutor(\n", + " agent=agent,\n", + " tools=tools,\n", + " verbose=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "48e578c0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n", + "\u001b[32;1m\u001b[1;3m\n", + "Invoking: `you_search` with `{'query': 'weather in NY today'}`\n", + "\n", + "\n", + "\u001b[0m\u001b[36;1m\u001b[1;3m[Document(page_content=\"New York City, NY Forecast\\nWeather Today in New York City, NY\\nFeels Like43°\\n7:17 am\\n4:32 pm\\nHigh / Low\\n--/39°\\nWind\\n3 mph\\nHumidity\\n63%\\nDew Point\\n31°\\nPressure\\n30.44 in\\nUV Index\\n0 of 11\\nVisibility\\n10 mi\\nMoon Phase\\nWaxing Gibbous\\nAdvertisement\\nLatest News\\nHealth & Activities\\nAdvertisement\\nAir Quality Index\\nGood\\nAir quality is considered satisfactory, and air pollution poses little or no risk.\\nAdvertisement\\nHappening Near New York City, NY\\nPopular Nextdoor posts\\nHas anyone announced --in which case I apologize--that Bo's Bagels--opened yesterday where Tommy's pizza once was on uptown side of Broadway, bet 155 and 156.They have been incredibly successful in Harlem ( W.116th) and are now spreading their wings. Open 8-3 now, will go later soon, delivers and has many varieties baked on site!!! Just had lunch there!!\\nWash Hts S (W155-W159-Bdway)\\nNeighbor\\n4d ago\", metadata={'url': 'https://weather.com/weather/today/l/96f2f84af9a5f5d452eb0574d4e4d8a840c71b05e22264ebdc0056433a642c84', 'thumbnail_url': None, 'title': 'Weather Forecast and Conditions for New York City, NY - The Weather ...', 'description': 'Today’s and tonight’s New York City, NY weather forecast, weather conditions and Doppler radar from The Weather Channel and Weather.com'}), Document(page_content='time hearing racial slurs in my life and in NYC. So I don’t know why tonight’s encounter makes me feel so helpless and sad. Her apartment is on our usual route for walks, and now I’m already thinking about how I need to behave if this situation repeats again. This just doesn’t feel fair. How would you handle it? Thanks for reading!', metadata={'url': 'https://weather.com/weather/today/l/96f2f84af9a5f5d452eb0574d4e4d8a840c71b05e22264ebdc0056433a642c84', 'thumbnail_url': None, 'title': 'Weather Forecast and Conditions for New York City, NY - The Weather ...', 'description': 'Today’s and tonight’s New York City, NY weather forecast, weather conditions and Doppler radar from The Weather Channel and Weather.com'}), Document(page_content=\"Hell's Kitchen\\nNeighbor\\n4d ago\\nJust curious.Are the M100/M100 and BX19 buses free? It seems like I'm the only one who pays. Even had a group of non-payers give attitude because I held up the line to pay my fare. Non-payers being whole families, men and women of all ages. I remember when you used to be scared to walk on for free but do understand the drivers are told not to intervene. Still it should either be free or paid fare.\\nHamilton Hts (W148St-Broadway)\\nNeighbor\\n4d ago\\nHey there, I am a journalism student at Craig Newmark School of Journalism looking for stories in the Lower East Side and East Village.Any ideas?\\nAlphabet City\\nNeighbor\\n8d ago\\nProvided by\\nAdvertisement\\nYour Privacy\\nTo personalize your product experience, we collect data from your device. We also may use or disclose to specific data vendors your precise geolocation data to provide the Services. To learn more please refer to our Privacy Policy.\\nReview All Privacy and Ad Settings\\nChoose how my information is shared\", metadata={'url': 'https://weather.com/weather/today/l/96f2f84af9a5f5d452eb0574d4e4d8a840c71b05e22264ebdc0056433a642c84', 'thumbnail_url': None, 'title': 'Weather Forecast and Conditions for New York City, NY - The Weather ...', 'description': 'Today’s and tonight’s New York City, NY weather forecast, weather conditions and Doppler radar from The Weather Channel and Weather.com'}), Document(page_content='Review All Privacy and Ad Settings\\nChoose how my information is shared', metadata={'url': 'https://weather.com/weather/today/l/96f2f84af9a5f5d452eb0574d4e4d8a840c71b05e22264ebdc0056433a642c84', 'thumbnail_url': None, 'title': 'Weather Forecast and Conditions for New York City, NY - The Weather ...', 'description': 'Today’s and tonight’s New York City, NY weather forecast, weather conditions and Doppler radar from The Weather Channel and Weather.com'}), Document(page_content='2d ago\\nProvided by\\nAdvertisement\\nYour Privacy\\nTo personalize your product experience, we collect data from your device. We also may use or disclose to specific data vendors your precise geolocation data to provide the Services. To learn more please refer to our Privacy Policy.\\nReview All Privacy and Ad Settings\\nChoose how my information is shared', metadata={'url': 'https://weather.com/weather/today/l/96f2f84af9a5f5d452eb0574d4e4d8a840c71b05e22264ebdc0056433a642c84', 'thumbnail_url': None, 'title': 'Weather Forecast and Conditions for New York City, NY - The Weather ...', 'description': 'Today’s and tonight’s New York City, NY weather forecast, weather conditions and Doppler radar from The Weather Channel and Weather.com'}), Document(page_content=\"Lenox Hill\\nNeighbor\\n8d ago\\nParking.It is getting worst. What happened to resident-only neighborhood parking program? With this upcoming congested traffic tolling below 96th Street, competition for parking bordering the zone is going to get tougher. This should be part of the congested traffic agenda.\\nCent Harlem S (120-FDouglass)\\nNeighbor\\n9d ago\\nI thought these sites were much more localized.If there is one for the UWS I'll join it but have no reason to be reading about Yorkville or Union NJ.\\nLincoln Sq (W70-Amst-66-WEnd)\\nNeighbor\\n10d ago\\nProvided by\\nAdvertisement\\nYour Privacy\\nTo personalize your product experience, we collect data from your device. We also may use or disclose to specific data vendors your precise geolocation data to provide the Services. To learn more please refer to our Privacy Policy.\\nReview All Privacy and Ad Settings\\nChoose how my information is shared\", metadata={'url': 'https://weather.com/weather/today/l/96f2f84af9a5f5d452eb0574d4e4d8a840c71b05e22264ebdc0056433a642c84', 'thumbnail_url': None, 'title': 'Weather Forecast and Conditions for New York City, NY - The Weather ...', 'description': 'Today’s and tonight’s New York City, NY weather forecast, weather conditions and Doppler radar from The Weather Channel and Weather.com'})]\u001b[0m\u001b[32;1m\u001b[1;3mThe weather in New York City today is as follows:\n", + "- Feels Like: 43°F\n", + "- High/Low: --/39°F\n", + "- Wind: 3 mph\n", + "- Humidity: 63%\n", + "- Dew Point: 31°F\n", + "- Pressure: 30.44 in\n", + "- UV Index: 0 of 11\n", + "- Visibility: 10 mi\n", + "- Moon Phase: Waxing Gibbous\n", + "\n", + "For more details, you can visit [The Weather Channel](https://weather.com/weather/today/l/96f2f84af9a5f5d452eb0574d4e4d8a840c71b05e22264ebdc0056433a642c84).\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "{'input': 'What is the weather in NY today?',\n", + " 'output': 'The weather in New York City today is as follows:\\n- Feels Like: 43°F\\n- High/Low: --/39°F\\n- Wind: 3 mph\\n- Humidity: 63%\\n- Dew Point: 31°F\\n- Pressure: 30.44 in\\n- UV Index: 0 of 11\\n- Visibility: 10 mi\\n- Moon Phase: Waxing Gibbous\\n\\nFor more details, you can visit [The Weather Channel](https://weather.com/weather/today/l/96f2f84af9a5f5d452eb0574d4e4d8a840c71b05e22264ebdc0056433a642c84).'}" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "agent_executor.invoke({\"input\": \"What is the weather in NY today?\"})" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/libs/community/langchain_community/retrievers/you.py b/libs/community/langchain_community/retrievers/you.py index 9564e7307e..5d3b19e430 100644 --- a/libs/community/langchain_community/retrievers/you.py +++ b/libs/community/langchain_community/retrievers/you.py @@ -1,6 +1,9 @@ from typing import Any, List -from langchain_core.callbacks import CallbackManagerForRetrieverRun +from langchain_core.callbacks import ( + AsyncCallbackManagerForRetrieverRun, + CallbackManagerForRetrieverRun, +) from langchain_core.documents import Document from langchain_core.retrievers import BaseRetriever @@ -21,3 +24,15 @@ class YouRetriever(BaseRetriever, YouSearchAPIWrapper): **kwargs: Any, ) -> List[Document]: return self.results(query, run_manager=run_manager.get_child(), **kwargs) + + async def _aget_relevant_documents( + self, + query: str, + *, + run_manager: AsyncCallbackManagerForRetrieverRun, + **kwargs: Any, + ) -> List[Document]: + results = await self.results_async( + query, run_manager=run_manager.get_child(), **kwargs + ) + return results diff --git a/libs/community/langchain_community/tools/__init__.py b/libs/community/langchain_community/tools/__init__.py index 57f999309f..96791b7db0 100644 --- a/libs/community/langchain_community/tools/__init__.py +++ b/libs/community/langchain_community/tools/__init__.py @@ -782,6 +782,12 @@ def _import_yahoo_finance_news() -> Any: return YahooFinanceNewsTool +def _import_you_tool() -> Any: + from langchain_community.tools.you.tool import YouSearchTool + + return YouSearchTool + + def _import_youtube_search() -> Any: from langchain_community.tools.youtube.search import YouTubeSearchTool @@ -1055,6 +1061,8 @@ def __getattr__(name: str) -> Any: return _import_wolfram_alpha_tool() elif name == "YahooFinanceNewsTool": return _import_yahoo_finance_news() + elif name == "YouSearchTool": + return _import_you_tool() elif name == "YouTubeSearchTool": return _import_youtube_search() elif name == "ZapierNLAListActions": @@ -1192,6 +1200,7 @@ __all__ = [ "WolframAlphaQueryRun", "WriteFileTool", "YahooFinanceNewsTool", + "YouSearchTool", "YouTubeSearchTool", "ZapierNLAListActions", "ZapierNLARunAction", diff --git a/libs/community/langchain_community/tools/you/__init__.py b/libs/community/langchain_community/tools/you/__init__.py new file mode 100644 index 0000000000..fcecca418a --- /dev/null +++ b/libs/community/langchain_community/tools/you/__init__.py @@ -0,0 +1,8 @@ +"""You.com API toolkit.""" + + +from langchain_community.tools.you.tool import YouSearchTool + +__all__ = [ + "YouSearchTool", +] diff --git a/libs/community/langchain_community/tools/you/tool.py b/libs/community/langchain_community/tools/you/tool.py new file mode 100644 index 0000000000..75e18a1b62 --- /dev/null +++ b/libs/community/langchain_community/tools/you/tool.py @@ -0,0 +1,43 @@ +from typing import List, Optional, Type + +from langchain_core.callbacks import ( + AsyncCallbackManagerForToolRun, + CallbackManagerForToolRun, +) +from langchain_core.documents import Document +from langchain_core.pydantic_v1 import BaseModel, Field +from langchain_core.tools import BaseTool + +from langchain_community.utilities.you import YouSearchAPIWrapper + + +class YouInput(BaseModel): + query: str = Field(description="should be a search query") + + +class YouSearchTool(BaseTool): + """Tool that searches the you.com API""" + + name = "you_search" + description = ( + "The YOU APIs make LLMs and search experiences more factual and" + "up to date with realtime web data." + ) + args_schema: Type[BaseModel] = YouInput + api_wrapper: YouSearchAPIWrapper = Field(default_factory=YouSearchAPIWrapper) + + def _run( + self, + query: str, + run_manager: Optional[CallbackManagerForToolRun] = None, + ) -> List[Document]: + """Use the you.com tool.""" + return self.api_wrapper.results(query) + + async def _arun( + self, + query: str, + run_manager: Optional[AsyncCallbackManagerForToolRun] = None, + ) -> List[Document]: + """Use the you.com tool asynchronously.""" + return await self.api_wrapper.results_async(query) diff --git a/libs/community/langchain_community/utilities/you.py b/libs/community/langchain_community/utilities/you.py index c01a1ed37a..119b79bc0e 100644 --- a/libs/community/langchain_community/utilities/you.py +++ b/libs/community/langchain_community/utilities/you.py @@ -2,7 +2,6 @@ In order to set this up, follow instructions at: """ -import json from typing import Any, Dict, List, Literal, Optional import aiohttp @@ -113,16 +112,16 @@ class YouSearchAPIWrapper(BaseModel): docs = [] for hit in raw_search_results["hits"]: - n_snippets_per_hit = self.n_snippets_per_hit or len(hit["snippets"]) - for snippet in hit["snippets"][:n_snippets_per_hit]: + n_snippets_per_hit = self.n_snippets_per_hit or len(hit.get("snippets")) + for snippet in hit.get("snippets")[:n_snippets_per_hit]: docs.append( Document( page_content=snippet, metadata={ - "url": hit["url"], - "thumbnail_url": hit["thumbnail_url"], - "title": hit["title"], - "description": hit["description"], + "url": hit.get("url"), + "thumbnail_url": hit.get("thumbnail_url"), + "title": hit.get("title"), + "description": hit.get("description"), }, ) ) @@ -188,43 +187,47 @@ class YouSearchAPIWrapper(BaseModel): async def raw_results_async( self, query: str, - num_web_results: Optional[int] = 5, - safesearch: Optional[str] = "moderate", - country: Optional[str] = "US", + **kwargs: Any, ) -> Dict: """Get results from the you.com Search API asynchronously.""" - # Function to perform the API call - async def fetch() -> str: - params = { - "query": query, - "num_web_results": num_web_results, - "safesearch": safesearch, - "country": country, - } - async with aiohttp.ClientSession() as session: - async with session.post(f"{YOU_API_URL}/search", json=params) as res: - if res.status == 200: - data = await res.text() - return data - else: - raise Exception(f"Error {res.status}: {res.reason}") + headers = {"X-API-Key": self.ydc_api_key or ""} + params = { + "query": query, + "num_web_results": self.num_web_results, + "safesearch": self.safesearch, + "country": self.country, + **kwargs, + } + params = {k: v for k, v in params.items() if v is not None} + # news endpoint expects `q` instead of `query` + if self.endpoint_type == "news": + params["q"] = params["query"] + del params["query"] - results_json_str = await fetch() - return json.loads(results_json_str) + # @todo deprecate `snippet`, not part of API + if self.endpoint_type == "snippet": + self.endpoint_type = "search" + + async with aiohttp.ClientSession() as session: + async with session.get( + url=f"{YOU_API_URL}/{self.endpoint_type}", + params=params, + headers=headers, + ) as res: + if res.status == 200: + results = await res.json() + return results + else: + raise Exception(f"Error {res.status}: {res.reason}") async def results_async( self, query: str, - num_web_results: Optional[int] = 5, - safesearch: Optional[str] = "moderate", - country: Optional[str] = "US", + **kwargs: Any, ) -> List[Document]: - results_json = await self.raw_results_async( - query=query, - num_web_results=num_web_results, - safesearch=safesearch, - country=country, + raw_search_results_async = await self.raw_results_async( + query, + **{key: value for key, value in kwargs.items() if value is not None}, ) - - return self._parse_results(results_json["results"]) + return self._parse_results(raw_search_results_async) diff --git a/libs/community/tests/unit_tests/retrievers/test_you.py b/libs/community/tests/unit_tests/retrievers/test_you.py index dbc8cc6509..2ac44c9bf1 100644 --- a/libs/community/tests/unit_tests/retrievers/test_you.py +++ b/libs/community/tests/unit_tests/retrievers/test_you.py @@ -1,3 +1,6 @@ +from unittest.mock import AsyncMock, patch + +import pytest import responses from langchain_community.retrievers.you import YouRetriever @@ -70,3 +73,39 @@ class TestYouRetriever: results = you_wrapper.results(query) expected_result = NEWS_RESPONSE_PARSED assert results == expected_result + + @pytest.mark.asyncio + async def test_aget_relevant_documents(self) -> None: + instance = YouRetriever(ydc_api_key="test_api_key") + + # Mock response object to simulate aiohttp response + mock_response = AsyncMock() + mock_response.__aenter__.return_value = ( + mock_response # Make the context manager return itself + ) + mock_response.__aexit__.return_value = None # No value needed for exit + mock_response.status = 200 + mock_response.json = AsyncMock(return_value=MOCK_RESPONSE_RAW) + + # Patch the aiohttp.ClientSession object + with patch("aiohttp.ClientSession.get", return_value=mock_response): + results = await instance.aget_relevant_documents("test query") + assert results == MOCK_PARSED_OUTPUT + + @pytest.mark.asyncio + async def test_ainvoke(self) -> None: + instance = YouRetriever(ydc_api_key="test_api_key") + + # Mock response object to simulate aiohttp response + mock_response = AsyncMock() + mock_response.__aenter__.return_value = ( + mock_response # Make the context manager return itself + ) + mock_response.__aexit__.return_value = None # No value needed for exit + mock_response.status = 200 + mock_response.json = AsyncMock(return_value=MOCK_RESPONSE_RAW) + + # Patch the aiohttp.ClientSession object + with patch("aiohttp.ClientSession.get", return_value=mock_response): + results = await instance.ainvoke("test query") + assert results == MOCK_PARSED_OUTPUT diff --git a/libs/community/tests/unit_tests/tools/test_imports.py b/libs/community/tests/unit_tests/tools/test_imports.py index 0c1b4a9c6d..9f455ba048 100644 --- a/libs/community/tests/unit_tests/tools/test_imports.py +++ b/libs/community/tests/unit_tests/tools/test_imports.py @@ -122,6 +122,7 @@ EXPECTED_ALL = [ "WolframAlphaQueryRun", "WriteFileTool", "YahooFinanceNewsTool", + "YouSearchTool", "YouTubeSearchTool", "ZapierNLAListActions", "ZapierNLARunAction", diff --git a/libs/community/tests/unit_tests/tools/test_public_api.py b/libs/community/tests/unit_tests/tools/test_public_api.py index 870abc5975..18a951b20a 100644 --- a/libs/community/tests/unit_tests/tools/test_public_api.py +++ b/libs/community/tests/unit_tests/tools/test_public_api.py @@ -124,6 +124,7 @@ _EXPECTED = [ "WolframAlphaQueryRun", "WriteFileTool", "YahooFinanceNewsTool", + "YouSearchTool", "YouTubeSearchTool", "ZapierNLAListActions", "ZapierNLARunAction", diff --git a/libs/community/tests/unit_tests/tools/test_you.py b/libs/community/tests/unit_tests/tools/test_you.py new file mode 100644 index 0000000000..acbf6154b0 --- /dev/null +++ b/libs/community/tests/unit_tests/tools/test_you.py @@ -0,0 +1,87 @@ +from unittest.mock import AsyncMock, patch + +import pytest +import responses + +from langchain_community.tools.you import YouSearchTool +from langchain_community.utilities.you import YouSearchAPIWrapper + +from ..utilities.test_you import ( + LIMITED_PARSED_OUTPUT, + MOCK_PARSED_OUTPUT, + MOCK_RESPONSE_RAW, + NEWS_RESPONSE_PARSED, + NEWS_RESPONSE_RAW, + TEST_ENDPOINT, +) + + +class TestYouSearchTool: + @responses.activate + def test_invoke(self) -> None: + responses.add( + responses.GET, f"{TEST_ENDPOINT}/search", json=MOCK_RESPONSE_RAW, status=200 + ) + query = "Test query text" + you_tool = YouSearchTool(api_wrapper=YouSearchAPIWrapper(ydc_api_key="test")) + results = you_tool.invoke(query) + expected_result = MOCK_PARSED_OUTPUT + assert results == expected_result + + @responses.activate + def test_invoke_max_docs(self) -> None: + responses.add( + responses.GET, f"{TEST_ENDPOINT}/search", json=MOCK_RESPONSE_RAW, status=200 + ) + query = "Test query text" + you_tool = YouSearchTool( + api_wrapper=YouSearchAPIWrapper(ydc_api_key="test", k=2) + ) + results = you_tool.invoke(query) + expected_result = [MOCK_PARSED_OUTPUT[0], MOCK_PARSED_OUTPUT[1]] + assert results == expected_result + + @responses.activate + def test_invoke_limit_snippets(self) -> None: + responses.add( + responses.GET, f"{TEST_ENDPOINT}/search", json=MOCK_RESPONSE_RAW, status=200 + ) + query = "Test query text" + you_tool = YouSearchTool( + api_wrapper=YouSearchAPIWrapper(ydc_api_key="test", n_snippets_per_hit=1) + ) + results = you_tool.invoke(query) + expected_result = LIMITED_PARSED_OUTPUT + assert results == expected_result + + @responses.activate + def test_invoke_news(self) -> None: + responses.add( + responses.GET, f"{TEST_ENDPOINT}/news", json=NEWS_RESPONSE_RAW, status=200 + ) + + query = "Test news text" + you_tool = YouSearchTool( + api_wrapper=YouSearchAPIWrapper(ydc_api_key="test", endpoint_type="news") + ) + results = you_tool.invoke(query) + expected_result = NEWS_RESPONSE_PARSED + assert results == expected_result + + @pytest.mark.asyncio + async def test_ainvoke(self) -> None: + you_tool = YouSearchTool(api_wrapper=YouSearchAPIWrapper(ydc_api_key="test")) + + # Mock response object to simulate aiohttp response + mock_response = AsyncMock() + mock_response.__aenter__.return_value = ( + mock_response # Make the context manager return itself + ) + mock_response.__aexit__.return_value = None # No value needed for exit + mock_response.status = 200 + mock_response.json = AsyncMock(return_value=MOCK_RESPONSE_RAW) + + # Patch the aiohttp.ClientSession object + with patch("aiohttp.ClientSession.get", return_value=mock_response): + results = await you_tool.ainvoke("test query") + assert results == MOCK_PARSED_OUTPUT diff --git a/libs/community/tests/unit_tests/utilities/test_you.py b/libs/community/tests/unit_tests/utilities/test_you.py index adc004ad43..5ac5009d22 100644 --- a/libs/community/tests/unit_tests/utilities/test_you.py +++ b/libs/community/tests/unit_tests/utilities/test_you.py @@ -1,5 +1,7 @@ from typing import Any, Dict, List, Optional, Union +from unittest.mock import AsyncMock, patch +import pytest import responses from langchain_core.documents import Document @@ -187,4 +189,58 @@ def test_results_news() -> None: assert raw_results == expected_result -# @todo test async methods +@pytest.mark.asyncio +async def test_raw_results_async() -> None: + instance = YouSearchAPIWrapper(ydc_api_key="test_api_key") + + # Mock response object to simulate aiohttp response + mock_response = AsyncMock() + mock_response.__aenter__.return_value = ( + mock_response # Make the context manager return itself + ) + mock_response.__aexit__.return_value = None # No value needed for exit + mock_response.status = 200 + mock_response.json = AsyncMock(return_value=MOCK_RESPONSE_RAW) + + # Patch the aiohttp.ClientSession object + with patch("aiohttp.ClientSession.get", return_value=mock_response): + results = await instance.raw_results_async("test query") + assert results == MOCK_RESPONSE_RAW + + +@pytest.mark.asyncio +async def test_results_async() -> None: + instance = YouSearchAPIWrapper(ydc_api_key="test_api_key") + + # Mock response object to simulate aiohttp response + mock_response = AsyncMock() + mock_response.__aenter__.return_value = ( + mock_response # Make the context manager return itself + ) + mock_response.__aexit__.return_value = None # No value needed for exit + mock_response.status = 200 + mock_response.json = AsyncMock(return_value=MOCK_RESPONSE_RAW) + + # Patch the aiohttp.ClientSession object + with patch("aiohttp.ClientSession.get", return_value=mock_response): + results = await instance.results_async("test query") + assert results == MOCK_PARSED_OUTPUT + + +@pytest.mark.asyncio +async def test_results_news_async() -> None: + instance = YouSearchAPIWrapper(endpoint_type="news", ydc_api_key="test_api_key") + + # Mock response object to simulate aiohttp response + mock_response = AsyncMock() + mock_response.__aenter__.return_value = ( + mock_response # Make the context manager return itself + ) + mock_response.__aexit__.return_value = None # No value needed for exit + mock_response.status = 200 + mock_response.json = AsyncMock(return_value=NEWS_RESPONSE_RAW) + + # Patch the aiohttp.ClientSession object + with patch("aiohttp.ClientSession.get", return_value=mock_response): + results = await instance.results_async("test query") + assert results == NEWS_RESPONSE_PARSED