Refactored requests (#8203)

Refactored `requests.py`. The same as
https://github.com/langchain-ai/langchain/pull/7961 #8098 #8099
requests.py is in the root code folder. This creates the
`langchain.requests: Requests` group on the API Reference navigation
ToC, on the same level as Chains and Agents which is incorrect.

Refactoring:

- copied requests.py content into utils/requests.py
- I added the backwards compatibility ref in the original requests.py. 
- updated imports to requests objects

@hwchase17, @baskaryan

libs/langchain/langchain/agents/agent_toolkits/nla/tool.py

@@ -5,10 +5,10 @@ from typing import Any, Optional
 
 from langchain.agents.tools import Tool
 from langchain.chains.api.openapi.chain import OpenAPIEndpointChain
-from langchain.requests import Requests
 from langchain.schema.language_model import BaseLanguageModel
 from langchain.tools.openapi.utils.api_models import APIOperation
 from langchain.tools.openapi.utils.openapi_utils import OpenAPISpec
+from langchain.utilities.requests import Requests
 
 
 class NLATool(Tool):

libs/langchain/langchain/agents/agent_toolkits/nla/toolkit.py

@@ -6,11 +6,11 @@ from pydantic import Field
 
 from langchain.agents.agent_toolkits.base import BaseToolkit
 from langchain.agents.agent_toolkits.nla.tool import NLATool
-from langchain.requests import Requests
 from langchain.schema.language_model import BaseLanguageModel
 from langchain.tools.base import BaseTool
 from langchain.tools.openapi.utils.openapi_utils import OpenAPISpec
 from langchain.tools.plugin import AIPlugin
+from langchain.utilities.requests import Requests
 
 
 class NLAToolkit(BaseToolkit):

libs/langchain/langchain/agents/agent_toolkits/openapi/planner.py

@@ -33,11 +33,11 @@ from langchain.chains.llm import LLMChain
 from langchain.llms.openai import OpenAI
 from langchain.memory import ReadOnlySharedMemory
 from langchain.prompts import PromptTemplate
-from langchain.requests import RequestsWrapper
 from langchain.schema import BasePromptTemplate
 from langchain.schema.language_model import BaseLanguageModel
 from langchain.tools.base import BaseTool
 from langchain.tools.requests.tool import BaseRequestsTool
+from langchain.utilities.requests import RequestsWrapper
 
 #
 # Requests tools with LLM-instructed extraction of truncated responses.

libs/langchain/langchain/agents/agent_toolkits/openapi/toolkit.py

@@ -9,7 +9,6 @@ from langchain.agents.agent_toolkits.json.base import create_json_agent
 from langchain.agents.agent_toolkits.json.toolkit import JsonToolkit
 from langchain.agents.agent_toolkits.openapi.prompt import DESCRIPTION
 from langchain.agents.tools import Tool
-from langchain.requests import TextRequestsWrapper
 from langchain.schema.language_model import BaseLanguageModel
 from langchain.tools import BaseTool
 from langchain.tools.json.tool import JsonSpec
@@ -20,6 +19,7 @@ from langchain.tools.requests.tool import (
     RequestsPostTool,
     RequestsPutTool,
 )
+from langchain.utilities.requests import TextRequestsWrapper
 
 
 class RequestsToolkit(BaseToolkit):

libs/langchain/langchain/agents/load_tools.py

@@ -12,7 +12,7 @@ from langchain.chains.api import news_docs, open_meteo_docs, podcast_docs, tmdb_
 from langchain.chains.api.base import APIChain
 from langchain.chains.llm_math.base import LLMMathChain
 from langchain.chains.pal.base import PALChain
-from langchain.requests import TextRequestsWrapper
+from langchain.utilities.requests import TextRequestsWrapper
 from langchain.tools.arxiv.tool import ArxivQueryRun
 from langchain.tools.golden_query.tool import GoldenQueryRun
 from langchain.tools.pubmed.tool import PubmedQueryRun

libs/langchain/langchain/chains/api/base.py

@@ -12,9 +12,9 @@ from langchain.callbacks.manager import (
 from langchain.chains.api.prompt import API_RESPONSE_PROMPT, API_URL_PROMPT
 from langchain.chains.base import Chain
 from langchain.chains.llm import LLMChain
-from langchain.requests import TextRequestsWrapper
 from langchain.schema import BasePromptTemplate
 from langchain.schema.language_model import BaseLanguageModel
+from langchain.utilities.requests import TextRequestsWrapper
 
 
 class APIChain(Chain):

libs/langchain/langchain/chains/api/openapi/chain.py

@@ -12,9 +12,9 @@ from langchain.chains.api.openapi.requests_chain import APIRequesterChain
 from langchain.chains.api.openapi.response_chain import APIResponderChain
 from langchain.chains.base import Chain
 from langchain.chains.llm import LLMChain
-from langchain.requests import Requests
 from langchain.schema.language_model import BaseLanguageModel
 from langchain.tools.openapi.utils.api_models import APIOperation
+from langchain.utilities.requests import Requests
 
 
 class _ParamMapping(NamedTuple):

libs/langchain/langchain/chains/llm_requests.py

@@ -8,7 +8,7 @@ from pydantic import Extra, Field, root_validator
 from langchain.callbacks.manager import CallbackManagerForChainRun
 from langchain.chains import LLMChain
 from langchain.chains.base import Chain
-from langchain.requests import TextRequestsWrapper
+from langchain.utilities.requests import TextRequestsWrapper
 
 DEFAULT_HEADERS = {
     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"  # noqa: E501

libs/langchain/langchain/requests.py

@@ -1,186 +1,8 @@
-"""Lightweight wrapper around requests library, with async support."""
+"""DEPRECATED: Kept for backwards compatibility."""
-from contextlib import asynccontextmanager
+from langchain.utilities import Requests, RequestsWrapper, TextRequestsWrapper
-from typing import Any, AsyncGenerator, Dict, Optional
 
-import aiohttp
+__all__ = [
-import requests
+    "Requests",
-from pydantic import BaseModel, Extra
+    "RequestsWrapper",
-
+    "TextRequestsWrapper",
-
+]
-class Requests(BaseModel):
-    """Wrapper around requests to handle auth and async.
-
-    The main purpose of this wrapper is to handle authentication (by saving
-    headers) and enable easy async methods on the same base object.
-    """
-
-    headers: Optional[Dict[str, str]] = None
-    aiosession: Optional[aiohttp.ClientSession] = None
-    auth: Optional[Any] = None
-
-    class Config:
-        """Configuration for this pydantic object."""
-
-        extra = Extra.forbid
-        arbitrary_types_allowed = True
-
-    def get(self, url: str, **kwargs: Any) -> requests.Response:
-        """GET the URL and return the text."""
-        return requests.get(url, headers=self.headers, auth=self.auth, **kwargs)
-
-    def post(self, url: str, data: Dict[str, Any], **kwargs: Any) -> requests.Response:
-        """POST to the URL and return the text."""
-        return requests.post(
-            url, json=data, headers=self.headers, auth=self.auth, **kwargs
-        )
-
-    def patch(self, url: str, data: Dict[str, Any], **kwargs: Any) -> requests.Response:
-        """PATCH the URL and return the text."""
-        return requests.patch(
-            url, json=data, headers=self.headers, auth=self.auth, **kwargs
-        )
-
-    def put(self, url: str, data: Dict[str, Any], **kwargs: Any) -> requests.Response:
-        """PUT the URL and return the text."""
-        return requests.put(
-            url, json=data, headers=self.headers, auth=self.auth, **kwargs
-        )
-
-    def delete(self, url: str, **kwargs: Any) -> requests.Response:
-        """DELETE the URL and return the text."""
-        return requests.delete(url, headers=self.headers, auth=self.auth, **kwargs)
-
-    @asynccontextmanager
-    async def _arequest(
-        self, method: str, url: str, **kwargs: Any
-    ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
-        """Make an async request."""
-        if not self.aiosession:
-            async with aiohttp.ClientSession() as session:
-                async with session.request(
-                    method, url, headers=self.headers, auth=self.auth, **kwargs
-                ) as response:
-                    yield response
-        else:
-            async with self.aiosession.request(
-                method, url, headers=self.headers, auth=self.auth, **kwargs
-            ) as response:
-                yield response
-
-    @asynccontextmanager
-    async def aget(
-        self, url: str, **kwargs: Any
-    ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
-        """GET the URL and return the text asynchronously."""
-        async with self._arequest("GET", url, auth=self.auth, **kwargs) as response:
-            yield response
-
-    @asynccontextmanager
-    async def apost(
-        self, url: str, data: Dict[str, Any], **kwargs: Any
-    ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
-        """POST to the URL and return the text asynchronously."""
-        async with self._arequest(
-            "POST", url, json=data, auth=self.auth, **kwargs
-        ) as response:
-            yield response
-
-    @asynccontextmanager
-    async def apatch(
-        self, url: str, data: Dict[str, Any], **kwargs: Any
-    ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
-        """PATCH the URL and return the text asynchronously."""
-        async with self._arequest(
-            "PATCH", url, json=data, auth=self.auth, **kwargs
-        ) as response:
-            yield response
-
-    @asynccontextmanager
-    async def aput(
-        self, url: str, data: Dict[str, Any], **kwargs: Any
-    ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
-        """PUT the URL and return the text asynchronously."""
-        async with self._arequest(
-            "PUT", url, json=data, auth=self.auth, **kwargs
-        ) as response:
-            yield response
-
-    @asynccontextmanager
-    async def adelete(
-        self, url: str, **kwargs: Any
-    ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
-        """DELETE the URL and return the text asynchronously."""
-        async with self._arequest("DELETE", url, auth=self.auth, **kwargs) as response:
-            yield response
-
-
-class TextRequestsWrapper(BaseModel):
-    """Lightweight wrapper around requests library.
-
-    The main purpose of this wrapper is to always return a text output.
-    """
-
-    headers: Optional[Dict[str, str]] = None
-    aiosession: Optional[aiohttp.ClientSession] = None
-    auth: Optional[Any] = None
-
-    class Config:
-        """Configuration for this pydantic object."""
-
-        extra = Extra.forbid
-        arbitrary_types_allowed = True
-
-    @property
-    def requests(self) -> Requests:
-        return Requests(
-            headers=self.headers, aiosession=self.aiosession, auth=self.auth
-        )
-
-    def get(self, url: str, **kwargs: Any) -> str:
-        """GET the URL and return the text."""
-        return self.requests.get(url, **kwargs).text
-
-    def post(self, url: str, data: Dict[str, Any], **kwargs: Any) -> str:
-        """POST to the URL and return the text."""
-        return self.requests.post(url, data, **kwargs).text
-
-    def patch(self, url: str, data: Dict[str, Any], **kwargs: Any) -> str:
-        """PATCH the URL and return the text."""
-        return self.requests.patch(url, data, **kwargs).text
-
-    def put(self, url: str, data: Dict[str, Any], **kwargs: Any) -> str:
-        """PUT the URL and return the text."""
-        return self.requests.put(url, data, **kwargs).text
-
-    def delete(self, url: str, **kwargs: Any) -> str:
-        """DELETE the URL and return the text."""
-        return self.requests.delete(url, **kwargs).text
-
-    async def aget(self, url: str, **kwargs: Any) -> str:
-        """GET the URL and return the text asynchronously."""
-        async with self.requests.aget(url, **kwargs) as response:
-            return await response.text()
-
-    async def apost(self, url: str, data: Dict[str, Any], **kwargs: Any) -> str:
-        """POST to the URL and return the text asynchronously."""
-        async with self.requests.apost(url, data, **kwargs) as response:
-            return await response.text()
-
-    async def apatch(self, url: str, data: Dict[str, Any], **kwargs: Any) -> str:
-        """PATCH the URL and return the text asynchronously."""
-        async with self.requests.apatch(url, data, **kwargs) as response:
-            return await response.text()
-
-    async def aput(self, url: str, data: Dict[str, Any], **kwargs: Any) -> str:
-        """PUT the URL and return the text asynchronously."""
-        async with self.requests.aput(url, data, **kwargs) as response:
-            return await response.text()
-
-    async def adelete(self, url: str, **kwargs: Any) -> str:
-        """DELETE the URL and return the text asynchronously."""
-        async with self.requests.adelete(url, **kwargs) as response:
-            return await response.text()
-
-
-# For backwards compatibility
-RequestsWrapper = TextRequestsWrapper

libs/langchain/langchain/tools/requests/tool.py

@@ -9,7 +9,7 @@ from langchain.callbacks.manager import (
     CallbackManagerForToolRun,
 )
 
-from langchain.requests import TextRequestsWrapper
+from langchain.utilities.requests import TextRequestsWrapper
 from langchain.tools.base import BaseTool
 
 

libs/langchain/langchain/utilities/__init__.py

@@ -1,5 +1,4 @@
-"""General utilities."""
+"""Generic integrations with third-part systems and packages."""
-from langchain.requests import TextRequestsWrapper
 from langchain.utilities.arxiv import ArxivAPIWrapper
 from langchain.utilities.awslambda import LambdaWrapper
 from langchain.utilities.bash import BashProcess
@@ -20,6 +19,7 @@ from langchain.utilities.portkey import Portkey
 from langchain.utilities.powerbi import PowerBIDataset
 from langchain.utilities.pupmed import PubMedAPIWrapper
 from langchain.utilities.python import PythonREPL
+from langchain.utilities.requests import Requests, RequestsWrapper, TextRequestsWrapper
 from langchain.utilities.scenexplain import SceneXplainAPIWrapper
 from langchain.utilities.searx_search import SearxSearchWrapper
 from langchain.utilities.serpapi import SerpAPIWrapper
@@ -32,12 +32,12 @@ from langchain.utilities.zapier import ZapierNLAWrapper
 
 __all__ = [
     "ArxivAPIWrapper",
-    "GoldenQueryAPIWrapper",
     "BashProcess",
     "BibtexparserWrapper",
     "BingSearchAPIWrapper",
     "BraveSearchWrapper",
     "DuckDuckGoSearchAPIWrapper",
+    "GoldenQueryAPIWrapper",
     "GooglePlacesAPIWrapper",
     "GoogleSearchAPIWrapper",
     "GoogleSerperAPIWrapper",
@@ -51,11 +51,14 @@ __all__ = [
     "PowerBIDataset",
     "PubMedAPIWrapper",
     "PythonREPL",
+    "Requests",
+    "RequestsWrapper",
+    "SQLDatabase",
     "SceneXplainAPIWrapper",
     "SearxSearchWrapper",
     "SerpAPIWrapper",
     "SparkSQL",
-    "SQLDatabase",
+    "TextRequestsWrapper",
     "TextRequestsWrapper",
     "TwilioAPIWrapper",
     "WikipediaAPIWrapper",

libs/langchain/langchain/utilities/requests.py

@@ -0,0 +1,186 @@
+"""Lightweight wrapper around requests library, with async support."""
+from contextlib import asynccontextmanager
+from typing import Any, AsyncGenerator, Dict, Optional
+
+import aiohttp
+import requests
+from pydantic import BaseModel, Extra
+
+
+class Requests(BaseModel):
+    """Wrapper around requests to handle auth and async.
+
+    The main purpose of this wrapper is to handle authentication (by saving
+    headers) and enable easy async methods on the same base object.
+    """
+
+    headers: Optional[Dict[str, str]] = None
+    aiosession: Optional[aiohttp.ClientSession] = None
+    auth: Optional[Any] = None
+
+    class Config:
+        """Configuration for this pydantic object."""
+
+        extra = Extra.forbid
+        arbitrary_types_allowed = True
+
+    def get(self, url: str, **kwargs: Any) -> requests.Response:
+        """GET the URL and return the text."""
+        return requests.get(url, headers=self.headers, auth=self.auth, **kwargs)
+
+    def post(self, url: str, data: Dict[str, Any], **kwargs: Any) -> requests.Response:
+        """POST to the URL and return the text."""
+        return requests.post(
+            url, json=data, headers=self.headers, auth=self.auth, **kwargs
+        )
+
+    def patch(self, url: str, data: Dict[str, Any], **kwargs: Any) -> requests.Response:
+        """PATCH the URL and return the text."""
+        return requests.patch(
+            url, json=data, headers=self.headers, auth=self.auth, **kwargs
+        )
+
+    def put(self, url: str, data: Dict[str, Any], **kwargs: Any) -> requests.Response:
+        """PUT the URL and return the text."""
+        return requests.put(
+            url, json=data, headers=self.headers, auth=self.auth, **kwargs
+        )
+
+    def delete(self, url: str, **kwargs: Any) -> requests.Response:
+        """DELETE the URL and return the text."""
+        return requests.delete(url, headers=self.headers, auth=self.auth, **kwargs)
+
+    @asynccontextmanager
+    async def _arequest(
+        self, method: str, url: str, **kwargs: Any
+    ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
+        """Make an async request."""
+        if not self.aiosession:
+            async with aiohttp.ClientSession() as session:
+                async with session.request(
+                    method, url, headers=self.headers, auth=self.auth, **kwargs
+                ) as response:
+                    yield response
+        else:
+            async with self.aiosession.request(
+                method, url, headers=self.headers, auth=self.auth, **kwargs
+            ) as response:
+                yield response
+
+    @asynccontextmanager
+    async def aget(
+        self, url: str, **kwargs: Any
+    ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
+        """GET the URL and return the text asynchronously."""
+        async with self._arequest("GET", url, auth=self.auth, **kwargs) as response:
+            yield response
+
+    @asynccontextmanager
+    async def apost(
+        self, url: str, data: Dict[str, Any], **kwargs: Any
+    ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
+        """POST to the URL and return the text asynchronously."""
+        async with self._arequest(
+            "POST", url, json=data, auth=self.auth, **kwargs
+        ) as response:
+            yield response
+
+    @asynccontextmanager
+    async def apatch(
+        self, url: str, data: Dict[str, Any], **kwargs: Any
+    ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
+        """PATCH the URL and return the text asynchronously."""
+        async with self._arequest(
+            "PATCH", url, json=data, auth=self.auth, **kwargs
+        ) as response:
+            yield response
+
+    @asynccontextmanager
+    async def aput(
+        self, url: str, data: Dict[str, Any], **kwargs: Any
+    ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
+        """PUT the URL and return the text asynchronously."""
+        async with self._arequest(
+            "PUT", url, json=data, auth=self.auth, **kwargs
+        ) as response:
+            yield response
+
+    @asynccontextmanager
+    async def adelete(
+        self, url: str, **kwargs: Any
+    ) -> AsyncGenerator[aiohttp.ClientResponse, None]:
+        """DELETE the URL and return the text asynchronously."""
+        async with self._arequest("DELETE", url, auth=self.auth, **kwargs) as response:
+            yield response
+
+
+class TextRequestsWrapper(BaseModel):
+    """Lightweight wrapper around requests library.
+
+    The main purpose of this wrapper is to always return a text output.
+    """
+
+    headers: Optional[Dict[str, str]] = None
+    aiosession: Optional[aiohttp.ClientSession] = None
+    auth: Optional[Any] = None
+
+    class Config:
+        """Configuration for this pydantic object."""
+
+        extra = Extra.forbid
+        arbitrary_types_allowed = True
+
+    @property
+    def requests(self) -> Requests:
+        return Requests(
+            headers=self.headers, aiosession=self.aiosession, auth=self.auth
+        )
+
+    def get(self, url: str, **kwargs: Any) -> str:
+        """GET the URL and return the text."""
+        return self.requests.get(url, **kwargs).text
+
+    def post(self, url: str, data: Dict[str, Any], **kwargs: Any) -> str:
+        """POST to the URL and return the text."""
+        return self.requests.post(url, data, **kwargs).text
+
+    def patch(self, url: str, data: Dict[str, Any], **kwargs: Any) -> str:
+        """PATCH the URL and return the text."""
+        return self.requests.patch(url, data, **kwargs).text
+
+    def put(self, url: str, data: Dict[str, Any], **kwargs: Any) -> str:
+        """PUT the URL and return the text."""
+        return self.requests.put(url, data, **kwargs).text
+
+    def delete(self, url: str, **kwargs: Any) -> str:
+        """DELETE the URL and return the text."""
+        return self.requests.delete(url, **kwargs).text
+
+    async def aget(self, url: str, **kwargs: Any) -> str:
+        """GET the URL and return the text asynchronously."""
+        async with self.requests.aget(url, **kwargs) as response:
+            return await response.text()
+
+    async def apost(self, url: str, data: Dict[str, Any], **kwargs: Any) -> str:
+        """POST to the URL and return the text asynchronously."""
+        async with self.requests.apost(url, data, **kwargs) as response:
+            return await response.text()
+
+    async def apatch(self, url: str, data: Dict[str, Any], **kwargs: Any) -> str:
+        """PATCH the URL and return the text asynchronously."""
+        async with self.requests.apatch(url, data, **kwargs) as response:
+            return await response.text()
+
+    async def aput(self, url: str, data: Dict[str, Any], **kwargs: Any) -> str:
+        """PUT the URL and return the text asynchronously."""
+        async with self.requests.aput(url, data, **kwargs) as response:
+            return await response.text()
+
+    async def adelete(self, url: str, **kwargs: Any) -> str:
+        """DELETE the URL and return the text asynchronously."""
+        async with self.requests.adelete(url, **kwargs) as response:
+            return await response.text()
+
+
+# For backwards compatibility
+RequestsWrapper = TextRequestsWrapper

libs/langchain/tests/unit_tests/chains/test_api.py

@@ -8,7 +8,7 @@ import pytest
 from langchain import LLMChain
 from langchain.chains.api.base import APIChain
 from langchain.chains.api.prompt import API_RESPONSE_PROMPT, API_URL_PROMPT
-from langchain.requests import TextRequestsWrapper
+from langchain.utilities.requests import TextRequestsWrapper
 from tests.unit_tests.llms.fake_llm import FakeLLM
 
 

libs/langchain/tests/unit_tests/tools/requests/test_tool.py

@@ -3,7 +3,6 @@ from typing import Any, Dict
 
 import pytest
 
-from langchain.requests import TextRequestsWrapper
 from langchain.tools.requests.tool import (
     RequestsDeleteTool,
     RequestsGetTool,
@@ -12,6 +11,7 @@ from langchain.tools.requests.tool import (
     RequestsPutTool,
     _parse_input,
 )
+from langchain.utilities.requests import TextRequestsWrapper
 
 
 class _MockTextRequestsWrapper(TextRequestsWrapper):