From 50970074076e3c5e2b4af5fb94264cf8ac685f6c Mon Sep 17 00:00:00 2001 From: Bagatur <22008038+baskaryan@users.noreply.github.com> Date: Thu, 21 Sep 2023 07:22:13 -0700 Subject: [PATCH] cleanup recursive url session (#10863) --- .../document_loaders/recursive_url_loader.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/libs/langchain/langchain/document_loaders/recursive_url_loader.py b/libs/langchain/langchain/document_loaders/recursive_url_loader.py index 45f27a8d78..f531de2c8f 100644 --- a/libs/langchain/langchain/document_loaders/recursive_url_loader.py +++ b/libs/langchain/langchain/document_loaders/recursive_url_loader.py @@ -179,10 +179,14 @@ class RecursiveUrlLoader(BaseLoader): # Disable SSL verification because websites may have invalid SSL certificates, # but won't cause any security issues for us. close_session = session is None - session = session or aiohttp.ClientSession( - connector=aiohttp.TCPConnector(ssl=False), - timeout=aiohttp.ClientTimeout(total=self.timeout), - headers=self.headers, + session = ( + session + if session is not None + else aiohttp.ClientSession( + connector=aiohttp.TCPConnector(ssl=False), + timeout=aiohttp.ClientTimeout(total=self.timeout), + headers=self.headers, + ) ) async with self._lock: # type: ignore visited.add(url) @@ -194,6 +198,8 @@ class RecursiveUrlLoader(BaseLoader): f"Unable to load {url}. Received error {e} of type " f"{e.__class__.__name__}" ) + if close_session: + await session.close() return [] results = [] content = self.extractor(text)