diff --git a/langchain/document_loaders/web_base.py b/langchain/document_loaders/web_base.py index bd699eb8..7e3ff0bb 100644 --- a/langchain/document_loaders/web_base.py +++ b/langchain/document_loaders/web_base.py @@ -100,7 +100,13 @@ class WebBaseLoader(BaseLoader): async def _fetch( self, url: str, retries: int = 3, cooldown: int = 2, backoff: float = 1.5 ) -> str: - async with aiohttp.ClientSession() as session: + # For SiteMap SSL verification + if not self.requests_kwargs.get("verify", True): + connector = aiohttp.TCPConnector(ssl=False) + else: + connector = None + + async with aiohttp.ClientSession(connector=connector) as session: for i in range(retries): try: async with session.get(