From 7ee8b2d1bf7a58a21b2bee8d5e45fb3835c3a3c5 Mon Sep 17 00:00:00 2001
From: Bagatur <22008038+baskaryan@users.noreply.github.com>
Date: Tue, 26 Sep 2023 09:59:04 -0700
Subject: [PATCH] exclude dirs in async recursive loading (#11077)

---
 .../langchain/langchain/document_loaders/recursive_url_loader.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libs/langchain/langchain/document_loaders/recursive_url_loader.py b/libs/langchain/langchain/document_loaders/recursive_url_loader.py
index 5bb2350c5c..a3fdbbcfcc 100644
--- a/libs/langchain/langchain/document_loaders/recursive_url_loader.py
+++ b/libs/langchain/langchain/document_loaders/recursive_url_loader.py
@@ -233,6 +233,7 @@ class RecursiveUrlLoader(BaseLoader):
                 base_url=self.url,
                 pattern=self.link_regex,
                 prevent_outside=self.prevent_outside,
+                exclude_prefixes=self.exclude_dirs,
             )
 
             # Recursively call the function to get the children of the children