fix RecursiveUrlLoader (#8582)

Description: the recursive url loader does not fully crawl for all urls under base url Maintainer: @baskaryan
1 year ago · b786335dd1
parent f81e613086
commit b786335dd1
1 changed files with 1 additions and 3 deletions
--- a/libs/langchain/langchain/document_loaders/recursive_url_loader.py
+++ b/libs/langchain/langchain/document_loaders/recursive_url_loader.py
@ -92,9 +92,7 @@ class RecursiveUrlLoader(BaseLoader):
                    yield from loaded_link
                else:
                    yield loaded_link
-                # If the link is a directory (w/ children) then visit it
-                if link.endswith("/"):
-                    yield from self.get_child_links_recursive(link, visited)
+                yield from self.get_child_links_recursive(link, visited)

        return visited