Add `recursive` parameter to `DirectoryLoader` (#1389)

This PR allows loading a directory recursively.

langchain/document_loaders/directory.py

@@ -30,6 +30,7 @@ class DirectoryLoader(BaseLoader):
         silent_errors: bool = False,
         load_hidden: bool = False,
         loader_cls: FILE_LOADER_TYPE = UnstructuredFileLoader,
+        recursive: bool = False,
     ):
         """Initialize with path to directory and how to glob over it."""
         self.path = path
@@ -37,12 +38,14 @@ class DirectoryLoader(BaseLoader):
         self.load_hidden = load_hidden
         self.loader_cls = loader_cls
         self.silent_errors = silent_errors
+        self.recursive = recursive
 
     def load(self) -> List[Document]:
         """Load documents."""
         p = Path(self.path)
         docs = []
-        for i in p.glob(self.glob):
+        items = p.rglob(self.glob) if self.recursive else p.glob(self.glob)
+        for i in items:
             if i.is_file():
                 if _is_visible(i.relative_to(p)) or self.load_hidden:
                     try: