Document loader/debug (#7750)

Description: Added debugging output in DirectoryLoader to identify the
file being processed.
Issue: [Need a trace or debug feature in Lanchain DirectoryLoader
#7725](https://github.com/hwchase17/langchain/issues/7725)
Dependencies: No additional dependencies are required.
Tag maintainer: @rlancemartin, @eyurtsev
This PR enhances the DirectoryLoader with debugging output to help
diagnose issues when loading documents. This new feature does not add
any dependencies and has been tested on a local machine.
pull/7752/head
rjarun8 1 year ago committed by GitHub
parent b015647e31
commit b7c409152a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -79,15 +79,17 @@ class DirectoryLoader(BaseLoader):
path: Directory path.
docs: List of documents to append to.
pbar: Progress bar. Defaults to None.
"""
if item.is_file():
if _is_visible(item.relative_to(path)) or self.load_hidden:
try:
logger.debug(f"Processing file: {str(item)}")
sub_docs = self.loader_cls(str(item), **self.loader_kwargs).load()
docs.extend(sub_docs)
except Exception as e:
if self.silent_errors:
logger.warning(e)
logger.warning(f"Error loading file {str(item)}: {e}")
else:
raise e
finally:
@ -134,6 +136,3 @@ class DirectoryLoader(BaseLoader):
pbar.close()
return docs
#

Loading…
Cancel
Save