Archives/langchain
2
0
Fork 1
mirror of https://github.com/hwchase17/langchain synced 2024-11-06 03:20:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Document loader/debug (#7750)

Browse Source 
Description: Added debugging output in DirectoryLoader to identify the
file being processed.
Issue: [Need a trace or debug feature in Lanchain DirectoryLoader
#7725](https://github.com/hwchase17/langchain/issues/7725)
Dependencies: No additional dependencies are required.
Tag maintainer: @rlancemartin, @eyurtsev
This PR enhances the DirectoryLoader with debugging output to help
diagnose issues when loading documents. This new feature does not add
any dependencies and has been tested on a local machine.
This commit is contained in:
rjarun8 2023-07-15 19:48:27 +05:30 committed by GitHub
parent b015647e31
commit b7c409152a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
langchain/document_loaders/directory.py
View File

@ -79,15 +79,17 @@ class DirectoryLoader(BaseLoader):
path: Directory path. path: Directory path.
docs: List of documents to append to. docs: List of documents to append to.
pbar: Progress bar. Defaults to None. pbar: Progress bar. Defaults to None.
""" """
if item.is_file(): if item.is_file():
if _is_visible(item.relative_to(path)) or self.load_hidden: if _is_visible(item.relative_to(path)) or self.load_hidden:
try: try:
logger.debug(f"Processing file: {str(item)}")
sub_docs = self.loader_cls(str(item), **self.loader_kwargs).load() sub_docs = self.loader_cls(str(item), **self.loader_kwargs).load()
docs.extend(sub_docs) docs.extend(sub_docs)
except Exception as e: except Exception as e:
if self.silent_errors: if self.silent_errors:
logger.warning(e) logger.warning(f"Error loading file {str(item)}: {e}")
else: else:
raise e raise e
finally: finally:
@ -134,6 +136,3 @@ class DirectoryLoader(BaseLoader):
pbar.close() pbar.close()
return docs return docs
#