Let Notion document loader support utf-8 and make it default. (#10613)

Use utf-8 encoding by default
bagatur/rfc_vecstore_interface
Richard Wang 9 months ago committed by GitHub
parent 3759a34229
commit 6402c33299
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -8,16 +8,17 @@ from langchain.document_loaders.base import BaseLoader
class NotionDirectoryLoader(BaseLoader):
"""Load `Notion directory` dump."""
def __init__(self, path: str):
def __init__(self, path: str, *, encoding: str = "utf-8") -> None:
"""Initialize with a file path."""
self.file_path = path
self.encoding = encoding
def load(self) -> List[Document]:
"""Load documents."""
ps = list(Path(self.file_path).glob("**/*.md"))
paths = list(Path(self.file_path).glob("**/*.md"))
docs = []
for p in ps:
with open(p) as f:
for p in paths:
with open(p, encoding=self.encoding) as f:
text = f.read()
metadata = {"source": str(p)}
docs.append(Document(page_content=text, metadata=metadata))

Loading…
Cancel
Save