Allow passing in encoding to csv_loader (#1836)

tool-patch
Daniel Dror (Dubovski) 1 year ago committed by GitHub
parent 9555bbd5bb
commit c592b12043
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -31,9 +31,11 @@ class CSVLoader(BaseLoader):
file_path: str,
source_column: Optional[str] = None,
csv_args: Optional[Dict] = None,
encoding: Optional[str] = None,
):
self.file_path = file_path
self.source_column = source_column
self.encoding = encoding
if csv_args is None:
self.csv_args = {
"delimiter": ",",
@ -45,7 +47,7 @@ class CSVLoader(BaseLoader):
def load(self) -> List[Document]:
docs = []
with open(self.file_path, newline="") as csvfile:
with open(self.file_path, newline="", encoding=self.encoding) as csvfile:
csv = DictReader(csvfile, **self.csv_args) # type: ignore
for i, row in enumerate(csv):
content = "\n".join(f"{k.strip()}: {v.strip()}" for k, v in row.items())

Loading…
Cancel
Save