community: Implement lazy_load() for GithubFileLoader (#18584)

pull/17861/head
Christophe Bornet 3 months ago committed by GitHub
parent 04d134df17
commit c8a171a154
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -217,9 +217,7 @@ class GithubFileLoader(BaseGitHubLoader, ABC):
return ""
def load(self) -> List[Document]:
documents = []
def lazy_load(self) -> Iterator[Document]:
files = self.get_file_paths()
for file in files:
content = self.get_file_content_by_path(file["path"])
@ -232,6 +230,4 @@ class GithubFileLoader(BaseGitHubLoader, ABC):
"source": f"{self.github_api_url}/{self.repo}/{file['type']}/"
f"{self.branch}/{file['path']}",
}
documents.append(Document(page_content=content, metadata=metadata))
return documents
yield Document(page_content=content, metadata=metadata)

Loading…
Cancel
Save