mirror of
https://github.com/hwchase17/langchain
synced 2024-11-11 19:11:02 +00:00
9e06991aae
Update the URL to Markdown endpoint. API information is available here: https://2markdown.com/docs#url2md
31 lines
848 B
Python
31 lines
848 B
Python
from __future__ import annotations
|
|
|
|
from typing import Iterator
|
|
|
|
import requests
|
|
from langchain_core.documents import Document
|
|
|
|
from langchain_community.document_loaders.base import BaseLoader
|
|
|
|
|
|
class ToMarkdownLoader(BaseLoader):
|
|
"""Load `HTML` using `2markdown API`."""
|
|
|
|
def __init__(self, url: str, api_key: str):
|
|
"""Initialize with url and api key."""
|
|
self.url = url
|
|
self.api_key = api_key
|
|
|
|
def lazy_load(
|
|
self,
|
|
) -> Iterator[Document]:
|
|
"""Lazily load the file."""
|
|
response = requests.post(
|
|
"https://api.2markdown.com/v1/url2md",
|
|
headers={"X-Api-Key": self.api_key},
|
|
json={"url": self.url},
|
|
)
|
|
text = response.json()["article"]
|
|
metadata = {"source": self.url}
|
|
yield Document(page_content=text, metadata=metadata)
|