mirror of https://github.com/hwchase17/langchain
parent
47c3221fda
commit
d5f3dfa1e1
@ -0,0 +1,59 @@
|
||||
"""Loader that loads HN."""
|
||||
from typing import Any, List
|
||||
|
||||
from langchain.docstore.document import Document
|
||||
from langchain.document_loaders.web_base import WebBaseLoader
|
||||
|
||||
|
||||
class HNLoader(WebBaseLoader):
|
||||
"""Load Hacker News data from either main page results or the comments page."""
|
||||
|
||||
def load(self) -> List[Document]:
|
||||
"""Get important HN webpage information.
|
||||
|
||||
Components are:
|
||||
- title
|
||||
- content
|
||||
- source url,
|
||||
- time of post
|
||||
- author of the post
|
||||
- number of comments
|
||||
- rank of the post
|
||||
"""
|
||||
soup_info = self.scrape()
|
||||
if "item" in self.web_path:
|
||||
return self.load_comments(soup_info)
|
||||
else:
|
||||
return self.load_results(soup_info)
|
||||
|
||||
def load_comments(self, soup_info: Any) -> List[Document]:
|
||||
"""Load comments from a HN post."""
|
||||
comments = soup_info.select("tr[class='athing comtr']")
|
||||
title = soup_info.select_one("tr[id='pagespace']").get("title")
|
||||
documents = []
|
||||
for comment in comments:
|
||||
text = comment.text.strip()
|
||||
metadata = {"source": self.web_path, "title": title}
|
||||
documents.append(Document(page_content=text, metadata=metadata))
|
||||
return documents
|
||||
|
||||
def load_results(self, soup: Any) -> List[Document]:
|
||||
"""Load items from an HN page."""
|
||||
items = soup.select("tr[class='athing']")
|
||||
documents = []
|
||||
for lineItem in items:
|
||||
ranking = lineItem.select_one("span[class='rank']").text
|
||||
link = lineItem.find("span", {"class": "titleline"}).find("a").get("href")
|
||||
title = lineItem.find("span", {"class": "titleline"}).text.strip()
|
||||
metadata = {
|
||||
"source": self.web_path,
|
||||
"title": title,
|
||||
"link": link,
|
||||
"ranking": ranking,
|
||||
}
|
||||
documents.append(
|
||||
Document(
|
||||
page_content=title, link=link, ranking=ranking, metadata=metadata
|
||||
)
|
||||
)
|
||||
return documents
|
Loading…
Reference in New Issue