mirror of https://github.com/arc53/DocsGPT
You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
27 lines
1000 B
Python
27 lines
1000 B
Python
from application.parser.remote.base import BaseRemote
|
|
from langchain_community.document_loaders import RedditPostsLoader
|
|
|
|
|
|
class RedditPostsLoaderRemote(BaseRemote):
|
|
def load_data(self, inputs):
|
|
data = eval(inputs)
|
|
client_id = data.get("client_id")
|
|
client_secret = data.get("client_secret")
|
|
user_agent = data.get("user_agent")
|
|
categories = data.get("categories", ["new", "hot"])
|
|
mode = data.get("mode", "subreddit")
|
|
search_queries = data.get("search_queries")
|
|
number_posts = data.get("number_posts", 10)
|
|
self.loader = RedditPostsLoader(
|
|
client_id=client_id,
|
|
client_secret=client_secret,
|
|
user_agent=user_agent,
|
|
categories=categories,
|
|
mode=mode,
|
|
search_queries=search_queries,
|
|
number_posts=number_posts,
|
|
)
|
|
documents = self.loader.load()
|
|
print(f"Loaded {len(documents)} documents from Reddit")
|
|
return documents
|