mirror of
https://github.com/hwchase17/langchain
synced 2024-11-10 01:10:59 +00:00
ed58eeb9c5
Moved the following modules to new package langchain-community in a backwards compatible fashion: ``` mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community ``` Moved the following to core ``` mv langchain/langchain/utils/json_schema.py core/langchain_core/utils mv langchain/langchain/utils/html.py core/langchain_core/utils mv langchain/langchain/utils/strings.py core/langchain_core/utils cat langchain/langchain/utils/env.py >> core/langchain_core/utils/env.py rm langchain/langchain/utils/env.py ``` See .scripts/community_split/script_integrations.sh for all changes
80 lines
2.4 KiB
Python
80 lines
2.4 KiB
Python
import json
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import Iterator, Union
|
|
|
|
from langchain_core.chat_sessions import ChatSession
|
|
from langchain_core.messages import HumanMessage
|
|
|
|
from langchain_community.chat_loaders.base import BaseChatLoader
|
|
|
|
logger = logging.getLogger(__file__)
|
|
|
|
|
|
class SingleFileFacebookMessengerChatLoader(BaseChatLoader):
|
|
"""Load `Facebook Messenger` chat data from a single file.
|
|
|
|
Args:
|
|
path (Union[Path, str]): The path to the chat file.
|
|
|
|
Attributes:
|
|
path (Path): The path to the chat file.
|
|
|
|
"""
|
|
|
|
def __init__(self, path: Union[Path, str]) -> None:
|
|
super().__init__()
|
|
self.file_path = path if isinstance(path, Path) else Path(path)
|
|
|
|
def lazy_load(self) -> Iterator[ChatSession]:
|
|
"""Lazy loads the chat data from the file.
|
|
|
|
Yields:
|
|
ChatSession: A chat session containing the loaded messages.
|
|
|
|
"""
|
|
with open(self.file_path) as f:
|
|
data = json.load(f)
|
|
sorted_data = sorted(data["messages"], key=lambda x: x["timestamp_ms"])
|
|
messages = []
|
|
for m in sorted_data:
|
|
messages.append(
|
|
HumanMessage(
|
|
content=m["content"], additional_kwargs={"sender": m["sender_name"]}
|
|
)
|
|
)
|
|
yield ChatSession(messages=messages)
|
|
|
|
|
|
class FolderFacebookMessengerChatLoader(BaseChatLoader):
|
|
"""Load `Facebook Messenger` chat data from a folder.
|
|
|
|
Args:
|
|
path (Union[str, Path]): The path to the directory
|
|
containing the chat files.
|
|
|
|
Attributes:
|
|
path (Path): The path to the directory containing the chat files.
|
|
|
|
"""
|
|
|
|
def __init__(self, path: Union[str, Path]) -> None:
|
|
super().__init__()
|
|
self.directory_path = Path(path) if isinstance(path, str) else path
|
|
|
|
def lazy_load(self) -> Iterator[ChatSession]:
|
|
"""Lazy loads the chat data from the folder.
|
|
|
|
Yields:
|
|
ChatSession: A chat session containing the loaded messages.
|
|
|
|
"""
|
|
inbox_path = self.directory_path / "inbox"
|
|
for _dir in inbox_path.iterdir():
|
|
if _dir.is_dir():
|
|
for _file in _dir.iterdir():
|
|
if _file.suffix.lower() == ".json":
|
|
file_loader = SingleFileFacebookMessengerChatLoader(path=_file)
|
|
for result in file_loader.lazy_load():
|
|
yield result
|