mirror of
https://github.com/hwchase17/langchain
synced 2024-11-04 06:00:26 +00:00
fa5d49f2c1
ran ```bash g grep -l "langchain.vectorstores" | xargs -L 1 sed -i '' "s/langchain\.vectorstores/langchain_community.vectorstores/g" g grep -l "langchain.document_loaders" | xargs -L 1 sed -i '' "s/langchain\.document_loaders/langchain_community.document_loaders/g" g grep -l "langchain.chat_loaders" | xargs -L 1 sed -i '' "s/langchain\.chat_loaders/langchain_community.chat_loaders/g" g grep -l "langchain.document_transformers" | xargs -L 1 sed -i '' "s/langchain\.document_transformers/langchain_community.document_transformers/g" g grep -l "langchain\.graphs" | xargs -L 1 sed -i '' "s/langchain\.graphs/langchain_community.graphs/g" g grep -l "langchain\.memory\.chat_message_histories" | xargs -L 1 sed -i '' "s/langchain\.memory\.chat_message_histories/langchain_community.chat_message_histories/g" gco master libs/langchain/tests/unit_tests/*/test_imports.py gco master libs/langchain/tests/unit_tests/**/test_public_api.py ```
46 lines
1.5 KiB
Python
46 lines
1.5 KiB
Python
import os
|
|
|
|
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
from langchain_community.document_loaders import UnstructuredFileLoader
|
|
from langchain_community.embeddings import HuggingFaceEmbeddings
|
|
from langchain_community.vectorstores import Redis
|
|
from rag_redis.config import EMBED_MODEL, INDEX_NAME, INDEX_SCHEMA, REDIS_URL
|
|
|
|
|
|
def ingest_documents():
|
|
"""
|
|
Ingest PDF to Redis from the data/ directory that
|
|
contains Edgar 10k filings data for Nike.
|
|
"""
|
|
# Load list of pdfs
|
|
company_name = "Nike"
|
|
data_path = "data/"
|
|
doc = [os.path.join(data_path, file) for file in os.listdir(data_path)][0]
|
|
|
|
print("Parsing 10k filing doc for NIKE", doc)
|
|
|
|
text_splitter = RecursiveCharacterTextSplitter(
|
|
chunk_size=1500, chunk_overlap=100, add_start_index=True
|
|
)
|
|
loader = UnstructuredFileLoader(doc, mode="single", strategy="fast")
|
|
chunks = loader.load_and_split(text_splitter)
|
|
|
|
print("Done preprocessing. Created", len(chunks), "chunks of the original pdf")
|
|
# Create vectorstore
|
|
embedder = HuggingFaceEmbeddings(model_name=EMBED_MODEL)
|
|
|
|
_ = Redis.from_texts(
|
|
# appending this little bit can sometimes help with semantic retrieval
|
|
# especially with multiple companies
|
|
texts=[f"Company: {company_name}. " + chunk.page_content for chunk in chunks],
|
|
metadatas=[chunk.metadata for chunk in chunks],
|
|
embedding=embedder,
|
|
index_name=INDEX_NAME,
|
|
index_schema=INDEX_SCHEMA,
|
|
redis_url=REDIS_URL,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
ingest_documents()
|