Archives/DocsGPT
2
0
Fork 0
mirror of https://github.com/arc53/DocsGPT synced 2024-11-17 21:26:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #91 from EricGao888/fix-90

Browse Source 
Add retry strategy to increase stability
This commit is contained in:
Alex 2023-02-15 13:08:12 +00:00 committed by GitHub
commit 12cbc14fc3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
application/requirements.txt
View File

@ -45,6 +45,7 @@ pytz==2022.7.1
PyYAML==6.0 PyYAML==6.0
regex==2022.10.31 regex==2022.10.31
requests==2.28.2 requests==2.28.2
retry==0.9.2
six==1.16.0 six==1.16.0
snowballstemmer==2.2.0 snowballstemmer==2.2.0
Sphinx==6.1.3 Sphinx==6.1.3

7
scripts/parser/open_ai_func.py
View File

@ -4,6 +4,7 @@ import pickle
import tiktoken import tiktoken
from langchain.vectorstores import FAISS from langchain.vectorstores import FAISS
from langchain.embeddings import OpenAIEmbeddings from langchain.embeddings import OpenAIEmbeddings
from retry import retry
def num_tokens_from_string(string: str, encoding_name: str) -> int: def num_tokens_from_string(string: str, encoding_name: str) -> int:
@ -13,6 +14,10 @@ def num_tokens_from_string(string: str, encoding_name: str) -> int:
total_price = ((num_tokens/1000) * 0.0004) total_price = ((num_tokens/1000) * 0.0004)
return num_tokens, total_price return num_tokens, total_price
@retry(tries=10, delay=60)
def store_add_texts_with_retry(store, i):
store.add_texts([i.page_content], metadatas=[i.metadata])
def call_openai_api(docs, folder_name): def call_openai_api(docs, folder_name):
# Function to create a vector store from the documents and save it to disk. # Function to create a vector store from the documents and save it to disk.
@ -31,7 +36,7 @@ def call_openai_api(docs, folder_name):
for i in tqdm(docs, desc="Embedding 🦖", unit="docs", total=len(docs), bar_format='{l_bar}{bar}| Time Left: {remaining}'): for i in tqdm(docs, desc="Embedding 🦖", unit="docs", total=len(docs), bar_format='{l_bar}{bar}| Time Left: {remaining}'):
try: try:
import time import time
store.add_texts([i.page_content], metadatas=[i.metadata]) store_add_texts_with_retry(store, i)
except Exception as e: except Exception as e:
print(e) print(e)
print("Error on ", i) print("Error on ", i)