diff --git a/scripts/parser/open_ai_func.py b/scripts/parser/open_ai_func.py index cbd947ee..70091324 100644 --- a/scripts/parser/open_ai_func.py +++ b/scripts/parser/open_ai_func.py @@ -32,11 +32,14 @@ def call_openai_api(docs): print("Saving progress") print(f"stopped at {c1} out of {len(docs)}") faiss.write_index(store.index, "docs.index") + store_index_bak = store.index store.index = None with open("faiss_store.pkl", "wb") as f: pickle.dump(store, f) - print("Sleeping for 10 seconds and trying again") - time.sleep(10) + print("Sleeping for 60 seconds and trying again") + time.sleep(60) + faiss.write_index(store_index_bak, "docs.index") + store.index = store_index_bak store.add_texts([i.page_content], metadatas=[i.metadata]) c1 += 1 @@ -66,4 +69,4 @@ def get_user_permission(docs): elif user_input == "": call_openai_api(docs) else: - print("The API was not called. No money was spent.") \ No newline at end of file + print("The API was not called. No money was spent.")