Merge pull request #109 from arc53/main

updates
pull/116/head
Alex 1 year ago committed by GitHub
commit 962be4d8ec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -61,5 +61,7 @@ Copy .env_sample and create .env with your openai api token
## [How to use any other documentation](https://github.com/arc53/docsgpt/wiki/How-to-train-on-other-documentation)
## [How to host it locally (so all data will stay on-premises)](https://github.com/arc53/DocsGPT/wiki/How-to-use-different-LLM's#hosting-everything-locally)
Built with [🦜️🔗 LangChain](https://github.com/hwchase17/langchain)

@ -1,54 +1,88 @@
import os
import pickle
import json
import dotenv
import datetime
import requests
from flask import Flask, request, render_template
# os.environ["LANGCHAIN_HANDLER"] = "langchain"
import faiss
from langchain import OpenAI, VectorDBQA
from langchain import FAISS
from langchain import OpenAI, VectorDBQA, HuggingFaceHub, Cohere
from langchain.chains.question_answering import load_qa_chain
from langchain.embeddings import OpenAIEmbeddings, HuggingFaceHubEmbeddings, CohereEmbeddings, HuggingFaceInstructEmbeddings
from langchain.prompts import PromptTemplate
import requests
# os.environ["LANGCHAIN_HANDLER"] = "langchain"
if os.getenv("LLM_NAME") is not None:
llm_choice = os.getenv("LLM_NAME")
else:
llm_choice = "openai"
if os.getenv("EMBEDDINGS_NAME") is not None:
embeddings_choice = os.getenv("EMBEDDINGS_NAME")
else:
embeddings_choice = "openai_text-embedding-ada-002"
if llm_choice == "manifest":
from manifest import Manifest
from langchain.llms.manifest import ManifestWrapper
manifest = Manifest(
client_name="huggingface",
client_connection="http://127.0.0.1:5000"
)
# Redirect PosixPath to WindowsPath on Windows
import platform
if platform.system() == "Windows":
import pathlib
temp = pathlib.PosixPath
pathlib.PosixPath = pathlib.WindowsPath
# loading the .env file
dotenv.load_dotenv()
with open("combine_prompt.txt", "r") as f:
template = f.read()
# check if OPENAI_API_KEY is set
if os.getenv("OPENAI_API_KEY") is not None:
api_key_set = True
with open("combine_prompt_hist.txt", "r") as f:
template_hist = f.read()
if os.getenv("API_KEY") is not None:
api_key_set = True
else:
api_key_set = False
if os.getenv("EMBEDDINGS_KEY") is not None:
embeddings_key_set = True
else:
embeddings_key_set = False
app = Flask(__name__)
@app.route("/")
def home():
return render_template("index.html", api_key_set=api_key_set)
return render_template("index.html", api_key_set=api_key_set, llm_choice=llm_choice,
embeddings_choice=embeddings_choice)
@app.route("/api/answer", methods=["POST"])
def api_answer():
data = request.get_json()
question = data["question"]
history = data["history"]
if not api_key_set:
api_key = data["api_key"]
else:
api_key = os.getenv("OPENAI_API_KEY")
api_key = os.getenv("API_KEY")
if not embeddings_key_set:
embeddings_key = data["embeddings_key"]
else:
embeddings_key = os.getenv("EMBEDDINGS_KEY")
# check if the vectorstore is set
if "active_docs" in data:
@ -59,25 +93,37 @@ def api_answer():
vectorstore = ""
# loading the index and the store and the prompt template
index = faiss.read_index(f"{vectorstore}docs.index")
# Note if you have used other embeddings than OpenAI, you need to change the embeddings
if embeddings_choice == "openai_text-embedding-ada-002":
docsearch = FAISS.load_local(vectorstore, OpenAIEmbeddings(openai_api_key=embeddings_key))
elif embeddings_choice == "huggingface_sentence-transformers/all-mpnet-base-v2":
docsearch = FAISS.load_local(vectorstore, HuggingFaceHubEmbeddings())
elif embeddings_choice == "huggingface_hkunlp/instructor-large":
docsearch = FAISS.load_local(vectorstore, HuggingFaceInstructEmbeddings())
elif embeddings_choice == "cohere_medium":
docsearch = FAISS.load_local(vectorstore, CohereEmbeddings(cohere_api_key=embeddings_key))
with open(f"{vectorstore}faiss_store.pkl", "rb") as f:
store = pickle.load(f)
store.index = index
# create a prompt template
c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template)
# create a chain with the prompt template and the store
#chain = VectorDBQA.from_llm(llm=OpenAI(openai_api_key=api_key, temperature=0), vectorstore=store, combine_prompt=c_prompt)
# chain = VectorDBQA.from_chain_type(llm=OpenAI(openai_api_key=api_key, temperature=0), chain_type='map_reduce',
# vectorstore=store)
qa_chain = load_qa_chain(OpenAI(openai_api_key=api_key, temperature=0), chain_type="map_reduce",
if history:
history = json.loads(history)
template_temp = template_hist.replace("{historyquestion}", history[0]).replace("{historyanswer}", history[1])
c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template_temp, template_format="jinja2")
else:
c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template, template_format="jinja2")
if llm_choice == "openai":
llm = OpenAI(openai_api_key=api_key, temperature=0)
elif llm_choice == "manifest":
llm = ManifestWrapper(client=manifest, llm_kwargs={"temperature": 0.001, "max_tokens": 2048})
elif llm_choice == "huggingface":
llm = HuggingFaceHub(repo_id="bigscience/bloom", huggingfacehub_api_token=api_key)
elif llm_choice == "cohere":
llm = Cohere(model="command-xlarge-nightly", cohere_api_key=api_key)
qa_chain = load_qa_chain(llm=llm, chain_type="map_reduce",
combine_prompt=c_prompt)
chain = VectorDBQA(combine_documents_chain=qa_chain, vectorstore=store)
chain = VectorDBQA(combine_documents_chain=qa_chain, vectorstore=docsearch, k=4)
# fetch the answer
result = chain({"query": question})
@ -94,30 +140,33 @@ def api_answer():
# }
return result
@app.route("/api/docs_check", methods=["POST"])
def check_docs():
# check if docs exist in a vectorstore folder
data = request.get_json()
vectorstore = "vectors/" + data["docs"]
base_path = 'https://raw.githubusercontent.com/arc53/DocsHUB/main/'
#
if os.path.exists(vectorstore):
if os.path.exists(vectorstore) or data["docs"] == "default":
return {"status": 'exists'}
else:
r = requests.get(base_path + vectorstore + "docs.index")
# save to vectors directory
# check if the directory exists
if not os.path.exists(vectorstore):
os.makedirs(vectorstore)
with open(vectorstore + "docs.index", "wb") as f:
f.write(r.content)
# download the store
r = requests.get(base_path + vectorstore + "faiss_store.pkl")
with open(vectorstore + "faiss_store.pkl", "wb") as f:
f.write(r.content)
return {"status": 'loaded'}
r = requests.get(base_path + vectorstore + "index.faiss")
if r.status_code != 200:
return {"status": 'null'}
else:
if not os.path.exists(vectorstore):
os.makedirs(vectorstore)
with open(vectorstore + "index.faiss", "wb") as f:
f.write(r.content)
# download the store
r = requests.get(base_path + vectorstore + "index.pkl")
with open(vectorstore + "index.pkl", "wb") as f:
f.write(r.content)
return {"status": 'loaded'}
# handling CORS
@app.after_request

@ -20,8 +20,8 @@ Source: 0-pl
FINAL ANSWER: You can't eat vegetables using pandas. You can only eat them using your mouth.
SOURCES:
QUESTION: {question}
QUESTION: {{ question }}
=========
{summaries}
{{ summaries }}
=========
FINAL ANSWER:

@ -0,0 +1,27 @@
You are a DocsGPT bot assistant by Arc53 that provides help with programming libraries. You give thorough answers with code examples.
Given the following extracted parts of a long document and a question, create a final answer with references ("SOURCES").
ALWAYS return a "SOURCES" part in your answer. You can also remeber things from previous questions and use them in your answer.
QUESTION: How to merge tables in pandas?
=========
Content: pandas provides various facilities for easily combining together Series or DataFrame with various kinds of set logic for the indexes and relational algebra functionality in the case of join / merge-type operations.
Source: 28-pl
Content: pandas provides a single function, merge(), as the entry point for all standard database join operations between DataFrame or named Series objects: \n\npandas.merge(left, right, how='inner', on=None, left_on=None, right_on=None, left_index=False, right_index=False, sort=False, suffixes=('_x', '_y'), copy=True, indicator=False, validate=None)
Source: 30-pl
=========
FINAL ANSWER: To merge two tables in pandas, you can use the pd.merge() function. The basic syntax is: \n\npd.merge(left, right, on, how) \n\nwhere left and right are the two tables to merge, on is the column to merge on, and how is the type of merge to perform. \n\nFor example, to merge the two tables df1 and df2 on the column 'id', you can use: \n\npd.merge(df1, df2, on='id', how='inner')
SOURCES: 28-pl 30-pl
QUESTION: {{ historyquestion }}
=========
CONTENT:
SOURCE:
=========
FINAL ANSWER: {{ historyanswer }}
SOURCES:
QUESTION: {{ question }}
=========
{{ summaries }}
=========
FINAL ANSWER:

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

@ -45,6 +45,7 @@ pytz==2022.7.1
PyYAML==6.0
regex==2022.10.31
requests==2.28.2
retry==0.9.2
six==1.16.0
snowballstemmer==2.2.0
Sphinx==6.1.3
@ -64,6 +65,6 @@ typer==0.7.0
typing-inspect==0.8.0
typing_extensions==4.4.0
urllib3==1.26.14
Werkzeug==2.2.2
Werkzeug==2.2.3
XlsxWriter==3.0.8
yarl==1.8.2

@ -25,6 +25,8 @@ if (el) {
body: JSON.stringify({question: message,
api_key: localStorage.getItem('apiKey'),
embeddings_key: localStorage.getItem('apiKey'),
history: localStorage.getItem('chatHistory'),
active_docs: localStorage.getItem('activeDocs')}),
})
.then(response => response.json())
@ -38,9 +40,12 @@ if (el) {
chatWindow.scrollTop = chatWindow.scrollHeight;
document.getElementById("button-submit").innerHTML = 'Send';
document.getElementById("button-submit").disabled = false;
let chatHistory = [message, data.answer];
localStorage.setItem('chatHistory', JSON.stringify(chatHistory));
})
.catch((error) => {
console.error('Error:', error);
console.log(error);
document.getElementById("button-submit").innerHTML = 'Send';
document.getElementById("button-submit").disabled = false;
});

@ -131,13 +131,17 @@ This will return a new DataFrame with all the columns from both tables, and only
var option = document.createElement("option");
if (docsIndex[key].name == docsIndex[key].language) {
option.text = docsIndex[key].name + " " + docsIndex[key].version;
option.value = docsIndex[key].name + "/" + ".project" + "/" + docsIndex[key].version + "/";
select.add(option);
option.value = docsIndex[key].name + "/" + ".project" + "/" + docsIndex[key].version + "/{{ embeddings_choice }}/";
if (docsIndex[key].model == "{{ embeddings_choice }}") {
select.add(option);
}
}
else {
option.text = docsIndex[key].name + " " + docsIndex[key].version;
option.value = docsIndex[key].language + "/" + docsIndex[key].name + "/" + docsIndex[key].version + "/";
select.add(option);
option.value = docsIndex[key].language + "/" + docsIndex[key].name + "/" + docsIndex[key].version + "/{{ embeddings_choice }}/";
if (docsIndex[key].model == "{{ embeddings_choice }}") {
select.add(option);
}
}
}

@ -19,6 +19,12 @@ module.exports = {
plugins: ['react'],
rules: {
'react/react-in-jsx-scope': 'off',
'prettier/prettier': [
'error',
{
endOfLine: 'auto',
},
],
},
settings: {
'import/parsers': {
@ -34,10 +40,4 @@ module.exports = {
},
},
},
'prettier/prettier': [
'error',
{
endOfLine: 'auto',
},
],
}
};

File diff suppressed because it is too large Load Diff

@ -19,8 +19,10 @@
]
},
"dependencies": {
"@reduxjs/toolkit": "^1.9.2",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-redux": "^8.0.5",
"react-router-dom": "^6.8.1"
},
"devDependencies": {

@ -0,0 +1,54 @@
//TODO - Add hyperlinks to text
//TODO - Styling
export default function About() {
return (
//Parent div for all content shown through App.tsx routing needs to have this styling. Might change when state management is updated.
<div className="grid min-h-screen">
<article className=" mx-auto my-auto flex w-full max-w-6xl flex-col place-items-center gap-6 rounded-lg bg-gray-100 p-6 text-jet lg:p-10 xl:p-16">
<p className="text-3xl font-semibold">About DocsGPT 🦖</p>
<p className="mt-4 text-xl font-bold">
Find the information in your documentation through AI-powered
open-source chatbot. Powered by GPT-3, Faiss and LangChain.
</p>
<div>
<p className="text-lg">
If you want to add your own documentation, please follow the
instruction below:
</p>
<p className="mt-4 text-lg">
1. Navigate to{' '}
<span className="bg-gray-200 italic"> /application</span> folder
</p>
<p className="mt-4 text-lg">
2. Install dependencies from{' '}
<span className="bg-gray-200 italic">
pip install -r requirements.txt
</span>
</p>
<p className="mt-4 text-lg">
3. Prepare a <span className="bg-gray-200 italic">.env</span> file.
Copy <span className="bg-gray-200 italic">.env_sample</span> and
create <span className="bg-gray-200 italic">.env</span> with your
OpenAI API token
</p>
<p className="mt-4 text-lg">
4. Run the app with{' '}
<span className="bg-gray-200 italic">python app.py</span>
</p>
</div>
<p className="text-lg">
Currently It uses python pandas documentation, so it will respond to
information relevant to pandas. If you want to train it on different
documentation - please follow this guide.
</p>
<p className="mt-4 text-lg">
If you want to launch it on your own server - follow this guide.
</p>
</article>
</div>
);
}

@ -1,4 +0,0 @@
html,
body {
min-height: 100vh;
}

@ -1,55 +1,29 @@
import { useEffect, useState } from 'react';
import { Routes, Route } from 'react-router-dom';
import Navigation from './components/Navigation/Navigation';
import DocsGPT from './components/DocsGPT';
import APIKeyModal from './components/APIKeyModal';
import './App.css';
import Navigation from './Navigation';
import Conversation from './conversation/Conversation';
import About from './About';
import { useState } from 'react';
import { ActiveState } from './models/misc';
export default function App() {
//Currently using primitive state management. Will most likely be replaced with Redux.
const [isMobile, setIsMobile] = useState(true);
const [isMenuOpen, setIsMenuOpen] = useState(true);
const [isApiModalOpen, setIsApiModalOpen] = useState(true);
const [apiKey, setApiKey] = useState('');
const handleResize = () => {
if (window.innerWidth > 768 && isMobile) {
setIsMobile(false);
} else {
setIsMobile(true);
}
};
useEffect(() => {
window.addEventListener('resize', handleResize);
handleResize();
return () => {
window.removeEventListener('resize', handleResize);
};
}, []);
const [navState, setNavState] = useState<ActiveState>('ACTIVE');
return (
<div
className={`${
isMobile ? 'flex-col' : 'flex-row'
} relative flex transition-all`}
>
<APIKeyModal
apiKey={apiKey}
setApiKey={setApiKey}
isApiModalOpen={isApiModalOpen}
setIsApiModalOpen={setIsApiModalOpen}
/>
<div className="min-h-full min-w-full">
<Navigation
isMobile={isMobile}
isMenuOpen={isMenuOpen}
setIsMenuOpen={setIsMenuOpen}
setIsApiModalOpen={setIsApiModalOpen}
navState={navState}
setNavState={(val: ActiveState) => setNavState(val)}
/>
<Routes>
<Route path="/" element={<DocsGPT isMenuOpen={isMenuOpen} />} />
</Routes>
<div
className={`transition-all duration-200 ${
navState === 'ACTIVE' ? 'ml-0 md:ml-72 lg:ml-96' : ' ml-0 md:ml-16'
}`}
>
<Routes>
<Route path="/" element={<Conversation />} />
<Route path="/about" element={<About />} />
</Routes>
</div>
</div>
);
}

@ -0,0 +1,9 @@
export default function Avatar({
avatar,
size,
}: {
avatar: string;
size?: 'SMALL' | 'MEDIUM' | 'LARGE';
}) {
return <div>{avatar}</div>;
}

@ -0,0 +1,21 @@
export default function Hero({ className = '' }: { className?: string }) {
return (
<div className={`flex flex-col ${className}`}>
<p className="mb-10 text-center text-4xl font-semibold">
DocsGPT <span className="text-3xl">🦖</span>
</p>
<p className="mb-3 text-center">
Welcome to DocsGPT, your technical documentation assistant!
</p>
<p className="mb-3 text-center">
Enter a query related to the information in the documentation you
selected to receive and we will provide you with the most relevant
answers.
</p>
<p className="text-center">
Start by entering your query in the input field below and we will do the
rest!
</p>
</div>
);
}

@ -0,0 +1,97 @@
import { NavLink } from 'react-router-dom';
import Arrow1 from './assets/arrow.svg';
import Hamburger from './assets/hamburger.svg';
import Key from './assets/key.svg';
import Info from './assets/info.svg';
import Link from './assets/link.svg';
import { ActiveState } from './models/misc';
import APIKeyModal from './preferences/APIKeyModal';
import { useSelector } from 'react-redux';
import { selectApiKeyStatus } from './preferences/preferenceSlice';
import { useState } from 'react';
//TODO - Need to replace Chat button to open secondary nav with scrollable past chats option and new chat at top
//TODO - Need to add Discord and Github links
export default function Navigation({
navState,
setNavState,
}: {
navState: ActiveState;
setNavState: (val: ActiveState) => void;
}) {
const isApiKeySet = useSelector(selectApiKeyStatus);
const [apiKeyModalState, setApiKeyModalState] = useState<ActiveState>(
isApiKeySet ? 'INACTIVE' : 'ACTIVE',
);
return (
<>
<div
className={`${
navState === 'INACTIVE' && '-ml-96 md:-ml-60 lg:-ml-80'
} fixed z-10 flex h-full w-72 flex-col border-r-2 border-gray-100 bg-gray-50 transition-all duration-200 lg:w-96`}
>
<div className={'h-16 w-full border-b-2 border-gray-100'}>
<button
className="float-right mr-5 mt-5 h-5 w-5"
onClick={() =>
setNavState(navState === 'ACTIVE' ? 'INACTIVE' : 'ACTIVE')
}
>
<img
src={Arrow1}
alt="menu toggle"
className={`${
navState === 'INACTIVE' ? 'rotate-180' : 'rotate-0'
} m-auto w-3 transition-all duration-200`}
/>
</button>
</div>
<div className="flex-grow border-b-2 border-gray-100"></div>
<div className="flex h-16 flex-col border-b-2 border-gray-100">
<div
className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
onClick={() => {
setApiKeyModalState('ACTIVE');
}}
>
<img src={Key} alt="key" className="ml-2 w-6" />
<p className="my-auto text-eerie-black">Reset Key</p>
</div>
</div>
<div className="flex h-48 flex-col border-b-2 border-gray-100">
<NavLink
to="/about"
className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
>
<img src={Info} alt="info" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">About</p>
</NavLink>
<div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
<img src={Link} alt="link" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">Discord</p>
</div>
<div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
<img src={Link} alt="link" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">Github</p>
</div>
</div>
</div>
<button
className="fixed mt-5 ml-6 h-6 w-6 md:hidden"
onClick={() => setNavState('ACTIVE')}
>
<img src={Hamburger} alt="menu toggle" className="w-7" />
</button>
<APIKeyModal
modalState={apiKeyModalState}
setModalState={setApiKeyModalState}
isCancellable={isApiKeySet}
/>
</>
);
}

Before

Width:  |  Height:  |  Size: 200 B

After

Width:  |  Height:  |  Size: 200 B

@ -0,0 +1,10 @@
<svg width="800" height="800" viewBox="0 0 800 800" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_202_7)">
<path d="M750 50L50 750M750 750L50 50" stroke="black" stroke-opacity="0.54" stroke-width="100" stroke-linecap="round" stroke-linejoin="round"/>
</g>
<defs>
<clipPath id="clip0_202_7">
<rect width="800" height="800" fill="white"/>
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 391 B

@ -0,0 +1,3 @@
<svg width="600" height="450" viewBox="0 0 600 450" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M25 25H575M25 225H575M25 425H575" stroke="black" stroke-opacity="0.54" stroke-width="50" stroke-linecap="round" stroke-linejoin="round"/>
</svg>

After

Width:  |  Height:  |  Size: 254 B

Before

Width:  |  Height:  |  Size: 273 B

After

Width:  |  Height:  |  Size: 273 B

Before

Width:  |  Height:  |  Size: 337 B

After

Width:  |  Height:  |  Size: 337 B

Before

Width:  |  Height:  |  Size: 293 B

After

Width:  |  Height:  |  Size: 293 B

@ -0,0 +1,3 @@
<svg width="21" height="18" viewBox="0 0 21 18" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M0.00999999 18L21 9L0.00999999 0L0 7L15 9L0 11L0.00999999 18Z" fill="black" fill-opacity="0.54"/>
</svg>

After

Width:  |  Height:  |  Size: 210 B

@ -0,0 +1,9 @@
<svg width="30" height="33" viewBox="0 0 30 33" fill="none" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<rect width="30" height="33" fill="url(#pattern0)"/>
<defs>
<pattern id="pattern0" patternContentUnits="objectBoundingBox" width="1" height="1">
<use xlink:href="#image0_1_917" transform="scale(0.0166667 0.0151515)"/>
</pattern>
<image id="image0_1_917" width="60" height="66" xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADwAAABCCAYAAAAL1LXDAAAGuElEQVRoge2aSXPaSheG3wYJBJiSQAx2mCInhqLiSjZJKllml1+cn5BVUnE5k4sEDAZjZgkHMUiIvgtf9FkxYIwxn+H63bXULZ1H3eo+ffoQSinFf0i2/7cBq9YD8KbrAXjT9QC86WJW9SJKKTqdDlRVhc1mg9frhcvlQq/XA8MwYFl2JXbcGbAsy1AUBZVKBaqqQtd1MAwDTdNACIGmaWAYBjzPY2trC06nE6FQCD6fD5RS2Gw22GzLH4BkGZ7WaDRCu92GqqpotVqoVqtQFAW6rkPXdQAXPfz3qwgh5jWO4+B0OuH1ehGNRuH1euHz+eDxeG5rnvWdywCu1Wr49esXFEVBp9NBr9e7Aje3Qf9+BI/Hg1gshng8jkgkAkopCCG3NfV2wM1mE1+/fkWpVDJ7cpY4joPL5QKlFL1eD4PBACzLglKK4XB41ThCwPM8kskktre3wfM87Hb7rcBvDGwYBlqtFsrlMkqlEprNJgzDmAgnCAICgQAEQTAnJZZlYRiG+T+rqmqODEVRJo4OlmXh9Xqxv7+PaDQKp9O5OmBZlvH582dUKpUrver1ehEMBuHxeOD3+81/0GazgRACu91uqT8ajcyJqdlsQtd1yLKMTCZjTnSj0cisHwwG8fjxY8RiMfA8vxrgDx8+oFarWXqBYRiIoghJkrC9vQ2PxwOHw7GQQYZhoNFooFQqodFooF6vWz6s2+1GJBJBKpVCIBC48fCeG/jPnz/49OkTTk5OLF+d4zjs7u4imUyC5/mlLSWGYUBRFOTzeWQyGfT7ffMewzDw+Xx49+7djWfxuYBbrRZ+/PiBfD5vfm2Hw4FEIoF0Og1RFG+IM780TUOhUMDBwQE6nY7lniRJePXqFdxu99w9fW139Pt9ZLNZFItFE5ZlWUSjUaTTafh8vgUw5pfD4YAkSXj9+jUCgQAY5n++Uj6fRzabhaqqcz/vWuCzszPkcjlzSBFCIIoiUqkURFG8E2/obzEMg3g8jhcvXiAcDpuTH6UUP3/+xPHx8cRlbeKzZt0cDAY4PDxEt9s1r1FK8fLlSwQCgVsg3FyEEMTjcXNZq1QqAIBut4vj42OEQiGEw+FrnzOzew4ODtBut80yz/N4//49RFFciteziILBIPb398FxnHlNlmVks9m52k8FrtfrqFarFqciFoutbBhPE8MwiMVi2N3dNf/n0WiEYrGIer1uWUEmaaLlg8EAhUIBsiybPenz+RCJRBZeX5etRCIBQRBM+8ZD+7oJbCKw3W5HNBqFJEngeR4ulwuSJCEYDC7f8gUliiISiYSlAyqVChqNxsx2EycthmEQDofh9/vR7XZxdHSEp0+frmyTPo9YlsXOzg7Ozs5QLpcBAJ1OB7VaDZIkTW039WckhMDhcEAQBLx582bp+9JlSBAERKNRs6xpGhRFmdlmrWNaLMsiGAzC5XIBuFgyz8/PIcvy1DZrDQxcQF/eOXW7XdRqtan11x7Y4/FYnKDRaIRmszm1/toD22w2CIJgKV/2DK/UX4VRdylCiGX1oJRC1/WpMbW1Bx5vJMauJqUUDMNc2UqOtfbAwMXENZ6pAZixsknaCODhcGjx+QkhU/39jQC22Ww4Pz83y4ZhIBQKTa67KqPuSpRSy1aREIJerze1/toDE0LQ6XTMTQSlFOFweCr02gMDQLlcvhJJ3dhlqd1uo9/vmzEth8MBp9MJt9s9sf7aA6uqitPTU7PMcRx2dnam1l9rYMMwIMuyuSQRQsBx3MwA41oD9/t95HI5s0wpRSwWm3lUu7bAuq7j9+/flvV3fJg3K8i4tsCtVgsnJyfQNA3AxXBOJBLw+/2bBzwajVCtVq/EzB89enTt2fHKsniWpeFwiFwuh+/fv5tnXeM9cSQSubb9WgFTSiHLMj5+/GiZmARBwNu3b+d6xloBV6tVfPnyxXLN7Xbj+fPnFn96ltYCWNM0NBoNHB0dodlsmr3rdruxt7c309H4W2sBfHp6im/fvlmcDIZhEIlE8OTJk7l7F7jHwONkt0wmYzmfHsvn8yGVSt04ueXeAY/DM8Vi0UyLunzYTQjBs2fPkE6nFzoNuRfAlFIoigJZltFqtaCqqpnsdnk2drvdSKfTSCaTNxrGl3UvgIfDIQ4PD1Eul68MXeAiMikIAiRJgiRJt0pMuxfAhmHAMIyJsJcTaERRtCS1LKJ7ATz2lAqFAgghcLlc2NragiAI2NvbmxqQW0T3Anh8Hs2yLMLhsHk27fF4Fk4xnKalpA8vQ5RStFqtO01yA+4R8Kq0ltvD2+gBeNP1ALzp+s8B/wPYvPrTcSgesQAAAABJRU5ErkJggg=="/>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 2.8 KiB

@ -1,60 +0,0 @@
import { useState } from 'react';
export default function APIKeyModal({
isApiModalOpen,
setIsApiModalOpen,
apiKey,
setApiKey,
}: {
isApiModalOpen: boolean;
setIsApiModalOpen: React.Dispatch<React.SetStateAction<boolean>>;
apiKey: string;
setApiKey: React.Dispatch<React.SetStateAction<string>>;
}) {
const [formError, setFormError] = useState(false);
const handleResetKey = () => {
if (!apiKey) {
setFormError(true);
} else {
setFormError(false);
setIsApiModalOpen(false);
}
};
return (
<div
className={`${
isApiModalOpen ? 'visible' : 'hidden'
} absolute z-30 h-screen w-screen bg-gray-alpha`}
>
<div className="mx-auto mt-24 flex w-128 flex-col gap-4 rounded-lg bg-white p-6 shadow-lg">
<p className="text-xl text-jet">OpenAI API Key</p>
<p className="text-lg leading-5 text-gray-500">
Before you can start using DocsGPT we need you to provide an API key
for llm. Currently, we support only OpenAI but soon many more. You can
find it here.
</p>
<input
type="text"
className="h-10 w-full border-b-2 border-jet focus:outline-none"
value={apiKey}
maxLength={100}
placeholder="API Key"
onChange={(e) => setApiKey(e.target.value)}
/>
<div className="flex justify-between">
{formError && (
<p className="text-sm text-red-500">Please enter a valid API key</p>
)}
<button
onClick={handleResetKey}
className="ml-auto h-10 w-20 rounded-lg bg-violet-800 text-white transition-all hover:bg-violet-700"
>
Save
</button>
</div>
</div>
</div>
);
}

@ -1,7 +0,0 @@
export default function DocsGPT({ isMenuOpen }: { isMenuOpen: boolean }) {
return (
<div className={`${isMenuOpen ? 'md:ml-72 lg:ml-96' : 'ml-16'}`}>
Docs GPT Chat Placeholder
</div>
);
}

@ -1,103 +0,0 @@
import React, { useState } from 'react';
import Arrow1 from './imgs/arrow.svg';
import Key from './imgs/key.svg';
import Info from './imgs/info.svg';
import Link from './imgs/link.svg';
function MobileNavigation() {
return <div>Mobile Navigation</div>;
}
function DesktopNavigation({
isMenuOpen,
setIsMenuOpen,
setIsApiModalOpen,
}: {
isMenuOpen: boolean;
setIsMenuOpen: React.Dispatch<React.SetStateAction<boolean>>;
setIsApiModalOpen: React.Dispatch<React.SetStateAction<boolean>>;
}) {
return (
<div
className={`${
isMenuOpen ? 'w-72 lg:w-96' : 'w-16'
} fixed flex h-screen flex-col border-r-2 border-gray-100 bg-gray-50 transition-all`}
>
<div
className={`${
isMenuOpen ? 'w-full' : 'w-16'
} ml-auto h-16 border-b-2 border-gray-100`}
>
<button
className="float-right mr-5 mt-5 h-5 w-5"
onClick={() => setIsMenuOpen(!isMenuOpen)}
>
<img
src={Arrow1}
alt="menu toggle"
className={`${
isMenuOpen ? 'rotate-0' : 'rotate-180'
} m-auto w-3 transition-all`}
/>
</button>
</div>
{isMenuOpen && (
<>
<div className="flex-grow border-b-2 border-gray-100"></div>
<div className="flex h-16 flex-col border-b-2 border-gray-100">
<div
className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
onClick={() => setIsApiModalOpen(true)}
>
<img src={Key} alt="key" className="ml-2 w-6" />
<p className="my-auto text-eerie-black">Reset Key</p>
</div>
</div>
<div className="flex h-48 flex-col border-b-2 border-gray-100">
<div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
<img src={Info} alt="info" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">About</p>
</div>
<div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
<img src={Link} alt="link" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">Discord</p>
</div>
<div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
<img src={Link} alt="link" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">Github</p>
</div>
</div>
</>
)}
</div>
);
}
export default function Navigation({
isMobile,
isMenuOpen,
setIsMenuOpen,
setIsApiModalOpen,
}: {
isMobile: boolean;
isMenuOpen: boolean;
setIsMenuOpen: React.Dispatch<React.SetStateAction<boolean>>;
setIsApiModalOpen: React.Dispatch<React.SetStateAction<boolean>>;
}) {
if (isMobile) {
return <MobileNavigation />;
} else {
return (
<DesktopNavigation
isMenuOpen={isMenuOpen}
setIsMenuOpen={setIsMenuOpen}
setIsApiModalOpen={setIsApiModalOpen}
/>
);
}
}

@ -1 +0,0 @@
export default function PastChat() {}

@ -0,0 +1,73 @@
import { useEffect, useRef } from 'react';
import { useDispatch, useSelector } from 'react-redux';
import Hero from '../Hero';
import { AppDispatch } from '../store';
import ConversationBubble from './ConversationBubble';
import {
addMessage,
fetchAnswer,
selectConversation,
selectStatus,
} from './conversationSlice';
import Send from './../assets/send.svg';
import Spinner from './../assets/spinner.svg';
export default function Conversation() {
const messages = useSelector(selectConversation);
const status = useSelector(selectStatus);
const dispatch = useDispatch<AppDispatch>();
const endMessageRef = useRef<HTMLDivElement>(null);
const inputRef = useRef<HTMLDivElement>(null);
useEffect(() =>
endMessageRef?.current?.scrollIntoView({ behavior: 'smooth' }),
);
const handleQuestion = (question: string) => {
dispatch(addMessage({ text: question, type: 'QUESTION' }));
dispatch(fetchAnswer({ question }));
};
return (
<div className="flex justify-center p-6">
<div className="w-10/12 transition-all md:w-1/2">
{messages.map((message, index) => {
return (
<ConversationBubble
ref={index === messages.length - 1 ? endMessageRef : null}
className="mb-7"
key={index}
message={message.text}
type={message.type}
></ConversationBubble>
);
})}
{messages.length === 0 && <Hero className="mt-24"></Hero>}
</div>
<div className="fixed bottom-2 flex w-10/12 md:w-[50%]">
<div
ref={inputRef}
contentEditable
className={`min-h-5 border-000000 overflow-x-hidden; max-h-24 w-full overflow-y-auto rounded-xl border bg-white p-2 pr-9 opacity-100 focus:border-2 focus:outline-none`}
></div>
{status === 'loading' ? (
<img
src={Spinner}
className="relative right-9 animate-spin cursor-pointer"
></img>
) : (
<img
onClick={() => {
if (inputRef.current?.textContent) {
handleQuestion(inputRef.current.textContent);
inputRef.current.textContent = '';
}
}}
src={Send}
className="relative right-9 cursor-pointer"
></img>
)}
</div>
</div>
);
}

@ -0,0 +1,26 @@
import { forwardRef } from 'react';
import Avatar from '../Avatar';
import { MESSAGE_TYPE } from './conversationModels';
const ConversationBubble = forwardRef<
HTMLDivElement,
{
message: string;
type: MESSAGE_TYPE;
className: string;
}
>(function ConversationBubble({ message, type, className }, ref) {
return (
<div
ref={ref}
className={`flex rounded-3xl ${
type === 'QUESTION' ? '' : 'bg-gray-1000'
} py-7 px-5 ${className}`}
>
<Avatar avatar={type === 'QUESTION' ? '👤' : '🦖'}></Avatar>
<p className="ml-5">{message}</p>
</div>
);
});
export default ConversationBubble;

@ -0,0 +1,24 @@
import { Answer } from './conversationModels';
export function fetchAnswerApi(
question: string,
apiKey: string,
): Promise<Answer> {
// a mock answer generator, this is going to be replaced with real http call
return new Promise((resolve) => {
setTimeout(() => {
let result = '';
const characters =
'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
const charactersLength = characters.length;
let counter = 0;
while (counter < 5) {
result += characters.charAt(
Math.floor(Math.random() * charactersLength),
);
counter += 1;
}
resolve({ answer: result, query: question, result });
}, 3000);
});
}

@ -0,0 +1,18 @@
export type MESSAGE_TYPE = 'QUESTION' | 'ANSWER';
export type Status = 'idle' | 'loading' | 'failed';
export interface Message {
text: string;
type: MESSAGE_TYPE;
}
export interface ConversationState {
conversation: Message[];
status: Status;
}
export interface Answer {
answer: string;
query: string;
result: string;
}

@ -0,0 +1,55 @@
import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit';
import store from '../store';
import { fetchAnswerApi } from './conversationApi';
import { Answer, ConversationState, Message } from './conversationModels';
const initialState: ConversationState = {
conversation: [],
status: 'idle',
};
export const fetchAnswer = createAsyncThunk<
Answer,
{ question: string },
{ state: RootState }
>('fetchAnswer', async ({ question }, { getState }) => {
const state = getState();
const answer = await fetchAnswerApi(question, state.preference.apiKey);
return answer;
});
export const conversationSlice = createSlice({
name: 'conversation',
initialState,
reducers: {
addMessage(state, action: PayloadAction<Message>) {
state.conversation.push(action.payload);
},
},
extraReducers(builder) {
builder
.addCase(fetchAnswer.pending, (state) => {
state.status = 'loading';
})
.addCase(fetchAnswer.fulfilled, (state, action) => {
state.status = 'idle';
state.conversation.push({
text: action.payload.answer,
type: 'ANSWER',
});
})
.addCase(fetchAnswer.rejected, (state) => {
state.status = 'failed';
});
},
});
type RootState = ReturnType<typeof store.getState>;
export const selectConversation = (state: RootState) =>
state.conversation.conversation;
export const selectStatus = (state: RootState) => state.conversation.status;
export const { addMessage } = conversationSlice.actions;
export default conversationSlice.reducer;

@ -15,6 +15,8 @@
html {
line-height: 1.15; /* 1 */
-webkit-text-size-adjust: 100%; /* 2 */
min-height: 100vh;
overflow-x: hidden;
}
/* Sections
@ -26,6 +28,9 @@ html {
body {
margin: 0;
min-height: 100vh;
overflow-x: hidden;
font-family: 'Inter', sans-serif;
}
/**

@ -2,12 +2,16 @@ import React from 'react';
import ReactDOM from 'react-dom/client';
import App from './App';
import { BrowserRouter } from 'react-router-dom';
import { Provider } from 'react-redux';
import store from './store';
import './index.css';
ReactDOM.createRoot(document.getElementById('root') as HTMLElement).render(
<React.StrictMode>
<BrowserRouter>
<App />
<Provider store={store}>
<App />
</Provider>
</BrowserRouter>
</React.StrictMode>,
);

@ -0,0 +1,5 @@
export type ActiveState = 'ACTIVE' | 'INACTIVE';
export type User = {
avatar: string;
};

@ -0,0 +1,83 @@
import { useState } from 'react';
import { useDispatch } from 'react-redux';
import { ActiveState } from '../models/misc';
import { setApiKey } from './preferenceSlice';
export default function APIKeyModal({
modalState,
setModalState,
isCancellable = true,
}: {
modalState: ActiveState;
setModalState: (val: ActiveState) => void;
isCancellable?: boolean;
}) {
const dispatch = useDispatch();
const [key, setKey] = useState('');
const [isError, setIsError] = useState(false);
function handleSubmit() {
if (key.length <= 1) {
setIsError(true);
} else {
dispatch(setApiKey(key));
setModalState('INACTIVE');
setKey('');
setIsError(false);
}
}
function handleCancel() {
setKey('');
setIsError(false);
setModalState('INACTIVE');
}
return (
<div
className={`${
modalState === 'ACTIVE' ? 'visible' : 'hidden'
} absolute z-30 h-screen w-screen bg-gray-alpha`}
>
<article className="mx-auto mt-24 flex w-[90vw] max-w-lg flex-col gap-4 rounded-lg bg-white p-6 shadow-lg">
<p className="text-xl text-jet">OpenAI API Key</p>
<p className="text-lg leading-5 text-gray-500">
Before you can start using DocsGPT we need you to provide an API key
for llm. Currently, we support only OpenAI but soon many more. You can
find it here.
</p>
<input
type="text"
className="h-10 w-full border-b-2 border-jet focus:outline-none"
value={key}
maxLength={100}
placeholder="API Key"
onChange={(e) => setKey(e.target.value)}
/>
<div className="flex flex-row-reverse">
<div>
<button
onClick={() => handleSubmit()}
className="ml-auto h-10 w-20 rounded-lg bg-violet-800 text-white transition-all hover:bg-violet-700"
>
Save
</button>
{isCancellable && (
<button
onClick={() => handleCancel()}
className="ml-5 h-10 w-20 rounded-lg border border-violet-700 bg-white text-violet-800 transition-all hover:bg-violet-700 hover:text-white"
>
Cancel
</button>
)}
</div>
{isError && (
<p className="mr-auto text-sm text-red-500">
Please enter a valid API key
</p>
)}
</div>
</article>
</div>
);
}

@ -0,0 +1,29 @@
import { createSlice } from '@reduxjs/toolkit';
import store from '../store';
interface Preference {
apiKey: string;
}
const initialState: Preference = {
apiKey: '',
};
export const prefSlice = createSlice({
name: 'preference',
initialState,
reducers: {
setApiKey: (state, action) => {
state.apiKey = action.payload;
},
},
});
export const { setApiKey } = prefSlice.actions;
export default prefSlice.reducer;
type RootState = ReturnType<typeof store.getState>;
export const selectApiKey = (state: RootState) => state.preference.apiKey;
export const selectApiKeyStatus = (state: RootState) =>
!!state.preference.apiKey;

@ -0,0 +1,13 @@
import { configureStore } from '@reduxjs/toolkit';
import { conversationSlice } from './conversation/conversationSlice';
import { prefSlice } from './preferences/preferenceSlice';
const store = configureStore({
reducer: {
preference: prefSlice.reducer,
conversation: conversationSlice.reducer,
},
});
export type AppDispatch = typeof store.dispatch;
export default store;

@ -11,6 +11,7 @@ module.exports = {
'eerie-black': '#212121',
jet: '#343541',
'gray-alpha': 'rgba(0,0,0, .1)',
'gray-1000': '#F6F6F6',
},
},
},

@ -1,3 +1,5 @@
from collections import defaultdict
import os
import sys
import nltk
import dotenv
@ -18,13 +20,17 @@ app = typer.Typer(add_completion=False)
nltk.download('punkt', quiet=True)
nltk.download('averaged_perceptron_tagger', quiet=True)
#Splits all files in specified folder to documents
@app.command()
def ingest(directory: Optional[str] = typer.Option("inputs",
help="Path to the directory for index creation."),
files: Optional[List[str]] = typer.Option(None,
help="""File paths to use (Optional; overrides directory).
E.g. --files inputs/1.md --files inputs/2.md"""),
def ingest(yes: bool = typer.Option(False, "-y", "--yes", prompt=False,
help="Whether to skip price confirmation"),
dir: Optional[List[str]] = typer.Option(["inputs"],
help="""List of paths to directory for index creation.
E.g. --dir inputs --dir inputs2"""),
file: Optional[List[str]] = typer.Option(None,
help="""File paths to use (Optional; overrides dir).
E.g. --file inputs/1.md --file inputs/2.md"""),
recursive: Optional[bool] = typer.Option(True,
help="Whether to recursively search in subdirectories."),
limit: Optional[int] = typer.Option(None,
@ -38,27 +44,39 @@ def ingest(directory: Optional[str] = typer.Option("inputs",
Creates index from specified location or files.
By default /inputs folder is used, .rst and .md are parsed.
"""
raw_docs = SimpleDirectoryReader(input_dir=directory, input_files=files, recursive=recursive,
required_exts=formats, num_files_limit=limit,
exclude_hidden=exclude).load_data()
raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
print(raw_docs)
# Here we split the documents, as needed, into smaller chunks.
# We do this due to the context limits of the LLMs.
text_splitter = RecursiveCharacterTextSplitter()
docs = text_splitter.split_documents(raw_docs)
# Here we check for command line arguments for bot calls.
# If no argument exists or the permission_bypass_flag argument is not '-y',
# user permission is requested to call the API.
if len(sys.argv) > 1:
permission_bypass_flag = sys.argv[1]
if permission_bypass_flag == '-y':
call_openai_api(docs)
def process_one_docs(directory, folder_name):
raw_docs = SimpleDirectoryReader(input_dir=directory, input_files=file, recursive=recursive,
required_exts=formats, num_files_limit=limit,
exclude_hidden=exclude).load_data()
raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
# Here we split the documents, as needed, into smaller chunks.
# We do this due to the context limits of the LLMs.
text_splitter = RecursiveCharacterTextSplitter()
docs = text_splitter.split_documents(raw_docs)
# Here we check for command line arguments for bot calls.
# If no argument exists or the yes is not True, then the
# user permission is requested to call the API.
if len(sys.argv) > 1:
if yes:
call_openai_api(docs, folder_name)
else:
get_user_permission(docs, folder_name)
else:
get_user_permission(docs)
else:
get_user_permission(docs)
get_user_permission(docs, folder_name)
folder_counts = defaultdict(int)
folder_names = []
for dir_path in dir:
folder_name = os.path.basename(os.path.normpath(dir_path))
folder_counts[folder_name] += 1
if folder_counts[folder_name] > 1:
folder_name = f"{folder_name}_{folder_counts[folder_name]}"
folder_names.append(folder_name)
for directory, folder_name in zip(dir, folder_names):
process_one_docs(directory, folder_name)
if __name__ == "__main__":
app()

@ -1,9 +1,17 @@
import os
import faiss
import pickle
import tiktoken
from langchain.vectorstores import FAISS
from langchain.embeddings import OpenAIEmbeddings
#from langchain.embeddings import HuggingFaceEmbeddings
#from langchain.embeddings import HuggingFaceInstructEmbeddings
#from langchain.embeddings import CohereEmbeddings
from retry import retry
def num_tokens_from_string(string: str, encoding_name: str) -> int:
# Function to convert string to tokens and estimate user cost.
@ -12,8 +20,17 @@ def num_tokens_from_string(string: str, encoding_name: str) -> int:
total_price = ((num_tokens/1000) * 0.0004)
return num_tokens, total_price
def call_openai_api(docs):
@retry(tries=10, delay=60)
def store_add_texts_with_retry(store, i):
store.add_texts([i.page_content], metadatas=[i.metadata])
def call_openai_api(docs, folder_name):
# Function to create a vector store from the documents and save it to disk.
# create output folder if it doesn't exist
if not os.path.exists(f"outputs/{folder_name}"):
os.makedirs(f"outputs/{folder_name}")
from tqdm import tqdm
docs_test = [docs[0]]
# remove the first element from docs
@ -22,34 +39,25 @@ def call_openai_api(docs):
#docs = docs[:n]
c1 = 0
store = FAISS.from_documents(docs_test, OpenAIEmbeddings())
# Uncomment for MPNet embeddings
# model_name = "sentence-transformers/all-mpnet-base-v2"
# hf = HuggingFaceEmbeddings(model_name=model_name)
# store = FAISS.from_documents(docs_test, hf)
for i in tqdm(docs, desc="Embedding 🦖", unit="docs", total=len(docs), bar_format='{l_bar}{bar}| Time Left: {remaining}'):
try:
import time
store.add_texts([i.page_content], metadatas=[i.metadata])
store_add_texts_with_retry(store, i)
except Exception as e:
print(e)
print("Error on ", i)
print("Saving progress")
print(f"stopped at {c1} out of {len(docs)}")
faiss.write_index(store.index, "docs.index")
store_index_bak = store.index
store.index = None
with open("faiss_store.pkl", "wb") as f:
pickle.dump(store, f)
print("Sleeping for 60 seconds and trying again")
time.sleep(60)
faiss.write_index(store_index_bak, "docs.index")
store.index = store_index_bak
store.add_texts([i.page_content], metadatas=[i.metadata])
store.save_local(f"outputs/{folder_name}")
break
c1 += 1
store.save_local(f"outputs/{folder_name}")
faiss.write_index(store.index, "docs.index")
store.index = None
with open("faiss_store.pkl", "wb") as f:
pickle.dump(store, f)
def get_user_permission(docs):
def get_user_permission(docs, folder_name):
# Function to ask user permission to call the OpenAI api and spend their OpenAI funds.
# Here we convert the docs list to a string and calculate the number of OpenAI tokens the string represents.
#docs_content = (" ".join(docs))
@ -65,8 +73,8 @@ def get_user_permission(docs):
#Here we check for user permission before calling the API.
user_input = input("Price Okay? (Y/N) \n").lower()
if user_input == "y":
call_openai_api(docs)
call_openai_api(docs, folder_name)
elif user_input == "":
call_openai_api(docs)
call_openai_api(docs, folder_name)
else:
print("The API was not called. No money was spent.")

Loading…
Cancel
Save