Merge branch 'arc53:main' into main

pull/103/head
Taylor Svec 2 years ago committed by GitHub
commit af198caa80
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -61,5 +61,7 @@ Copy .env_sample and create .env with your openai api token
## [How to use any other documentation](https://github.com/arc53/docsgpt/wiki/How-to-train-on-other-documentation)
## [How to host it locally (so all data will stay on-premises)](https://github.com/arc53/DocsGPT/wiki/How-to-use-different-LLM's#hosting-everything-locally)
Built with [🦜️🔗 LangChain](https://github.com/hwchase17/langchain)

@ -1,54 +1,88 @@
import os
import pickle
import json
import dotenv
import datetime
import requests
from flask import Flask, request, render_template
# os.environ["LANGCHAIN_HANDLER"] = "langchain"
import faiss
from langchain import OpenAI, VectorDBQA
from langchain import FAISS
from langchain import OpenAI, VectorDBQA, HuggingFaceHub, Cohere
from langchain.chains.question_answering import load_qa_chain
from langchain.embeddings import OpenAIEmbeddings, HuggingFaceHubEmbeddings, CohereEmbeddings, HuggingFaceInstructEmbeddings
from langchain.prompts import PromptTemplate
import requests
# os.environ["LANGCHAIN_HANDLER"] = "langchain"
if os.getenv("LLM_NAME") is not None:
llm_choice = os.getenv("LLM_NAME")
else:
llm_choice = "openai"
if os.getenv("EMBEDDINGS_NAME") is not None:
embeddings_choice = os.getenv("EMBEDDINGS_NAME")
else:
embeddings_choice = "openai_text-embedding-ada-002"
if llm_choice == "manifest":
from manifest import Manifest
from langchain.llms.manifest import ManifestWrapper
manifest = Manifest(
client_name="huggingface",
client_connection="http://127.0.0.1:5000"
)
# Redirect PosixPath to WindowsPath on Windows
import platform
if platform.system() == "Windows":
import pathlib
temp = pathlib.PosixPath
pathlib.PosixPath = pathlib.WindowsPath
# loading the .env file
dotenv.load_dotenv()
with open("combine_prompt.txt", "r") as f:
template = f.read()
# check if OPENAI_API_KEY is set
if os.getenv("OPENAI_API_KEY") is not None:
api_key_set = True
with open("combine_prompt_hist.txt", "r") as f:
template_hist = f.read()
if os.getenv("API_KEY") is not None:
api_key_set = True
else:
api_key_set = False
if os.getenv("EMBEDDINGS_KEY") is not None:
embeddings_key_set = True
else:
embeddings_key_set = False
app = Flask(__name__)
@app.route("/")
def home():
return render_template("index.html", api_key_set=api_key_set)
return render_template("index.html", api_key_set=api_key_set, llm_choice=llm_choice,
embeddings_choice=embeddings_choice)
@app.route("/api/answer", methods=["POST"])
def api_answer():
data = request.get_json()
question = data["question"]
history = data["history"]
if not api_key_set:
api_key = data["api_key"]
else:
api_key = os.getenv("OPENAI_API_KEY")
api_key = os.getenv("API_KEY")
if not embeddings_key_set:
embeddings_key = data["embeddings_key"]
else:
embeddings_key = os.getenv("EMBEDDINGS_KEY")
# check if the vectorstore is set
if "active_docs" in data:
@ -59,25 +93,37 @@ def api_answer():
vectorstore = ""
# loading the index and the store and the prompt template
index = faiss.read_index(f"{vectorstore}docs.index")
# Note if you have used other embeddings than OpenAI, you need to change the embeddings
if embeddings_choice == "openai_text-embedding-ada-002":
docsearch = FAISS.load_local(vectorstore, OpenAIEmbeddings(openai_api_key=embeddings_key))
elif embeddings_choice == "huggingface_sentence-transformers/all-mpnet-base-v2":
docsearch = FAISS.load_local(vectorstore, HuggingFaceHubEmbeddings())
elif embeddings_choice == "huggingface_hkunlp/instructor-large":
docsearch = FAISS.load_local(vectorstore, HuggingFaceInstructEmbeddings())
elif embeddings_choice == "cohere_medium":
docsearch = FAISS.load_local(vectorstore, CohereEmbeddings(cohere_api_key=embeddings_key))
with open(f"{vectorstore}faiss_store.pkl", "rb") as f:
store = pickle.load(f)
store.index = index
# create a prompt template
c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template)
# create a chain with the prompt template and the store
#chain = VectorDBQA.from_llm(llm=OpenAI(openai_api_key=api_key, temperature=0), vectorstore=store, combine_prompt=c_prompt)
# chain = VectorDBQA.from_chain_type(llm=OpenAI(openai_api_key=api_key, temperature=0), chain_type='map_reduce',
# vectorstore=store)
qa_chain = load_qa_chain(OpenAI(openai_api_key=api_key, temperature=0), chain_type="map_reduce",
if history:
history = json.loads(history)
template_temp = template_hist.replace("{historyquestion}", history[0]).replace("{historyanswer}", history[1])
c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template_temp)
else:
c_prompt = PromptTemplate(input_variables=["summaries", "question"], template=template)
if llm_choice == "openai":
llm = OpenAI(openai_api_key=api_key, temperature=0)
elif llm_choice == "manifest":
llm = ManifestWrapper(client=manifest, llm_kwargs={"temperature": 0.001, "max_tokens": 2048})
elif llm_choice == "huggingface":
llm = HuggingFaceHub(repo_id="bigscience/bloom", huggingfacehub_api_token=api_key)
elif llm_choice == "cohere":
llm = Cohere(model="command-xlarge-nightly", cohere_api_key=api_key)
qa_chain = load_qa_chain(llm=llm, chain_type="map_reduce",
combine_prompt=c_prompt)
chain = VectorDBQA(combine_documents_chain=qa_chain, vectorstore=store)
chain = VectorDBQA(combine_documents_chain=qa_chain, vectorstore=docsearch, k=4)
# fetch the answer
result = chain({"query": question})
@ -94,6 +140,7 @@ def api_answer():
# }
return result
@app.route("/api/docs_check", methods=["POST"])
def check_docs():
# check if docs exist in a vectorstore folder
@ -104,20 +151,21 @@ def check_docs():
if os.path.exists(vectorstore):
return {"status": 'exists'}
else:
r = requests.get(base_path + vectorstore + "docs.index")
r = requests.get(base_path + vectorstore + "index.faiss")
# save to vectors directory
# check if the directory exists
if not os.path.exists(vectorstore):
os.makedirs(vectorstore)
with open(vectorstore + "docs.index", "wb") as f:
with open(vectorstore + "index.faiss", "wb") as f:
f.write(r.content)
# download the store
r = requests.get(base_path + vectorstore + "faiss_store.pkl")
with open(vectorstore + "faiss_store.pkl", "wb") as f:
r = requests.get(base_path + vectorstore + "index.pkl")
with open(vectorstore + "index.pkl", "wb") as f:
f.write(r.content)
return {"status": 'loaded'}
return {"status": 'loaded'}
# handling CORS
@app.after_request

@ -0,0 +1,27 @@
You are a DocsGPT bot assistant by Arc53 that provides help with programming libraries. You give thorough answers with code examples.
Given the following extracted parts of a long document and a question, create a final answer with references ("SOURCES").
ALWAYS return a "SOURCES" part in your answer. You can also remeber things from previous questions and use them in your answer.
QUESTION: How to merge tables in pandas?
=========
Content: pandas provides various facilities for easily combining together Series or DataFrame with various kinds of set logic for the indexes and relational algebra functionality in the case of join / merge-type operations.
Source: 28-pl
Content: pandas provides a single function, merge(), as the entry point for all standard database join operations between DataFrame or named Series objects: \n\npandas.merge(left, right, how='inner', on=None, left_on=None, right_on=None, left_index=False, right_index=False, sort=False, suffixes=('_x', '_y'), copy=True, indicator=False, validate=None)
Source: 30-pl
=========
FINAL ANSWER: To merge two tables in pandas, you can use the pd.merge() function. The basic syntax is: \n\npd.merge(left, right, on, how) \n\nwhere left and right are the two tables to merge, on is the column to merge on, and how is the type of merge to perform. \n\nFor example, to merge the two tables df1 and df2 on the column 'id', you can use: \n\npd.merge(df1, df2, on='id', how='inner')
SOURCES: 28-pl 30-pl
QUESTION: {historyquestion}
=========
CONTENT:
SOURCE:
=========
FINAL ANSWER: {historyanswer}
SOURCES:
QUESTION: {question}
=========
{summaries}
=========
FINAL ANSWER:

Binary file not shown.

Binary file not shown.

@ -45,6 +45,7 @@ pytz==2022.7.1
PyYAML==6.0
regex==2022.10.31
requests==2.28.2
retry==0.9.2
six==1.16.0
snowballstemmer==2.2.0
Sphinx==6.1.3
@ -64,6 +65,6 @@ typer==0.7.0
typing-inspect==0.8.0
typing_extensions==4.4.0
urllib3==1.26.14
Werkzeug==2.2.2
Werkzeug==2.2.3
XlsxWriter==3.0.8
yarl==1.8.2

@ -25,6 +25,8 @@ if (el) {
body: JSON.stringify({question: message,
api_key: localStorage.getItem('apiKey'),
embeddings_key: localStorage.getItem('apiKey'),
history: localStorage.getItem('chatHistory'),
active_docs: localStorage.getItem('activeDocs')}),
})
.then(response => response.json())
@ -38,9 +40,12 @@ if (el) {
chatWindow.scrollTop = chatWindow.scrollHeight;
document.getElementById("button-submit").innerHTML = 'Send';
document.getElementById("button-submit").disabled = false;
let chatHistory = [message, data.answer];
localStorage.setItem('chatHistory', JSON.stringify(chatHistory));
})
.catch((error) => {
console.error('Error:', error);
console.log(error);
document.getElementById("button-submit").innerHTML = 'Send';
document.getElementById("button-submit").disabled = false;
});

@ -131,13 +131,17 @@ This will return a new DataFrame with all the columns from both tables, and only
var option = document.createElement("option");
if (docsIndex[key].name == docsIndex[key].language) {
option.text = docsIndex[key].name + " " + docsIndex[key].version;
option.value = docsIndex[key].name + "/" + ".project" + "/" + docsIndex[key].version + "/";
select.add(option);
option.value = docsIndex[key].name + "/" + ".project" + "/" + docsIndex[key].version + "/{{ embeddings_choice }}/";
if (docsIndex[key].model == "{{ embeddings_choice }}") {
select.add(option);
}
}
else {
option.text = docsIndex[key].name + " " + docsIndex[key].version;
option.value = docsIndex[key].language + "/" + docsIndex[key].name + "/" + docsIndex[key].version + "/";
select.add(option);
option.value = docsIndex[key].language + "/" + docsIndex[key].name + "/" + docsIndex[key].version + "/{{ embeddings_choice }}/";
if (docsIndex[key].model == "{{ embeddings_choice }}") {
select.add(option);
}
}
}

@ -19,6 +19,12 @@ module.exports = {
plugins: ['react'],
rules: {
'react/react-in-jsx-scope': 'off',
'prettier/prettier': [
'error',
{
endOfLine: 'auto',
},
],
},
settings: {
'import/parsers': {
@ -34,10 +40,4 @@ module.exports = {
},
},
},
'prettier/prettier': [
'error',
{
endOfLine: 'auto',
},
],
}
};

File diff suppressed because it is too large Load Diff

@ -1,71 +1,54 @@
import { useSelector } from 'react-redux';
import { useMediaQuery } from '../hooks';
import { selectIsMenuOpen } from '../store';
//TODO - Add hyperlinks to text
//TODO - Styling
export default function About() {
const isMobile = useMediaQuery('(max-width: 768px)');
const isMenuOpen = useSelector(selectIsMenuOpen);
return (
//Parent div for all content shown through App.tsx routing needs to have this styling. Might change when state management is updated.
<div
className={`${
isMobile
? isMenuOpen
? 'mt-80'
: 'mt-16'
: isMenuOpen
? 'md:ml-72 lg:ml-96'
: 'ml-16'
} h-full w-full p-6 transition-all`}
>
<article className="mx-auto my-auto flex w-full max-w-6xl flex-col gap-6 rounded-lg bg-gray-100 p-6 text-jet lg:p-10 xl:p-16">
<p className="text-3xl font-semibold">About DocsGPT 🦖</p>
<p className="mt-4 text-xl font-bold">
Find the information in your documentation through AI-powered
open-source chatbot. Powered by GPT-3, Faiss and LangChain.
</p>
<div>
<p className="text-lg">
If you want to add your own documentation, please follow the
instruction below:
</p>
<p className="mt-4 text-lg">
1. Navigate to{' '}
<span className="bg-gray-200 italic"> /application</span> folder
</p>
<p className="mt-4 text-lg">
2. Install dependencies from{' '}
<span className="bg-gray-200 italic">
pip install -r requirements.txt
</span>
</p>
<p className="mt-4 text-lg">
3. Prepare a <span className="bg-gray-200 italic">.env</span> file.
Copy <span className="bg-gray-200 italic">.env_sample</span> and
create <span className="bg-gray-200 italic">.env</span> with your
OpenAI API token
</p>
<p className="mt-4 text-lg">
4. Run the app with{' '}
<span className="bg-gray-200 italic">python app.py</span>
</p>
</div>
<p className="text-lg">
Currently It uses python pandas documentation, so it will respond to
information relevant to pandas. If you want to train it on different
documentation - please follow this guide.
</p>
<p className="mt-4 text-lg">
If you want to launch it on your own server - follow this guide.
</p>
</article>
</div>
);
}
//TODO - Add hyperlinks to text
//TODO - Styling
export default function About() {
return (
//Parent div for all content shown through App.tsx routing needs to have this styling. Might change when state management is updated.
<div className="grid min-h-screen">
<article className=" mx-auto my-auto flex w-full max-w-6xl flex-col place-items-center gap-6 rounded-lg bg-gray-100 p-6 text-jet lg:p-10 xl:p-16">
<p className="text-3xl font-semibold">About DocsGPT 🦖</p>
<p className="mt-4 text-xl font-bold">
Find the information in your documentation through AI-powered
open-source chatbot. Powered by GPT-3, Faiss and LangChain.
</p>
<div>
<p className="text-lg">
If you want to add your own documentation, please follow the
instruction below:
</p>
<p className="mt-4 text-lg">
1. Navigate to{' '}
<span className="bg-gray-200 italic"> /application</span> folder
</p>
<p className="mt-4 text-lg">
2. Install dependencies from{' '}
<span className="bg-gray-200 italic">
pip install -r requirements.txt
</span>
</p>
<p className="mt-4 text-lg">
3. Prepare a <span className="bg-gray-200 italic">.env</span> file.
Copy <span className="bg-gray-200 italic">.env_sample</span> and
create <span className="bg-gray-200 italic">.env</span> with your
OpenAI API token
</p>
<p className="mt-4 text-lg">
4. Run the app with{' '}
<span className="bg-gray-200 italic">python app.py</span>
</p>
</div>
<p className="text-lg">
Currently It uses python pandas documentation, so it will respond to
information relevant to pandas. If you want to train it on different
documentation - please follow this guide.
</p>
<p className="mt-4 text-lg">
If you want to launch it on your own server - follow this guide.
</p>
</article>
</div>
);
}

@ -1,18 +1,29 @@
import { Routes, Route } from 'react-router-dom';
import Navigation from './components/Navigation';
import Conversation from './components/Conversation/Conversation';
import APIKeyModal from './components/APIKeyModal';
import About from './components/About';
import Navigation from './Navigation';
import Conversation from './conversation/Conversation';
import About from './About';
import { useState } from 'react';
import { ActiveState } from './models/misc';
export default function App() {
const [navState, setNavState] = useState<ActiveState>('ACTIVE');
return (
<div className="relative flex flex-col transition-all md:flex-row">
<APIKeyModal />
<Navigation />
<Routes>
<Route path="/" element={<Conversation />} />
<Route path="/about" element={<About />} />
</Routes>
<div className="min-h-full min-w-full">
<Navigation
navState={navState}
setNavState={(val: ActiveState) => setNavState(val)}
/>
<div
className={`transition-all duration-200 ${
navState === 'ACTIVE' ? 'ml-0 md:ml-72 lg:ml-96' : ' ml-0 md:ml-16'
}`}
>
<Routes>
<Route path="/" element={<Conversation />} />
<Route path="/about" element={<About />} />
</Routes>
</div>
</div>
);
}

@ -0,0 +1,9 @@
export default function Avatar({
avatar,
size,
}: {
avatar: string;
size?: 'SMALL' | 'MEDIUM' | 'LARGE';
}) {
return <div>{avatar}</div>;
}

@ -0,0 +1,21 @@
export default function Hero({ className = '' }: { className?: string }) {
return (
<div className={`flex flex-col ${className}`}>
<p className="mb-10 text-center text-4xl font-semibold">
DocsGPT <span className="text-3xl">🦖</span>
</p>
<p className="mb-3 text-center">
Welcome to DocsGPT, your technical documentation assistant!
</p>
<p className="mb-3 text-center">
Enter a query related to the information in the documentation you
selected to receive and we will provide you with the most relevant
answers.
</p>
<p className="text-center">
Start by entering your query in the input field below and we will do the
rest!
</p>
</div>
);
}

@ -0,0 +1,97 @@
import { NavLink } from 'react-router-dom';
import Arrow1 from './assets/arrow.svg';
import Hamburger from './assets/hamburger.svg';
import Key from './assets/key.svg';
import Info from './assets/info.svg';
import Link from './assets/link.svg';
import { ActiveState } from './models/misc';
import APIKeyModal from './preferences/APIKeyModal';
import { useSelector } from 'react-redux';
import { selectApiKeyStatus } from './preferences/preferenceSlice';
import { useState } from 'react';
//TODO - Need to replace Chat button to open secondary nav with scrollable past chats option and new chat at top
//TODO - Need to add Discord and Github links
export default function Navigation({
navState,
setNavState,
}: {
navState: ActiveState;
setNavState: (val: ActiveState) => void;
}) {
const isApiKeySet = useSelector(selectApiKeyStatus);
const [apiKeyModalState, setApiKeyModalState] = useState<ActiveState>(
isApiKeySet ? 'INACTIVE' : 'ACTIVE',
);
return (
<>
<div
className={`${
navState === 'INACTIVE' && '-ml-96 md:-ml-60 lg:-ml-80'
} fixed z-10 flex h-full w-72 flex-col border-r-2 border-gray-100 bg-gray-50 transition-all duration-200 lg:w-96`}
>
<div className={'h-16 w-full border-b-2 border-gray-100'}>
<button
className="float-right mr-5 mt-5 h-5 w-5"
onClick={() =>
setNavState(navState === 'ACTIVE' ? 'INACTIVE' : 'ACTIVE')
}
>
<img
src={Arrow1}
alt="menu toggle"
className={`${
navState === 'INACTIVE' ? 'rotate-180' : 'rotate-0'
} m-auto w-3 transition-all duration-200`}
/>
</button>
</div>
<div className="flex-grow border-b-2 border-gray-100"></div>
<div className="flex h-16 flex-col border-b-2 border-gray-100">
<div
className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
onClick={() => {
setApiKeyModalState('ACTIVE');
}}
>
<img src={Key} alt="key" className="ml-2 w-6" />
<p className="my-auto text-eerie-black">Reset Key</p>
</div>
</div>
<div className="flex h-48 flex-col border-b-2 border-gray-100">
<NavLink
to="/about"
className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
>
<img src={Info} alt="info" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">About</p>
</NavLink>
<div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
<img src={Link} alt="link" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">Discord</p>
</div>
<div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
<img src={Link} alt="link" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">Github</p>
</div>
</div>
</div>
<button
className="fixed mt-5 ml-6 h-6 w-6 md:hidden"
onClick={() => setNavState('ACTIVE')}
>
<img src={Hamburger} alt="menu toggle" className="w-7" />
</button>
<APIKeyModal
modalState={apiKeyModalState}
setModalState={setApiKeyModalState}
isCancellable={isApiKeySet}
/>
</>
);
}

Before

Width:  |  Height:  |  Size: 200 B

After

Width:  |  Height:  |  Size: 200 B

Before

Width:  |  Height:  |  Size: 391 B

After

Width:  |  Height:  |  Size: 391 B

Before

Width:  |  Height:  |  Size: 254 B

After

Width:  |  Height:  |  Size: 254 B

Before

Width:  |  Height:  |  Size: 273 B

After

Width:  |  Height:  |  Size: 273 B

Before

Width:  |  Height:  |  Size: 337 B

After

Width:  |  Height:  |  Size: 337 B

Before

Width:  |  Height:  |  Size: 293 B

After

Width:  |  Height:  |  Size: 293 B

@ -0,0 +1,3 @@
<svg width="21" height="18" viewBox="0 0 21 18" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M0.00999999 18L21 9L0.00999999 0L0 7L15 9L0 11L0.00999999 18Z" fill="black" fill-opacity="0.54"/>
</svg>

After

Width:  |  Height:  |  Size: 210 B

@ -1,63 +0,0 @@
import { useState } from 'react';
import { useDispatch, useSelector } from 'react-redux';
import {
setApiKey,
toggleApiKeyModal,
selectIsApiKeyModalOpen,
} from '../store';
export default function APIKeyModal({}) {
//TODO - Add form validation?
//TODO - Connect to backend
//TODO - Add link to OpenAI API Key page
const dispatch = useDispatch();
const isApiModalOpen = useSelector(selectIsApiKeyModalOpen);
const [key, setKey] = useState('');
const [formError, setFormError] = useState(false);
function handleSubmit() {
if (key.length < 1) {
setFormError(true);
return;
}
dispatch(setApiKey(key));
dispatch(toggleApiKeyModal());
}
return (
<div
className={`${
isApiModalOpen ? 'visible' : 'hidden'
} absolute z-30 h-screen w-screen bg-gray-alpha`}
>
<article className="mx-auto mt-24 flex w-[90vw] max-w-lg flex-col gap-4 rounded-lg bg-white p-6 shadow-lg">
<p className="text-xl text-jet">OpenAI API Key</p>
<p className="text-lg leading-5 text-gray-500">
Before you can start using DocsGPT we need you to provide an API key
for llm. Currently, we support only OpenAI but soon many more. You can
find it here.
</p>
<input
type="text"
className="h-10 w-full border-b-2 border-jet focus:outline-none"
value={key}
maxLength={100}
placeholder="API Key"
onChange={(e) => setKey(e.target.value)}
/>
<div className="flex justify-between">
{formError && (
<p className="text-sm text-red-500">Please enter a valid API key</p>
)}
<button
onClick={() => handleSubmit()}
className="ml-auto h-10 w-20 rounded-lg bg-violet-800 text-white transition-all hover:bg-violet-700"
>
Save
</button>
</div>
</article>
</div>
);
}

@ -1,25 +0,0 @@
import { useMediaQuery } from '../../hooks';
import { selectIsMenuOpen } from '../../store';
import { useSelector } from 'react-redux';
export default function Conversation() {
const isMobile = useMediaQuery('(max-width: 768px)');
const isMenuOpen = useSelector(selectIsMenuOpen);
return (
//Parent div for all content shown through App.tsx routing needs to have this styling.
<div
className={`${
isMobile
? isMenuOpen
? 'mt-80'
: 'mt-16'
: isMenuOpen
? 'md:ml-72 lg:ml-96'
: 'ml-16'
} h-full w-full p-6 transition-all`}
>
Docs GPT Chat Placeholder
</div>
);
}

@ -1,163 +0,0 @@
import { useDispatch, useSelector } from 'react-redux';
import { NavLink } from 'react-router-dom';
import { useMediaQuery } from '../hooks';
import {
toggleApiKeyModal,
selectIsMenuOpen,
toggleIsMenuOpen,
} from '../store';
import Arrow1 from '../imgs/arrow.svg';
import Hamburger from '../imgs/hamburger.svg';
import Key from '../imgs/key.svg';
import Info from '../imgs/info.svg';
import Link from '../imgs/link.svg';
import Exit from '../imgs/exit.svg';
//TODO - Need to replace Chat button to open secondary nav with scrollable past chats option and new chat at top
//TODO - Need to add Discord and Github links
function MobileNavigation({}) {
const dispatch = useDispatch();
const isMenuOpen = useSelector(selectIsMenuOpen);
return (
<div
className={`${
isMenuOpen ? 'border-b-2 border-gray-100' : 'h-16'
} fixed flex w-full flex-col bg-gray-50 transition-all`}
>
<div className="h-16 w-full border-b-2 border-gray-100">
{isMenuOpen ? (
<>
<button
className="mt-5 ml-6 h-6 w-6"
onClick={() => dispatch(toggleIsMenuOpen())}
>
<img src={Exit} alt="menu toggle" className="w-5" />
</button>
</>
) : (
<>
<button
className="mt-5 ml-6 h-6 w-6"
onClick={() => dispatch(toggleIsMenuOpen())}
>
<img src={Hamburger} alt="menu toggle" className="w-7" />
</button>
</>
)}
</div>
{isMenuOpen && (
<nav className="my-4 flex flex-col">
<NavLink
to="/"
className="flex h-12 cursor-pointer gap-4 rounded-md px-6 hover:bg-gray-100"
>
<img src={Info} alt="info" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">Chat</p>
</NavLink>
<NavLink
to="/about"
className="flex h-12 cursor-pointer gap-4 rounded-md px-6 hover:bg-gray-100"
>
<img src={Info} alt="info" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">About</p>
</NavLink>
<div className="flex h-12 cursor-pointer gap-4 rounded-md px-6 hover:bg-gray-100">
<img src={Link} alt="info" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">Discord</p>
</div>
<div className="flex h-12 cursor-pointer gap-4 rounded-md px-6 hover:bg-gray-100">
<img src={Link} alt="info" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">Github</p>
</div>
<div
className="flex h-12 cursor-pointer gap-4 rounded-md px-6 hover:bg-gray-100"
onClick={() => dispatch(toggleApiKeyModal())}
>
<img src={Key} alt="info" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">Reset Key</p>
</div>
</nav>
)}
</div>
);
}
function DesktopNavigation() {
const dispatch = useDispatch();
const isMenuOpen = useSelector(selectIsMenuOpen);
return (
<div
className={`${
isMenuOpen ? 'w-72 lg:w-96' : 'w-16'
} fixed flex h-screen flex-col border-r-2 border-gray-100 bg-gray-50 transition-all`}
>
<div
className={`${
isMenuOpen ? 'w-full' : 'w-16'
} ml-auto h-16 border-b-2 border-gray-100`}
>
<button
className="float-right mr-5 mt-5 h-5 w-5"
onClick={() => dispatch(toggleIsMenuOpen())}
>
<img
src={Arrow1}
alt="menu toggle"
className={`${
isMenuOpen ? 'rotate-0' : 'rotate-180'
} m-auto w-3 transition-all`}
/>
</button>
</div>
{isMenuOpen && (
<>
<div className="flex-grow border-b-2 border-gray-100"></div>
<div className="flex h-16 flex-col border-b-2 border-gray-100">
<div
className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
onClick={() => dispatch(toggleApiKeyModal())}
>
<img src={Key} alt="key" className="ml-2 w-6" />
<p className="my-auto text-eerie-black">Reset Key</p>
</div>
</div>
<div className="flex h-48 flex-col border-b-2 border-gray-100">
<NavLink
to="/about"
className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100"
>
<img src={Info} alt="info" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">About</p>
</NavLink>
<div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
<img src={Link} alt="link" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">Discord</p>
</div>
<div className="my-auto mx-4 flex h-12 cursor-pointer gap-4 rounded-md hover:bg-gray-100">
<img src={Link} alt="link" className="ml-2 w-5" />
<p className="my-auto text-eerie-black">Github</p>
</div>
</div>
</>
)}
</div>
);
}
export default function Navigation() {
const isMobile = useMediaQuery('(max-width: 768px)');
if (isMobile) {
return <MobileNavigation />;
} else {
return <DesktopNavigation />;
}
}

@ -0,0 +1,33 @@
import { useEffect, useRef } from 'react';
import { useSelector } from 'react-redux';
import Hero from '../Hero';
import ConversationBubble from './ConversationBubble';
import ConversationInput from './ConversationInput';
import { selectConversation } from './conversationSlice';
export default function Conversation() {
const messages = useSelector(selectConversation);
const endMessageRef = useRef<HTMLDivElement>(null);
useEffect(() => endMessageRef?.current?.scrollIntoView());
return (
<div className="flex justify-center p-6">
<div className="w-10/12 transition-all md:w-1/2">
{messages.map((message, index) => {
return (
<ConversationBubble
ref={index === messages.length - 1 ? endMessageRef : null}
className="mb-7"
key={index}
message={message.text}
type={message.type}
></ConversationBubble>
);
})}
{messages.length === 0 && <Hero className="mt-24"></Hero>}
</div>
<ConversationInput className="fixed bottom-2 w-10/12 md:w-[50%]"></ConversationInput>
</div>
);
}

@ -0,0 +1,26 @@
import { forwardRef } from 'react';
import Avatar from '../Avatar';
import { MESSAGE_TYPE } from './conversationModels';
const ConversationBubble = forwardRef<
HTMLDivElement,
{
message: string;
type: MESSAGE_TYPE;
className: string;
}
>(function ConversationBubble({ message, type, className }, ref) {
return (
<div
ref={ref}
className={`flex rounded-3xl ${
type === 'QUESTION' ? '' : 'bg-gray-1000'
} py-7 px-5 ${className}`}
>
<Avatar avatar={type === 'QUESTION' ? '👤' : '🦖'}></Avatar>
<p className="ml-5">{message}</p>
</div>
);
});
export default ConversationBubble;

@ -0,0 +1,21 @@
import Send from './../assets/send.svg';
export default function ConversationInput({
className,
}: {
className?: string;
}) {
return (
<div className={`${className} flex`}>
<div
contentEditable
className={`min-h-5 border-000000 overflow-x-hidden; max-h-24 w-full overflow-y-auto rounded-xl border bg-white p-2 pr-9 opacity-100 focus:border-2 focus:outline-none`}
></div>
<img
onClick={() => console.log('here')}
src={Send}
className="relative right-9"
></img>
</div>
);
}

@ -0,0 +1,10 @@
export type MESSAGE_TYPE = 'QUESTION' | 'ANSWER';
export interface Message {
text: string;
type: MESSAGE_TYPE;
}
export interface ConversationState {
conversation: Message[];
}

@ -0,0 +1,50 @@
import { createSlice, PayloadAction } from '@reduxjs/toolkit';
import store from '../store';
import { ConversationState, Message } from './conversationModels';
// harcoding the initial state just for demo
const initialState: ConversationState = {
conversation: [
{ text: 'what is ChatGPT', type: 'QUESTION' },
{ text: 'ChatGPT is large learning model', type: 'ANSWER' },
{ text: 'what is ChatGPT', type: 'QUESTION' },
{ text: 'ChatGPT is large learning model', type: 'ANSWER' },
{ text: 'what is ChatGPT', type: 'QUESTION' },
{ text: 'ChatGPT is large learning model', type: 'ANSWER' },
{ text: 'what is ChatGPT', type: 'QUESTION' },
{ text: 'ChatGPT is large learning model', type: 'ANSWER' },
{ text: 'what is ChatGPT', type: 'QUESTION' },
{ text: 'ChatGPT is large learning model', type: 'ANSWER' },
{ text: 'what is ChatGPT', type: 'QUESTION' },
{ text: 'ChatGPT is large learning model', type: 'ANSWER' },
{ text: 'what is ChatGPT', type: 'QUESTION' },
{ text: 'ChatGPT is large learning model', type: 'ANSWER' },
{ text: 'what is ChatGPT', type: 'QUESTION' },
{ text: 'ChatGPT is large learning model', type: 'ANSWER' },
{ text: 'what is ChatGPT', type: 'QUESTION' },
{ text: 'ChatGPT is large learning model', type: 'ANSWER' },
{ text: 'what is ChatGPT', type: 'QUESTION' },
{
text: 'ChatGPT is large learning model',
type: 'ANSWER',
},
],
};
export const conversationSlice = createSlice({
name: 'conversation',
initialState,
reducers: {
addMessage(state, action: PayloadAction<Message>) {
state.conversation.push(action.payload);
},
},
});
export const { addMessage } = conversationSlice.actions;
type RootState = ReturnType<typeof store.getState>;
export const selectConversation = (state: RootState) =>
state.conversation.conversation;
export default conversationSlice.reducer;

@ -1,22 +0,0 @@
import { useState, useEffect } from 'react';
export function useMediaQuery(query: string): boolean {
const [matches, setMatches] = useState(false);
useEffect(() => {
const media = window.matchMedia(query);
if (media.matches !== matches) {
setMatches(media.matches);
}
const listener = () => {
setMatches(media.matches);
};
media.addEventListener('resize', listener);
return () => media.removeEventListener('resize', listener);
}, [matches, query]);
return matches;
}

@ -0,0 +1,5 @@
export type ActiveState = 'ACTIVE' | 'INACTIVE';
export type User = {
avatar: string;
};

@ -0,0 +1,83 @@
import { useState } from 'react';
import { useDispatch } from 'react-redux';
import { ActiveState } from '../models/misc';
import { setApiKey } from './preferenceSlice';
export default function APIKeyModal({
modalState,
setModalState,
isCancellable = true,
}: {
modalState: ActiveState;
setModalState: (val: ActiveState) => void;
isCancellable?: boolean;
}) {
const dispatch = useDispatch();
const [key, setKey] = useState('');
const [isError, setIsError] = useState(false);
function handleSubmit() {
if (key.length <= 1) {
setIsError(true);
} else {
dispatch(setApiKey(key));
setModalState('INACTIVE');
setKey('');
setIsError(false);
}
}
function handleCancel() {
setKey('');
setIsError(false);
setModalState('INACTIVE');
}
return (
<div
className={`${
modalState === 'ACTIVE' ? 'visible' : 'hidden'
} absolute z-30 h-screen w-screen bg-gray-alpha`}
>
<article className="mx-auto mt-24 flex w-[90vw] max-w-lg flex-col gap-4 rounded-lg bg-white p-6 shadow-lg">
<p className="text-xl text-jet">OpenAI API Key</p>
<p className="text-lg leading-5 text-gray-500">
Before you can start using DocsGPT we need you to provide an API key
for llm. Currently, we support only OpenAI but soon many more. You can
find it here.
</p>
<input
type="text"
className="h-10 w-full border-b-2 border-jet focus:outline-none"
value={key}
maxLength={100}
placeholder="API Key"
onChange={(e) => setKey(e.target.value)}
/>
<div className="flex flex-row-reverse">
<div>
<button
onClick={() => handleSubmit()}
className="ml-auto h-10 w-20 rounded-lg bg-violet-800 text-white transition-all hover:bg-violet-700"
>
Save
</button>
{isCancellable && (
<button
onClick={() => handleCancel()}
className="ml-5 h-10 w-20 rounded-lg border border-violet-700 bg-white text-violet-800 transition-all hover:bg-violet-700 hover:text-white"
>
Cancel
</button>
)}
</div>
{isError && (
<p className="mr-auto text-sm text-red-500">
Please enter a valid API key
</p>
)}
</div>
</article>
</div>
);
}

@ -0,0 +1,29 @@
import { createSlice } from '@reduxjs/toolkit';
import store from '../store';
interface Preference {
apiKey: string;
}
const initialState: Preference = {
apiKey: '',
};
export const prefSlice = createSlice({
name: 'preference',
initialState,
reducers: {
setApiKey: (state, action) => {
state.apiKey = action.payload;
},
},
});
export const { setApiKey } = prefSlice.actions;
export default prefSlice.reducer;
type RootState = ReturnType<typeof store.getState>;
export const selectApiKey = (state: RootState) => state.preference.apiKey;
export const selectApiKeyStatus = (state: RootState) =>
!!state.preference.apiKey;

@ -1,48 +1,12 @@
import { configureStore, createSlice, PayloadAction } from '@reduxjs/toolkit';
interface State {
isApiKeyModalOpen: boolean;
apiKey: string;
isMenuOpen: boolean;
}
const initialState: State = {
isApiKeyModalOpen: false,
apiKey: '',
isMenuOpen: false,
};
export const slice = createSlice({
name: 'app',
initialState,
reducers: {
toggleApiKeyModal: (state) => {
state.isApiKeyModalOpen = !state.isApiKeyModalOpen;
console.log('showApiKeyModal', state.isApiKeyModalOpen);
},
setApiKey: (state, action: PayloadAction<string>) => {
state.apiKey = action.payload;
console.log('setApiKey', action.payload);
},
toggleIsMenuOpen: (state) => {
state.isMenuOpen = !state.isMenuOpen;
},
},
});
export const { toggleApiKeyModal, setApiKey, toggleIsMenuOpen } = slice.actions;
const store = configureStore({
reducer: {
app: slice.reducer,
},
});
type RootState = ReturnType<typeof store.getState>;
export const selectIsApiKeyModalOpen = (state: RootState) =>
state.app.isApiKeyModalOpen;
export const selectApiKey = (state: RootState) => state.app.apiKey;
export const selectIsMenuOpen = (state: RootState) => state.app.isMenuOpen;
export default store;
import { configureStore } from '@reduxjs/toolkit';
import { conversationSlice } from './conversation/conversationSlice';
import { prefSlice } from './preferences/preferenceSlice';
const store = configureStore({
reducer: {
preference: prefSlice.reducer,
conversation: conversationSlice.reducer,
},
});
export default store;

@ -11,6 +11,7 @@ module.exports = {
'eerie-black': '#212121',
jet: '#343541',
'gray-alpha': 'rgba(0,0,0, .1)',
'gray-1000': '#F6F6F6',
},
},
},

@ -1,3 +1,5 @@
from collections import defaultdict
import os
import sys
import nltk
import dotenv
@ -18,13 +20,17 @@ app = typer.Typer(add_completion=False)
nltk.download('punkt', quiet=True)
nltk.download('averaged_perceptron_tagger', quiet=True)
#Splits all files in specified folder to documents
@app.command()
def ingest(directory: Optional[str] = typer.Option("inputs",
help="Path to the directory for index creation."),
files: Optional[List[str]] = typer.Option(None,
help="""File paths to use (Optional; overrides directory).
E.g. --files inputs/1.md --files inputs/2.md"""),
def ingest(yes: bool = typer.Option(False, "-y", "--yes", prompt=False,
help="Whether to skip price confirmation"),
dir: Optional[List[str]] = typer.Option(["inputs"],
help="""List of paths to directory for index creation.
E.g. --dir inputs --dir inputs2"""),
file: Optional[List[str]] = typer.Option(None,
help="""File paths to use (Optional; overrides dir).
E.g. --file inputs/1.md --file inputs/2.md"""),
recursive: Optional[bool] = typer.Option(True,
help="Whether to recursively search in subdirectories."),
limit: Optional[int] = typer.Option(None,
@ -38,27 +44,39 @@ def ingest(directory: Optional[str] = typer.Option("inputs",
Creates index from specified location or files.
By default /inputs folder is used, .rst and .md are parsed.
"""
raw_docs = SimpleDirectoryReader(input_dir=directory, input_files=files, recursive=recursive,
required_exts=formats, num_files_limit=limit,
exclude_hidden=exclude).load_data()
raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
print(raw_docs)
# Here we split the documents, as needed, into smaller chunks.
# We do this due to the context limits of the LLMs.
text_splitter = RecursiveCharacterTextSplitter()
docs = text_splitter.split_documents(raw_docs)
# Here we check for command line arguments for bot calls.
# If no argument exists or the permission_bypass_flag argument is not '-y',
# user permission is requested to call the API.
if len(sys.argv) > 1:
permission_bypass_flag = sys.argv[1]
if permission_bypass_flag == '-y':
call_openai_api(docs)
def process_one_docs(directory, folder_name):
raw_docs = SimpleDirectoryReader(input_dir=directory, input_files=file, recursive=recursive,
required_exts=formats, num_files_limit=limit,
exclude_hidden=exclude).load_data()
raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
# Here we split the documents, as needed, into smaller chunks.
# We do this due to the context limits of the LLMs.
text_splitter = RecursiveCharacterTextSplitter()
docs = text_splitter.split_documents(raw_docs)
# Here we check for command line arguments for bot calls.
# If no argument exists or the yes is not True, then the
# user permission is requested to call the API.
if len(sys.argv) > 1:
if yes:
call_openai_api(docs, folder_name)
else:
get_user_permission(docs, folder_name)
else:
get_user_permission(docs)
else:
get_user_permission(docs)
get_user_permission(docs, folder_name)
folder_counts = defaultdict(int)
folder_names = []
for dir_path in dir:
folder_name = os.path.basename(os.path.normpath(dir_path))
folder_counts[folder_name] += 1
if folder_counts[folder_name] > 1:
folder_name = f"{folder_name}_{folder_counts[folder_name]}"
folder_names.append(folder_name)
for directory, folder_name in zip(dir, folder_names):
process_one_docs(directory, folder_name)
if __name__ == "__main__":
app()

@ -1,9 +1,17 @@
import os
import faiss
import pickle
import tiktoken
from langchain.vectorstores import FAISS
from langchain.embeddings import OpenAIEmbeddings
#from langchain.embeddings import HuggingFaceEmbeddings
#from langchain.embeddings import HuggingFaceInstructEmbeddings
#from langchain.embeddings import CohereEmbeddings
from retry import retry
def num_tokens_from_string(string: str, encoding_name: str) -> int:
# Function to convert string to tokens and estimate user cost.
@ -12,8 +20,17 @@ def num_tokens_from_string(string: str, encoding_name: str) -> int:
total_price = ((num_tokens/1000) * 0.0004)
return num_tokens, total_price
def call_openai_api(docs):
@retry(tries=10, delay=60)
def store_add_texts_with_retry(store, i):
store.add_texts([i.page_content], metadatas=[i.metadata])
def call_openai_api(docs, folder_name):
# Function to create a vector store from the documents and save it to disk.
# create output folder if it doesn't exist
if not os.path.exists(f"outputs/{folder_name}"):
os.makedirs(f"outputs/{folder_name}")
from tqdm import tqdm
docs_test = [docs[0]]
# remove the first element from docs
@ -22,34 +39,25 @@ def call_openai_api(docs):
#docs = docs[:n]
c1 = 0
store = FAISS.from_documents(docs_test, OpenAIEmbeddings())
# Uncomment for MPNet embeddings
# model_name = "sentence-transformers/all-mpnet-base-v2"
# hf = HuggingFaceEmbeddings(model_name=model_name)
# store = FAISS.from_documents(docs_test, hf)
for i in tqdm(docs, desc="Embedding 🦖", unit="docs", total=len(docs), bar_format='{l_bar}{bar}| Time Left: {remaining}'):
try:
import time
store.add_texts([i.page_content], metadatas=[i.metadata])
store_add_texts_with_retry(store, i)
except Exception as e:
print(e)
print("Error on ", i)
print("Saving progress")
print(f"stopped at {c1} out of {len(docs)}")
faiss.write_index(store.index, "docs.index")
store_index_bak = store.index
store.index = None
with open("faiss_store.pkl", "wb") as f:
pickle.dump(store, f)
print("Sleeping for 60 seconds and trying again")
time.sleep(60)
faiss.write_index(store_index_bak, "docs.index")
store.index = store_index_bak
store.add_texts([i.page_content], metadatas=[i.metadata])
store.save_local(f"outputs/{folder_name}")
break
c1 += 1
store.save_local(f"outputs/{folder_name}")
faiss.write_index(store.index, "docs.index")
store.index = None
with open("faiss_store.pkl", "wb") as f:
pickle.dump(store, f)
def get_user_permission(docs):
def get_user_permission(docs, folder_name):
# Function to ask user permission to call the OpenAI api and spend their OpenAI funds.
# Here we convert the docs list to a string and calculate the number of OpenAI tokens the string represents.
#docs_content = (" ".join(docs))
@ -65,8 +73,8 @@ def get_user_permission(docs):
#Here we check for user permission before calling the API.
user_input = input("Price Okay? (Y/N) \n").lower()
if user_input == "y":
call_openai_api(docs)
call_openai_api(docs, folder_name)
elif user_input == "":
call_openai_api(docs)
call_openai_api(docs, folder_name)
else:
print("The API was not called. No money was spent.")

Loading…
Cancel
Save