+ 3. Prepare a .env file.
+ Copy .env_sample and
+ create .env with your
+ OpenAI API token
+
+
+ 4. Run the app with{' '}
+ python app.py
+
+
+
+
+ Currently It uses python pandas documentation, so it will respond to
+ information relevant to pandas. If you want to train it on different
+ documentation - please follow this guide.
+
+
+
+ If you want to launch it on your own server - follow this guide.
+
+
+
+ );
+}
diff --git a/frontend/src/App.css b/frontend/src/App.css
deleted file mode 100644
index a43cf76..0000000
--- a/frontend/src/App.css
+++ /dev/null
@@ -1,4 +0,0 @@
-html,
-body {
- min-height: 100vh;
-}
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 2c0a6da..b0c6cb2 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -1,55 +1,29 @@
-import { useEffect, useState } from 'react';
import { Routes, Route } from 'react-router-dom';
-import Navigation from './components/Navigation/Navigation';
-import DocsGPT from './components/DocsGPT';
-import APIKeyModal from './components/APIKeyModal';
-import './App.css';
+import Navigation from './Navigation';
+import Conversation from './conversation/Conversation';
+import About from './About';
+import { useState } from 'react';
+import { ActiveState } from './models/misc';
export default function App() {
- //Currently using primitive state management. Will most likely be replaced with Redux.
- const [isMobile, setIsMobile] = useState(true);
- const [isMenuOpen, setIsMenuOpen] = useState(true);
- const [isApiModalOpen, setIsApiModalOpen] = useState(true);
- const [apiKey, setApiKey] = useState('');
-
- const handleResize = () => {
- if (window.innerWidth > 768 && isMobile) {
- setIsMobile(false);
- } else {
- setIsMobile(true);
- }
- };
-
- useEffect(() => {
- window.addEventListener('resize', handleResize);
- handleResize();
-
- return () => {
- window.removeEventListener('resize', handleResize);
- };
- }, []);
+ const [navState, setNavState] = useState('ACTIVE');
return (
-
+ Welcome to DocsGPT, your technical documentation assistant!
+
+
+ Enter a query related to the information in the documentation you
+ selected to receive and we will provide you with the most relevant
+ answers.
+
+
+ Start by entering your query in the input field below and we will do the
+ rest!
+
+
+ );
+}
diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx
new file mode 100644
index 0000000..2b4fd23
--- /dev/null
+++ b/frontend/src/Navigation.tsx
@@ -0,0 +1,97 @@
+import { NavLink } from 'react-router-dom';
+import Arrow1 from './assets/arrow.svg';
+import Hamburger from './assets/hamburger.svg';
+import Key from './assets/key.svg';
+import Info from './assets/info.svg';
+import Link from './assets/link.svg';
+import { ActiveState } from './models/misc';
+import APIKeyModal from './preferences/APIKeyModal';
+import { useSelector } from 'react-redux';
+import { selectApiKeyStatus } from './preferences/preferenceSlice';
+import { useState } from 'react';
+
+//TODO - Need to replace Chat button to open secondary nav with scrollable past chats option and new chat at top
+//TODO - Need to add Discord and Github links
+
+export default function Navigation({
+ navState,
+ setNavState,
+}: {
+ navState: ActiveState;
+ setNavState: (val: ActiveState) => void;
+}) {
+ const isApiKeySet = useSelector(selectApiKeyStatus);
+ const [apiKeyModalState, setApiKeyModalState] = useState(
+ isApiKeySet ? 'INACTIVE' : 'ACTIVE',
+ );
+ return (
+ <>
+
+
+
+
+
+
+
+
{
+ setApiKeyModalState('ACTIVE');
+ }}
+ >
+
+
Reset Key
+
+
+
+
+
+
+
About
+
+
+
+
+
Discord
+
+
+
+
+
Github
+
+
+
+
+
+
+ );
+}
diff --git a/frontend/src/components/Navigation/imgs/arrow.svg b/frontend/src/assets/arrow.svg
similarity index 100%
rename from frontend/src/components/Navigation/imgs/arrow.svg
rename to frontend/src/assets/arrow.svg
diff --git a/frontend/src/assets/exit.svg b/frontend/src/assets/exit.svg
new file mode 100644
index 0000000..e27c3ee
--- /dev/null
+++ b/frontend/src/assets/exit.svg
@@ -0,0 +1,10 @@
+
diff --git a/frontend/src/assets/hamburger.svg b/frontend/src/assets/hamburger.svg
new file mode 100644
index 0000000..cfed15f
--- /dev/null
+++ b/frontend/src/assets/hamburger.svg
@@ -0,0 +1,3 @@
+
diff --git a/frontend/src/components/Navigation/imgs/info.svg b/frontend/src/assets/info.svg
similarity index 100%
rename from frontend/src/components/Navigation/imgs/info.svg
rename to frontend/src/assets/info.svg
diff --git a/frontend/src/components/Navigation/imgs/key.svg b/frontend/src/assets/key.svg
similarity index 100%
rename from frontend/src/components/Navigation/imgs/key.svg
rename to frontend/src/assets/key.svg
diff --git a/frontend/src/components/Navigation/imgs/link.svg b/frontend/src/assets/link.svg
similarity index 100%
rename from frontend/src/components/Navigation/imgs/link.svg
rename to frontend/src/assets/link.svg
diff --git a/frontend/src/assets/send.svg b/frontend/src/assets/send.svg
new file mode 100644
index 0000000..f31b69b
--- /dev/null
+++ b/frontend/src/assets/send.svg
@@ -0,0 +1,3 @@
+
diff --git a/frontend/src/assets/spinner.svg b/frontend/src/assets/spinner.svg
new file mode 100644
index 0000000..e18eaac
--- /dev/null
+++ b/frontend/src/assets/spinner.svg
@@ -0,0 +1,9 @@
+
diff --git a/frontend/src/components/APIKeyModal.tsx b/frontend/src/components/APIKeyModal.tsx
deleted file mode 100644
index 4a23bb7..0000000
--- a/frontend/src/components/APIKeyModal.tsx
+++ /dev/null
@@ -1,60 +0,0 @@
-import { useState } from 'react';
-
-export default function APIKeyModal({
- isApiModalOpen,
- setIsApiModalOpen,
- apiKey,
- setApiKey,
-}: {
- isApiModalOpen: boolean;
- setIsApiModalOpen: React.Dispatch>;
- apiKey: string;
- setApiKey: React.Dispatch>;
-}) {
- const [formError, setFormError] = useState(false);
-
- const handleResetKey = () => {
- if (!apiKey) {
- setFormError(true);
- } else {
- setFormError(false);
- setIsApiModalOpen(false);
- }
- };
-
- return (
-
-
-
OpenAI API Key
-
- Before you can start using DocsGPT we need you to provide an API key
- for llm. Currently, we support only OpenAI but soon many more. You can
- find it here.
-
+ Before you can start using DocsGPT we need you to provide an API key
+ for llm. Currently, we support only OpenAI but soon many more. You can
+ find it here.
+
+ setKey(e.target.value)}
+ />
+
+
+
+ {isCancellable && (
+
+ )}
+
+ {isError && (
+
+ Please enter a valid API key
+
+ )}
+
+
+
+ );
+}
diff --git a/frontend/src/preferences/preferenceSlice.ts b/frontend/src/preferences/preferenceSlice.ts
new file mode 100644
index 0000000..6121f9d
--- /dev/null
+++ b/frontend/src/preferences/preferenceSlice.ts
@@ -0,0 +1,29 @@
+import { createSlice } from '@reduxjs/toolkit';
+import store from '../store';
+
+interface Preference {
+ apiKey: string;
+}
+
+const initialState: Preference = {
+ apiKey: '',
+};
+
+export const prefSlice = createSlice({
+ name: 'preference',
+ initialState,
+ reducers: {
+ setApiKey: (state, action) => {
+ state.apiKey = action.payload;
+ },
+ },
+});
+
+export const { setApiKey } = prefSlice.actions;
+export default prefSlice.reducer;
+
+type RootState = ReturnType;
+
+export const selectApiKey = (state: RootState) => state.preference.apiKey;
+export const selectApiKeyStatus = (state: RootState) =>
+ !!state.preference.apiKey;
diff --git a/frontend/src/store.ts b/frontend/src/store.ts
new file mode 100644
index 0000000..49f5cc4
--- /dev/null
+++ b/frontend/src/store.ts
@@ -0,0 +1,13 @@
+import { configureStore } from '@reduxjs/toolkit';
+import { conversationSlice } from './conversation/conversationSlice';
+import { prefSlice } from './preferences/preferenceSlice';
+
+const store = configureStore({
+ reducer: {
+ preference: prefSlice.reducer,
+ conversation: conversationSlice.reducer,
+ },
+});
+
+export type AppDispatch = typeof store.dispatch;
+export default store;
diff --git a/frontend/tailwind.config.cjs b/frontend/tailwind.config.cjs
index 8a8d3bf..8596bc5 100644
--- a/frontend/tailwind.config.cjs
+++ b/frontend/tailwind.config.cjs
@@ -11,6 +11,7 @@ module.exports = {
'eerie-black': '#212121',
jet: '#343541',
'gray-alpha': 'rgba(0,0,0, .1)',
+ 'gray-1000': '#F6F6F6',
},
},
},
diff --git a/scripts/ingest.py b/scripts/ingest.py
index 3082cf4..2194a7c 100644
--- a/scripts/ingest.py
+++ b/scripts/ingest.py
@@ -1,3 +1,5 @@
+from collections import defaultdict
+import os
import sys
import nltk
import dotenv
@@ -18,13 +20,17 @@ app = typer.Typer(add_completion=False)
nltk.download('punkt', quiet=True)
nltk.download('averaged_perceptron_tagger', quiet=True)
+
#Splits all files in specified folder to documents
@app.command()
-def ingest(directory: Optional[str] = typer.Option("inputs",
- help="Path to the directory for index creation."),
- files: Optional[List[str]] = typer.Option(None,
- help="""File paths to use (Optional; overrides directory).
- E.g. --files inputs/1.md --files inputs/2.md"""),
+def ingest(yes: bool = typer.Option(False, "-y", "--yes", prompt=False,
+ help="Whether to skip price confirmation"),
+ dir: Optional[List[str]] = typer.Option(["inputs"],
+ help="""List of paths to directory for index creation.
+ E.g. --dir inputs --dir inputs2"""),
+ file: Optional[List[str]] = typer.Option(None,
+ help="""File paths to use (Optional; overrides dir).
+ E.g. --file inputs/1.md --file inputs/2.md"""),
recursive: Optional[bool] = typer.Option(True,
help="Whether to recursively search in subdirectories."),
limit: Optional[int] = typer.Option(None,
@@ -38,27 +44,39 @@ def ingest(directory: Optional[str] = typer.Option("inputs",
Creates index from specified location or files.
By default /inputs folder is used, .rst and .md are parsed.
"""
- raw_docs = SimpleDirectoryReader(input_dir=directory, input_files=files, recursive=recursive,
- required_exts=formats, num_files_limit=limit,
- exclude_hidden=exclude).load_data()
- raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
- print(raw_docs)
- # Here we split the documents, as needed, into smaller chunks.
- # We do this due to the context limits of the LLMs.
- text_splitter = RecursiveCharacterTextSplitter()
- docs = text_splitter.split_documents(raw_docs)
- # Here we check for command line arguments for bot calls.
- # If no argument exists or the permission_bypass_flag argument is not '-y',
- # user permission is requested to call the API.
- if len(sys.argv) > 1:
- permission_bypass_flag = sys.argv[1]
- if permission_bypass_flag == '-y':
- call_openai_api(docs)
+ def process_one_docs(directory, folder_name):
+ raw_docs = SimpleDirectoryReader(input_dir=directory, input_files=file, recursive=recursive,
+ required_exts=formats, num_files_limit=limit,
+ exclude_hidden=exclude).load_data()
+ raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+ # Here we split the documents, as needed, into smaller chunks.
+ # We do this due to the context limits of the LLMs.
+ text_splitter = RecursiveCharacterTextSplitter()
+ docs = text_splitter.split_documents(raw_docs)
+
+ # Here we check for command line arguments for bot calls.
+ # If no argument exists or the yes is not True, then the
+ # user permission is requested to call the API.
+ if len(sys.argv) > 1:
+ if yes:
+ call_openai_api(docs, folder_name)
+ else:
+ get_user_permission(docs, folder_name)
else:
- get_user_permission(docs)
- else:
- get_user_permission(docs)
+ get_user_permission(docs, folder_name)
+
+ folder_counts = defaultdict(int)
+ folder_names = []
+ for dir_path in dir:
+ folder_name = os.path.basename(os.path.normpath(dir_path))
+ folder_counts[folder_name] += 1
+ if folder_counts[folder_name] > 1:
+ folder_name = f"{folder_name}_{folder_counts[folder_name]}"
+ folder_names.append(folder_name)
+
+ for directory, folder_name in zip(dir, folder_names):
+ process_one_docs(directory, folder_name)
if __name__ == "__main__":
app()
diff --git a/scripts/parser/open_ai_func.py b/scripts/parser/open_ai_func.py
index 7009132..a89f948 100644
--- a/scripts/parser/open_ai_func.py
+++ b/scripts/parser/open_ai_func.py
@@ -1,9 +1,17 @@
+import os
import faiss
import pickle
import tiktoken
from langchain.vectorstores import FAISS
from langchain.embeddings import OpenAIEmbeddings
+#from langchain.embeddings import HuggingFaceEmbeddings
+#from langchain.embeddings import HuggingFaceInstructEmbeddings
+#from langchain.embeddings import CohereEmbeddings
+
+from retry import retry
+
+
def num_tokens_from_string(string: str, encoding_name: str) -> int:
# Function to convert string to tokens and estimate user cost.
@@ -12,8 +20,17 @@ def num_tokens_from_string(string: str, encoding_name: str) -> int:
total_price = ((num_tokens/1000) * 0.0004)
return num_tokens, total_price
-def call_openai_api(docs):
+@retry(tries=10, delay=60)
+def store_add_texts_with_retry(store, i):
+ store.add_texts([i.page_content], metadatas=[i.metadata])
+
+def call_openai_api(docs, folder_name):
# Function to create a vector store from the documents and save it to disk.
+
+ # create output folder if it doesn't exist
+ if not os.path.exists(f"outputs/{folder_name}"):
+ os.makedirs(f"outputs/{folder_name}")
+
from tqdm import tqdm
docs_test = [docs[0]]
# remove the first element from docs
@@ -22,34 +39,25 @@ def call_openai_api(docs):
#docs = docs[:n]
c1 = 0
store = FAISS.from_documents(docs_test, OpenAIEmbeddings())
+
+ # Uncomment for MPNet embeddings
+ # model_name = "sentence-transformers/all-mpnet-base-v2"
+ # hf = HuggingFaceEmbeddings(model_name=model_name)
+ # store = FAISS.from_documents(docs_test, hf)
for i in tqdm(docs, desc="Embedding 🦖", unit="docs", total=len(docs), bar_format='{l_bar}{bar}| Time Left: {remaining}'):
try:
- import time
- store.add_texts([i.page_content], metadatas=[i.metadata])
+ store_add_texts_with_retry(store, i)
except Exception as e:
print(e)
print("Error on ", i)
print("Saving progress")
print(f"stopped at {c1} out of {len(docs)}")
- faiss.write_index(store.index, "docs.index")
- store_index_bak = store.index
- store.index = None
- with open("faiss_store.pkl", "wb") as f:
- pickle.dump(store, f)
- print("Sleeping for 60 seconds and trying again")
- time.sleep(60)
- faiss.write_index(store_index_bak, "docs.index")
- store.index = store_index_bak
- store.add_texts([i.page_content], metadatas=[i.metadata])
+ store.save_local(f"outputs/{folder_name}")
+ break
c1 += 1
+ store.save_local(f"outputs/{folder_name}")
-
- faiss.write_index(store.index, "docs.index")
- store.index = None
- with open("faiss_store.pkl", "wb") as f:
- pickle.dump(store, f)
-
-def get_user_permission(docs):
+def get_user_permission(docs, folder_name):
# Function to ask user permission to call the OpenAI api and spend their OpenAI funds.
# Here we convert the docs list to a string and calculate the number of OpenAI tokens the string represents.
#docs_content = (" ".join(docs))
@@ -65,8 +73,8 @@ def get_user_permission(docs):
#Here we check for user permission before calling the API.
user_input = input("Price Okay? (Y/N) \n").lower()
if user_input == "y":
- call_openai_api(docs)
+ call_openai_api(docs, folder_name)
elif user_input == "":
- call_openai_api(docs)
+ call_openai_api(docs, folder_name)
else:
print("The API was not called. No money was spent.")