From 1a1f66d2a0b704f5e66349880175165af06525e5 Mon Sep 17 00:00:00 2001 From: ajaythapliyal Date: Mon, 13 Mar 2023 09:59:29 +0530 Subject: [PATCH 01/16] adds upload modal --- frontend/src/upload/Upload.tsx | 37 ++++++++++++++++++++++++++++++++++ frontend/tailwind.config.cjs | 3 +++ 2 files changed, 40 insertions(+) create mode 100644 frontend/src/upload/Upload.tsx diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx new file mode 100644 index 00000000..a2af7c22 --- /dev/null +++ b/frontend/src/upload/Upload.tsx @@ -0,0 +1,37 @@ +import { useState } from 'react'; + +export default function Upload() { + // return null; + + const [docName, setDocName] = useState(''); + return ( +
+
+

Upload New Documentation

+ setDocName(e.target.value)} + > +
+ Name +
+
+ +
+
+ + +
+
+
+ ); +} + +// TODO: sanitize all inputs diff --git a/frontend/tailwind.config.cjs b/frontend/tailwind.config.cjs index 2c44007a..ca65c28b 100644 --- a/frontend/tailwind.config.cjs +++ b/frontend/tailwind.config.cjs @@ -16,10 +16,13 @@ module.exports = { 'gray-2000': 'rgba(0, 0, 0, 0.5)', 'gray-3000': 'rgba(243, 243, 243, 1)', 'gray-4000': '#949494', + 'gray-5000': '#BBBBBB', 'red-1000': 'rgb(254, 202, 202)', 'red-2000': '#F44336', 'red-3000': '#621B16', 'blue-1000': '#7D54D1', + 'blue-2000': '#002B49', + 'blue-3000': '#4B02E2', }, }, }, From fa31f1ee262a428cd01c9208e9ba02e9eb0789cd Mon Sep 17 00:00:00 2001 From: ajaythapliyal Date: Wed, 15 Mar 2023 08:10:44 +0530 Subject: [PATCH 02/16] modal ui done --- frontend/src/upload/Upload.tsx | 6 +++++- frontend/tailwind.config.cjs | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index a2af7c22..f35ce059 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -14,7 +14,7 @@ export default function Upload() { value={docName} onChange={(e) => setDocName(e.target.value)} > -
+
Name
@@ -23,6 +23,10 @@ export default function Upload() { Choose Files
+
+

Uploaded Files

+

None

+
- + setUploadModalState('ACTIVE')} + > {isDocsListOpen && (
{docs ? ( @@ -224,7 +232,10 @@ export default function Navigation({ setModalState={setApiKeyModalState} isCancellable={isApiKeySet} /> - {/* */} + ); } diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 5fb1a6b2..88afa960 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -1,7 +1,14 @@ import { useCallback, useState } from 'react'; import { useDropzone } from 'react-dropzone'; +import { ActiveState } from '../models/misc'; -export default function Upload() { +export default function Upload({ + modalState, + setModalState, +}: { + modalState: ActiveState; + setModalState: (state: ActiveState) => void; +}) { const [docName, setDocName] = useState(''); const [files, setfiles] = useState([]); @@ -20,7 +27,11 @@ export default function Upload() { onDragLeave: doNothing, }); return ( -
+

Upload New Documentation

Train - +
From 2523d039fb244cbd327b86e4c2bbf78dae351e48 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 17 Mar 2023 11:19:09 +0000 Subject: [PATCH 07/16] Little cloud - right size and location --- frontend/src/Navigation.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx index 562be920..b2b63281 100644 --- a/frontend/src/Navigation.tsx +++ b/frontend/src/Navigation.tsx @@ -133,7 +133,7 @@ export default function Navigation({ />
setUploadModalState('ACTIVE')} > From b1a3ff6cb10c74e4d1dbbcb98ba2dc3f271122b3 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 17 Mar 2023 11:56:21 +0000 Subject: [PATCH 08/16] api-request --- frontend/.env.development | 2 +- frontend/src/upload/Upload.tsx | 37 +++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/frontend/.env.development b/frontend/.env.development index d9292fb3..701f5bac 100644 --- a/frontend/.env.development +++ b/frontend/.env.development @@ -1,2 +1,2 @@ # Please put appropriate value -VITE_API_HOST = https://docsapi.arc53.com \ No newline at end of file +VITE_API_HOST = http://localhost:5001 \ No newline at end of file diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 88afa960..ce2dc829 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -19,6 +19,38 @@ export default function Upload({ const doNothing = () => undefined; + const uploadFile = async () => { + const formData = new FormData(); + + // Add the uploaded files to formData + files.forEach((file) => { + formData.append('file', file); + }); + + // Add the document name to formData + formData.append('name', docName); + formData.append('user', 'local'); + const apiHost = import.meta.env.VITE_API_HOST; + + try { + const response = await fetch(apiHost + '/api/upload', { + method: 'POST', + body: formData, + }); + + if (response.ok) { + console.log('Files uploaded successfully'); + setDocName(''); + setfiles([]); + setModalState('INACTIVE'); + } else { + console.error('File upload failed'); + } + } catch (error) { + console.error('Error during file upload:', error); + } + }; + const { getRootProps, getInputProps, isDragActive } = useDropzone({ onDrop, multiple: true, @@ -59,7 +91,10 @@ export default function Upload({ {files.length === 0 &&

None

}
- +
+ ); + } + + function UploadProgress() { + return ; + } + + function TrainingProgress() { + const dispatch = useDispatch(); + useEffect(() => { + const id = setInterval(() => { + const apiHost = import.meta.env.VITE_API_HOST; + fetch(`${apiHost}/api/task_status}?task_id=${progress?.taskId}`) + .then((data) => data.json()) + .then((data) => { + if (data.status == 'SUCCESS') { + clearInterval(id); + getDocs().then((data) => dispatch(setSourceDocs(data))); + } + setProgress( + (progress) => + progress && { ...progress, percentage: data.result.current }, + ); + }); + }); + return () => clearInterval(id); + }, []); + return ( + + ); + } const onDrop = useCallback((acceptedFiles: File[]) => { setfiles(acceptedFiles); @@ -19,36 +95,25 @@ export default function Upload({ const doNothing = () => undefined; - const uploadFile = async () => { + const uploadFile = () => { const formData = new FormData(); - - // Add the uploaded files to formData files.forEach((file) => { formData.append('file', file); }); - - // Add the document name to formData formData.append('name', docName); formData.append('user', 'local'); const apiHost = import.meta.env.VITE_API_HOST; - - try { - const response = await fetch(apiHost + '/api/upload', { - method: 'POST', - body: formData, - }); - - if (response.ok) { - console.log('Files uploaded successfully'); - setDocName(''); - setfiles([]); - setModalState('INACTIVE'); - } else { - console.error('File upload failed'); - } - } catch (error) { - console.error('Error during file upload:', error); - } + const xhr = new XMLHttpRequest(); + xhr.upload.addEventListener('progress', (event) => { + const progress = +((event.loaded / event.total) * 100).toFixed(2); + setProgress({ type: 'UPLOAD', percentage: progress }); + }); + xhr.onload = () => { + const { task_id } = JSON.parse(xhr.responseText); + setProgress({ type: 'TRAINIING', percentage: 0, taskId: task_id }); + }; + xhr.open('POST', `${apiHost + '/api/upload'}`); + xhr.send(formData); }; const { getRootProps, getInputProps, isDragActive } = useDropzone({ @@ -58,13 +123,15 @@ export default function Upload({ onDragOver: doNothing, onDragLeave: doNothing, }); - return ( -
-
+ + let view; + if (progress?.type === 'UPLOAD') { + view = ; + } else if (progress?.type === 'TRAINIING') { + view = ; + } else { + view = ( + <>

Upload New Documentation

+ + ); + } + + return ( +
+
+ {view}
); } - // TODO: sanitize all inputs diff --git a/frontend/tailwind.config.cjs b/frontend/tailwind.config.cjs index a824c5cd..64fb76f2 100644 --- a/frontend/tailwind.config.cjs +++ b/frontend/tailwind.config.cjs @@ -24,6 +24,8 @@ module.exports = { 'blue-1000': '#7D54D1', 'blue-2000': '#002B49', 'blue-3000': '#4B02E2', + 'blue-4000': 'rgba(0, 125, 255, 0.36)', + 'blue-5000': 'rgba(0, 125, 255)', }, }, }, From 9b75524d43c99a49dfe51c2b24fac6258420cddd Mon Sep 17 00:00:00 2001 From: ajaythapliyal Date: Sat, 18 Mar 2023 18:38:10 +0530 Subject: [PATCH 10/16] memoize the training model modal --- frontend/src/upload/Upload.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 1343929c..a7393f22 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -1,3 +1,4 @@ +import React from 'react'; import { useCallback, useEffect, useState } from 'react'; import { useDropzone } from 'react-dropzone'; import { useDispatch } from 'react-redux'; @@ -128,7 +129,8 @@ export default function Upload({ if (progress?.type === 'UPLOAD') { view = ; } else if (progress?.type === 'TRAINIING') { - view = ; + const MemoTrainingProgress = React.memo(TrainingProgress); + view = ; } else { view = ( <> From e6fe01876bb6a010fba036f9c5f1d659620b171e Mon Sep 17 00:00:00 2001 From: ajaythapliyal Date: Sun, 19 Mar 2023 09:10:53 +0530 Subject: [PATCH 11/16] some fixes --- frontend/src/upload/Upload.tsx | 42 ++++++++++++++++------------------ 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index a7393f22..fb5415e7 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -32,12 +32,12 @@ export default function Upload({

{title}...

This may take several minutes

-

{progress?.percentage}%

+

{progress?.percentage || 0}%

- +

Uploaded Files

From 796b4899aa8c88dbc6df687aaa4f052b7487dff2 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 19 Mar 2023 14:38:29 +0000 Subject: [PATCH 13/16] Fixed progress + new path + combined new Co-Authored-By: Ajay Thapliyal --- application/app.py | 1 + frontend/src/conversation/conversationApi.ts | 25 ++++++++++--------- frontend/src/preferences/preferenceApi.ts | 26 +++++++++----------- frontend/src/upload/Upload.tsx | 19 +++++++++----- 4 files changed, 40 insertions(+), 31 deletions(-) diff --git a/application/app.py b/application/app.py index 357f00d3..164db166 100644 --- a/application/app.py +++ b/application/app.py @@ -136,6 +136,7 @@ def api_answer(): vectorstore = "" else: vectorstore = "" + print(vectorstore) # vectorstore = "outputs/inputs/" # loading the index and the store and the prompt template # Note if you have used other embeddings than OpenAI, you need to change the embeddings diff --git a/frontend/src/conversation/conversationApi.ts b/frontend/src/conversation/conversationApi.ts index a0575682..c7320342 100644 --- a/frontend/src/conversation/conversationApi.ts +++ b/frontend/src/conversation/conversationApi.ts @@ -13,17 +13,20 @@ export function fetchAnswerApi( namePath = '.project'; } - const docPath = - selectedDocs.name === 'default' - ? 'default' - : selectedDocs.language + - '/' + - namePath + - '/' + - selectedDocs.version + - '/' + - selectedDocs.model + - '/'; + let docPath = 'default'; + if (selectedDocs.location === 'local') { + docPath = 'local' + '/' + selectedDocs.name + '/'; + } else if (selectedDocs.location === 'remote') { + docPath = + selectedDocs.language + + '/' + + namePath + + '/' + + selectedDocs.version + + '/' + + selectedDocs.model + + '/'; + } return fetch(apiHost + '/api/answer', { method: 'POST', diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts index d4d5979c..b9646a15 100644 --- a/frontend/src/preferences/preferenceApi.ts +++ b/frontend/src/preferences/preferenceApi.ts @@ -1,5 +1,6 @@ // not all properties in Doc are going to be present. Make some optional export type Doc = { + location: string; name: string; language: string; version: string; @@ -13,9 +14,10 @@ export type Doc = { //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later. export async function getDocs(): Promise { try { - const response = await fetch( - 'https://d3dg1063dc54p9.cloudfront.net/combined.json', - ); + const apiHost = + import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com'; + + const response = await fetch(apiHost + '/api/combine'); const data = await response.json(); const docs: Doc[] = []; @@ -52,17 +54,13 @@ export function setLocalRecentDocs(doc: Doc): void { namePath = '.project'; } - const docPath = - doc.name === 'default' - ? 'default' - : doc.language + - '/' + - namePath + - '/' + - doc.version + - '/' + - doc.model + - '/'; + let docPath = 'default'; + if (doc.location === 'local') { + docPath = 'local' + '/' + doc.name + '/'; + } else if (doc.location === 'remote') { + docPath = + doc.language + '/' + namePath + '/' + doc.version + '/' + doc.model + '/'; + } const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com'; fetch(apiHost + '/api/docs_check', { method: 'POST', diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 53f4772d..e404bd12 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -67,19 +67,26 @@ export default function Upload({ (progress?.percentage ?? 0) < 100 && setTimeout(() => { const apiHost = import.meta.env.VITE_API_HOST; - fetch(`${apiHost}/api/task_status}?task_id=${progress?.taskId}`) + fetch(`${apiHost}/api/task_status?task_id=${progress?.taskId}`) .then((data) => data.json()) .then((data) => { if (data.status == 'SUCCESS') { getDocs().then((data) => dispatch(setSourceDocs(data))); + setProgress( + (progress) => progress && { ...progress, percentage: 100 }, + ); + } else { + setProgress( + (progress) => + progress && { + ...progress, + percentage: data.result.current, + }, + ); } - setProgress( - (progress) => - progress && { ...progress, percentage: data.result.current }, - ); }); }, 5000); - }, []); + }, [progress, dispatch]); return ( Date: Sun, 19 Mar 2023 14:44:17 +0000 Subject: [PATCH 14/16] fixes + combined new + path --- frontend/src/store.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/store.ts b/frontend/src/store.ts index 6bea07b2..dd0a2c9c 100644 --- a/frontend/src/store.ts +++ b/frontend/src/store.ts @@ -15,6 +15,7 @@ const store = configureStore({ selectedDocs: doc !== null ? JSON.parse(doc) : null, sourceDocs: [ { + location: '', language: '', name: 'default', version: '', From e7bd9b63236a4bbdbc9ab9c297c96e4bbad386de Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 20 Mar 2023 14:19:12 +0000 Subject: [PATCH 15/16] Update Navigation.tsx --- frontend/src/Navigation.tsx | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx index b2b63281..053b9482 100644 --- a/frontend/src/Navigation.tsx +++ b/frontend/src/Navigation.tsx @@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from 'react'; import { NavLink } from 'react-router-dom'; import Arrow1 from './assets/arrow.svg'; import Arrow2 from './assets/dropdown-arrow.svg'; +import Exit from './assets/exit.svg'; import Message from './assets/message.svg'; import Hamburger from './assets/hamburger.svg'; import Key from './assets/key.svg'; @@ -21,6 +22,7 @@ import { } from './preferences/preferenceSlice'; import { useOutsideAlerter } from './hooks'; import Upload from './upload/Upload'; +import { Doc } from './preferences/preferenceApi'; export default function Navigation({ navState, @@ -48,6 +50,23 @@ export default function Navigation({ useState('INACTIVE'); const navRef = useRef(null); + const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com'; + + const handleDeleteClick = (index: number, doc: Doc) => { + const docPath = 'indexes/' + 'local' + '/' + doc.name; + + fetch(`${apiHost}/api/delete_old?path=${docPath}`, { + method: 'GET', + }) + .then(() => { + // remove the image element from the DOM + const imageElement = document.querySelector( + `#img-${index}`, + ) as HTMLElement; + imageElement.parentNode?.removeChild(imageElement); + }) + .catch((error) => console.error(error)); + }; useOutsideAlerter( navRef, () => { @@ -149,11 +168,18 @@ export default function Navigation({ dispatch(setSelectedDocs(doc)); setIsDocsListOpen(false); }} - className="h-10 w-full cursor-pointer border-x-2 border-b-2 hover:bg-gray-100" + className="flex h-10 w-full cursor-pointer items-center justify-between border-x-2 border-b-2 hover:bg-gray-100" > -

+

{doc.name} {doc.version}

+ Exit handleDeleteClick(index, doc)} + />
); } From 3e98f9e6bd16762f31813aeca021ce5e85de1a7d Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 20 Mar 2023 14:34:51 +0000 Subject: [PATCH 16/16] Button working now --- application/app.py | 5 ++++- frontend/src/Navigation.tsx | 23 +++++++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/application/app.py b/application/app.py index 164db166..dfce4970 100644 --- a/application/app.py +++ b/application/app.py @@ -410,8 +410,11 @@ def delete_old(): if dirs[0] not in ['indexes', 'vectors']: return {"status": 'error'} path_clean = '/'.join(dirs) - shutil.rmtree(path) vectors_collection.delete_one({'location': path}) + try: + shutil.rmtree(path_clean) + except FileNotFoundError: + pass return {"status": 'ok'} # handling CORS diff --git a/frontend/src/Navigation.tsx b/frontend/src/Navigation.tsx index 053b9482..f4b9eb96 100644 --- a/frontend/src/Navigation.tsx +++ b/frontend/src/Navigation.tsx @@ -63,7 +63,8 @@ export default function Navigation({ const imageElement = document.querySelector( `#img-${index}`, ) as HTMLElement; - imageElement.parentNode?.removeChild(imageElement); + const parentElement = imageElement.parentNode as HTMLElement; + parentElement.parentNode?.removeChild(parentElement); }) .catch((error) => console.error(error)); }; @@ -173,13 +174,18 @@ export default function Navigation({

{doc.name} {doc.version}

- Exit handleDeleteClick(index, doc)} - /> + {doc.location === 'local' ? ( + Exit { + event.stopPropagation(); + handleDeleteClick(index, doc); + }} + /> + ) : null} ); } @@ -189,6 +195,7 @@ export default function Navigation({

No default documentation.

)} + ) )}