From d98c876f82c09b374a659f7671db0f36242b2404 Mon Sep 17 00:00:00 2001 From: ajaythapliyal Date: Sat, 18 Mar 2023 18:25:23 +0530 Subject: [PATCH 1/6] adds training modals --- frontend/src/upload/Upload.tsx | 142 +++++++++++++++++++++++++-------- frontend/tailwind.config.cjs | 2 + 2 files changed, 112 insertions(+), 32 deletions(-) diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index ce2dc82..1343929 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -1,6 +1,9 @@ -import { useCallback, useState } from 'react'; +import { useCallback, useEffect, useState } from 'react'; import { useDropzone } from 'react-dropzone'; +import { useDispatch } from 'react-redux'; import { ActiveState } from '../models/misc'; +import { getDocs } from '../preferences/preferenceApi'; +import { setSourceDocs } from '../preferences/preferenceSlice'; export default function Upload({ modalState, @@ -11,6 +14,79 @@ export default function Upload({ }) { const [docName, setDocName] = useState(''); const [files, setfiles] = useState([]); + const [progress, setProgress] = useState<{ + type: 'UPLOAD' | 'TRAINIING'; + percentage: number; + taskId?: string; + }>(); + + function Progress({ + title, + isCancellable = false, + }: { + title: string; + isCancellable?: boolean; + }) { + return ( +
+

{title}...

+

This may take several minutes

+

{progress?.percentage}%

+
+
+
+
+ +
+ ); + } + + function UploadProgress() { + return ; + } + + function TrainingProgress() { + const dispatch = useDispatch(); + useEffect(() => { + const id = setInterval(() => { + const apiHost = import.meta.env.VITE_API_HOST; + fetch(`${apiHost}/api/task_status}?task_id=${progress?.taskId}`) + .then((data) => data.json()) + .then((data) => { + if (data.status == 'SUCCESS') { + clearInterval(id); + getDocs().then((data) => dispatch(setSourceDocs(data))); + } + setProgress( + (progress) => + progress && { ...progress, percentage: data.result.current }, + ); + }); + }); + return () => clearInterval(id); + }, []); + return ( + + ); + } const onDrop = useCallback((acceptedFiles: File[]) => { setfiles(acceptedFiles); @@ -19,36 +95,25 @@ export default function Upload({ const doNothing = () => undefined; - const uploadFile = async () => { + const uploadFile = () => { const formData = new FormData(); - - // Add the uploaded files to formData files.forEach((file) => { formData.append('file', file); }); - - // Add the document name to formData formData.append('name', docName); formData.append('user', 'local'); const apiHost = import.meta.env.VITE_API_HOST; - - try { - const response = await fetch(apiHost + '/api/upload', { - method: 'POST', - body: formData, - }); - - if (response.ok) { - console.log('Files uploaded successfully'); - setDocName(''); - setfiles([]); - setModalState('INACTIVE'); - } else { - console.error('File upload failed'); - } - } catch (error) { - console.error('Error during file upload:', error); - } + const xhr = new XMLHttpRequest(); + xhr.upload.addEventListener('progress', (event) => { + const progress = +((event.loaded / event.total) * 100).toFixed(2); + setProgress({ type: 'UPLOAD', percentage: progress }); + }); + xhr.onload = () => { + const { task_id } = JSON.parse(xhr.responseText); + setProgress({ type: 'TRAINIING', percentage: 0, taskId: task_id }); + }; + xhr.open('POST', `${apiHost + '/api/upload'}`); + xhr.send(formData); }; const { getRootProps, getInputProps, isDragActive } = useDropzone({ @@ -58,13 +123,15 @@ export default function Upload({ onDragOver: doNothing, onDragLeave: doNothing, }); - return ( -
-
+ + let view; + if (progress?.type === 'UPLOAD') { + view = ; + } else if (progress?.type === 'TRAINIING') { + view = ; + } else { + view = ( + <>

Upload New Documentation

+ + ); + } + + return ( +
+
+ {view}
); } - // TODO: sanitize all inputs diff --git a/frontend/tailwind.config.cjs b/frontend/tailwind.config.cjs index a824c5c..64fb76f 100644 --- a/frontend/tailwind.config.cjs +++ b/frontend/tailwind.config.cjs @@ -24,6 +24,8 @@ module.exports = { 'blue-1000': '#7D54D1', 'blue-2000': '#002B49', 'blue-3000': '#4B02E2', + 'blue-4000': 'rgba(0, 125, 255, 0.36)', + 'blue-5000': 'rgba(0, 125, 255)', }, }, }, From 9b75524d43c99a49dfe51c2b24fac6258420cddd Mon Sep 17 00:00:00 2001 From: ajaythapliyal Date: Sat, 18 Mar 2023 18:38:10 +0530 Subject: [PATCH 2/6] memoize the training model modal --- frontend/src/upload/Upload.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 1343929..a7393f2 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -1,3 +1,4 @@ +import React from 'react'; import { useCallback, useEffect, useState } from 'react'; import { useDropzone } from 'react-dropzone'; import { useDispatch } from 'react-redux'; @@ -128,7 +129,8 @@ export default function Upload({ if (progress?.type === 'UPLOAD') { view = ; } else if (progress?.type === 'TRAINIING') { - view = ; + const MemoTrainingProgress = React.memo(TrainingProgress); + view = ; } else { view = ( <> From e6fe01876bb6a010fba036f9c5f1d659620b171e Mon Sep 17 00:00:00 2001 From: ajaythapliyal Date: Sun, 19 Mar 2023 09:10:53 +0530 Subject: [PATCH 3/6] some fixes --- frontend/src/upload/Upload.tsx | 42 ++++++++++++++++------------------ 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index a7393f2..fb5415e 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -32,12 +32,12 @@ export default function Upload({

{title}...

This may take several minutes

-

{progress?.percentage}%

+

{progress?.percentage || 0}%

- +

Uploaded Files

From 796b4899aa8c88dbc6df687aaa4f052b7487dff2 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 19 Mar 2023 14:38:29 +0000 Subject: [PATCH 5/6] Fixed progress + new path + combined new Co-Authored-By: Ajay Thapliyal --- application/app.py | 1 + frontend/src/conversation/conversationApi.ts | 25 ++++++++++--------- frontend/src/preferences/preferenceApi.ts | 26 +++++++++----------- frontend/src/upload/Upload.tsx | 19 +++++++++----- 4 files changed, 40 insertions(+), 31 deletions(-) diff --git a/application/app.py b/application/app.py index 357f00d..164db16 100644 --- a/application/app.py +++ b/application/app.py @@ -136,6 +136,7 @@ def api_answer(): vectorstore = "" else: vectorstore = "" + print(vectorstore) # vectorstore = "outputs/inputs/" # loading the index and the store and the prompt template # Note if you have used other embeddings than OpenAI, you need to change the embeddings diff --git a/frontend/src/conversation/conversationApi.ts b/frontend/src/conversation/conversationApi.ts index a057568..c732034 100644 --- a/frontend/src/conversation/conversationApi.ts +++ b/frontend/src/conversation/conversationApi.ts @@ -13,17 +13,20 @@ export function fetchAnswerApi( namePath = '.project'; } - const docPath = - selectedDocs.name === 'default' - ? 'default' - : selectedDocs.language + - '/' + - namePath + - '/' + - selectedDocs.version + - '/' + - selectedDocs.model + - '/'; + let docPath = 'default'; + if (selectedDocs.location === 'local') { + docPath = 'local' + '/' + selectedDocs.name + '/'; + } else if (selectedDocs.location === 'remote') { + docPath = + selectedDocs.language + + '/' + + namePath + + '/' + + selectedDocs.version + + '/' + + selectedDocs.model + + '/'; + } return fetch(apiHost + '/api/answer', { method: 'POST', diff --git a/frontend/src/preferences/preferenceApi.ts b/frontend/src/preferences/preferenceApi.ts index d4d5979..b9646a1 100644 --- a/frontend/src/preferences/preferenceApi.ts +++ b/frontend/src/preferences/preferenceApi.ts @@ -1,5 +1,6 @@ // not all properties in Doc are going to be present. Make some optional export type Doc = { + location: string; name: string; language: string; version: string; @@ -13,9 +14,10 @@ export type Doc = { //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later. export async function getDocs(): Promise { try { - const response = await fetch( - 'https://d3dg1063dc54p9.cloudfront.net/combined.json', - ); + const apiHost = + import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com'; + + const response = await fetch(apiHost + '/api/combine'); const data = await response.json(); const docs: Doc[] = []; @@ -52,17 +54,13 @@ export function setLocalRecentDocs(doc: Doc): void { namePath = '.project'; } - const docPath = - doc.name === 'default' - ? 'default' - : doc.language + - '/' + - namePath + - '/' + - doc.version + - '/' + - doc.model + - '/'; + let docPath = 'default'; + if (doc.location === 'local') { + docPath = 'local' + '/' + doc.name + '/'; + } else if (doc.location === 'remote') { + docPath = + doc.language + '/' + namePath + '/' + doc.version + '/' + doc.model + '/'; + } const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com'; fetch(apiHost + '/api/docs_check', { method: 'POST', diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 53f4772..e404bd1 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -67,19 +67,26 @@ export default function Upload({ (progress?.percentage ?? 0) < 100 && setTimeout(() => { const apiHost = import.meta.env.VITE_API_HOST; - fetch(`${apiHost}/api/task_status}?task_id=${progress?.taskId}`) + fetch(`${apiHost}/api/task_status?task_id=${progress?.taskId}`) .then((data) => data.json()) .then((data) => { if (data.status == 'SUCCESS') { getDocs().then((data) => dispatch(setSourceDocs(data))); + setProgress( + (progress) => progress && { ...progress, percentage: 100 }, + ); + } else { + setProgress( + (progress) => + progress && { + ...progress, + percentage: data.result.current, + }, + ); } - setProgress( - (progress) => - progress && { ...progress, percentage: data.result.current }, - ); }); }, 5000); - }, []); + }, [progress, dispatch]); return ( Date: Sun, 19 Mar 2023 14:44:17 +0000 Subject: [PATCH 6/6] fixes + combined new + path --- frontend/src/store.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/store.ts b/frontend/src/store.ts index 6bea07b..dd0a2c9 100644 --- a/frontend/src/store.ts +++ b/frontend/src/store.ts @@ -15,6 +15,7 @@ const store = configureStore({ selectedDocs: doc !== null ? JSON.parse(doc) : null, sourceDocs: [ { + location: '', language: '', name: 'default', version: '',