Merge pull request #185 from arc53/feature/upload

pull/192/head
Alex 2 years ago committed by GitHub
commit ce579293fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -136,6 +136,7 @@ def api_answer():
vectorstore = ""
else:
vectorstore = ""
print(vectorstore)
# vectorstore = "outputs/inputs/"
# loading the index and the store and the prompt template
# Note if you have used other embeddings than OpenAI, you need to change the embeddings
@ -409,8 +410,11 @@ def delete_old():
if dirs[0] not in ['indexes', 'vectors']:
return {"status": 'error'}
path_clean = '/'.join(dirs)
shutil.rmtree(path)
vectors_collection.delete_one({'location': path})
try:
shutil.rmtree(path_clean)
except FileNotFoundError:
pass
return {"status": 'ok'}
# handling CORS

@ -1,2 +1,2 @@
# Please put appropriate value
VITE_API_HOST = https://docsapi.arc53.com
VITE_API_HOST = http://localhost:5001

@ -1105,6 +1105,11 @@
"integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
"dev": true
},
"attr-accept": {
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.2.tgz",
"integrity": "sha512-7prDjvt9HmqiZ0cl5CRjtS84sEyhsHP2coDkaZKRKVfCDo9s7iw7ChVmar78Gu9pC4SoR/28wFu/G5JJhTnqEg=="
},
"autoprefixer": {
"version": "10.4.13",
"resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz",
@ -2168,6 +2173,21 @@
"flat-cache": "^3.0.4"
}
},
"file-selector": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/file-selector/-/file-selector-0.6.0.tgz",
"integrity": "sha512-QlZ5yJC0VxHxQQsQhXvBaC7VRJ2uaxTf+Tfpu4Z/OcVQJVpZO+DGU0rkoVW5ce2SccxugvpBJoMvUs59iILYdw==",
"requires": {
"tslib": "^2.4.0"
},
"dependencies": {
"tslib": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
"integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
}
}
},
"fill-range": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@ -3088,8 +3108,7 @@
"object-assign": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==",
"dev": true
"integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="
},
"object-hash": {
"version": "3.0.0",
@ -3395,7 +3414,6 @@
"version": "15.8.1",
"resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
"integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
"dev": true,
"requires": {
"loose-envify": "^1.4.0",
"object-assign": "^4.1.1",
@ -3437,6 +3455,16 @@
"scheduler": "^0.23.0"
}
},
"react-dropzone": {
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-14.2.3.tgz",
"integrity": "sha512-O3om8I+PkFKbxCukfIR3QAGftYXDZfOE2N1mr/7qebQJHs7U+/RSL/9xomJNpRg9kM5h9soQSdf0Gc7OHF5Fug==",
"requires": {
"attr-accept": "^2.2.2",
"file-selector": "^0.6.0",
"prop-types": "^15.8.1"
}
},
"react-is": {
"version": "16.13.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",

@ -23,6 +23,7 @@
"@vercel/analytics": "^0.1.10",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-dropzone": "^14.2.3",
"react-redux": "^8.0.5",
"react-router-dom": "^6.8.1"
},

@ -2,11 +2,13 @@ import { useEffect, useRef, useState } from 'react';
import { NavLink } from 'react-router-dom';
import Arrow1 from './assets/arrow.svg';
import Arrow2 from './assets/dropdown-arrow.svg';
import Exit from './assets/exit.svg';
import Message from './assets/message.svg';
import Hamburger from './assets/hamburger.svg';
import Key from './assets/key.svg';
import Info from './assets/info.svg';
import Link from './assets/link.svg';
import UploadIcon from './assets/upload.svg';
import { ActiveState } from './models/misc';
import APIKeyModal from './preferences/APIKeyModal';
import SelectDocsModal from './preferences/SelectDocsModal';
@ -19,6 +21,8 @@ import {
setSelectedDocs,
} from './preferences/preferenceSlice';
import { useOutsideAlerter } from './hooks';
import Upload from './upload/Upload';
import { Doc } from './preferences/preferenceApi';
export default function Navigation({
navState,
@ -42,7 +46,28 @@ export default function Navigation({
const [selectedDocsModalState, setSelectedDocsModalState] =
useState<ActiveState>(isSelectedDocsSet ? 'INACTIVE' : 'ACTIVE');
const [uploadModalState, setUploadModalState] =
useState<ActiveState>('INACTIVE');
const navRef = useRef(null);
const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
const handleDeleteClick = (index: number, doc: Doc) => {
const docPath = 'indexes/' + 'local' + '/' + doc.name;
fetch(`${apiHost}/api/delete_old?path=${docPath}`, {
method: 'GET',
})
.then(() => {
// remove the image element from the DOM
const imageElement = document.querySelector(
`#img-${index}`,
) as HTMLElement;
const parentElement = imageElement.parentNode as HTMLElement;
parentElement.parentNode?.removeChild(parentElement);
})
.catch((error) => console.error(error));
};
useOutsideAlerter(
navRef,
() => {
@ -109,7 +134,7 @@ export default function Navigation({
<div className="flex-grow border-b-2 border-gray-100"></div>
<div className="flex flex-col-reverse border-b-2">
<div className="relative my-4 px-6">
<div className="relative my-4 flex gap-2 px-2">
<div
className="flex h-12 w-full cursor-pointer justify-between rounded-md border-2 bg-white"
onClick={() => setIsDocsListOpen(!isDocsListOpen)}
@ -127,8 +152,13 @@ export default function Navigation({
} mr-3 w-3 transition-all`}
/>
</div>
<img
className="mt-2 h-9 w-9 hover:cursor-pointer"
src={UploadIcon}
onClick={() => setUploadModalState('ACTIVE')}
></img>
{isDocsListOpen && (
<div className="absolute top-12 left-0 right-0 mx-6 max-h-52 overflow-y-scroll bg-white shadow-lg">
<div className="absolute top-12 left-0 right-6 ml-2 mr-4 max-h-52 overflow-y-scroll bg-white shadow-lg">
{docs ? (
docs.map((doc, index) => {
if (doc.model) {
@ -139,11 +169,23 @@ export default function Navigation({
dispatch(setSelectedDocs(doc));
setIsDocsListOpen(false);
}}
className="h-10 w-full cursor-pointer border-x-2 border-b-2 hover:bg-gray-100"
className="flex h-10 w-full cursor-pointer items-center justify-between border-x-2 border-b-2 hover:bg-gray-100"
>
<p className="ml-5 py-3">
<p className="ml-5 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3">
{doc.name} {doc.version}
</p>
{doc.location === 'local' ? (
<img
src={Exit}
alt="Exit"
className="mr-4 h-3 w-3 cursor-pointer hover:opacity-50"
id={`img-${index}`}
onClick={(event) => {
event.stopPropagation();
handleDeleteClick(index, doc);
}}
/>
) : null}
</div>
);
}
@ -153,6 +195,7 @@ export default function Navigation({
<p className="ml-5 py-3">No default documentation.</p>
</div>
)}
)
</div>
)}
</div>
@ -222,6 +265,10 @@ export default function Navigation({
setModalState={setApiKeyModalState}
isCancellable={isApiKeySet}
/>
<Upload
modalState={uploadModalState}
setModalState={setUploadModalState}
></Upload>
</>
);
}

@ -0,0 +1,3 @@
<svg width="24" height="16" viewBox="0 0 24 16" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M19.35 6.04C18.67 2.59 15.64 0 12 0C9.11 0 6.6 1.64 5.35 4.04C2.34 4.36 0 6.91 0 10C0 13.31 2.69 16 6 16H19C21.76 16 24 13.76 24 11C24 8.36 21.95 6.22 19.35 6.04ZM14 9V13H10V9H7L12 4L17 9H14Z" fill="black" fill-opacity="0.54"/>
</svg>

After

Width:  |  Height:  |  Size: 340 B

@ -13,17 +13,20 @@ export function fetchAnswerApi(
namePath = '.project';
}
const docPath =
selectedDocs.name === 'default'
? 'default'
: selectedDocs.language +
'/' +
namePath +
'/' +
selectedDocs.version +
'/' +
selectedDocs.model +
'/';
let docPath = 'default';
if (selectedDocs.location === 'local') {
docPath = 'local' + '/' + selectedDocs.name + '/';
} else if (selectedDocs.location === 'remote') {
docPath =
selectedDocs.language +
'/' +
namePath +
'/' +
selectedDocs.version +
'/' +
selectedDocs.model +
'/';
}
return fetch(apiHost + '/api/answer', {
method: 'POST',

@ -1,5 +1,6 @@
// not all properties in Doc are going to be present. Make some optional
export type Doc = {
location: string;
name: string;
language: string;
version: string;
@ -13,9 +14,10 @@ export type Doc = {
//Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later.
export async function getDocs(): Promise<Doc[] | null> {
try {
const response = await fetch(
'https://d3dg1063dc54p9.cloudfront.net/combined.json',
);
const apiHost =
import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
const response = await fetch(apiHost + '/api/combine');
const data = await response.json();
const docs: Doc[] = [];
@ -52,17 +54,13 @@ export function setLocalRecentDocs(doc: Doc): void {
namePath = '.project';
}
const docPath =
doc.name === 'default'
? 'default'
: doc.language +
'/' +
namePath +
'/' +
doc.version +
'/' +
doc.model +
'/';
let docPath = 'default';
if (doc.location === 'local') {
docPath = 'local' + '/' + doc.name + '/';
} else if (doc.location === 'remote') {
docPath =
doc.language + '/' + namePath + '/' + doc.version + '/' + doc.model + '/';
}
const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
fetch(apiHost + '/api/docs_check', {
method: 'POST',

@ -15,6 +15,7 @@ const store = configureStore({
selectedDocs: doc !== null ? JSON.parse(doc) : null,
sourceDocs: [
{
location: '',
language: '',
name: 'default',
version: '',

@ -0,0 +1,201 @@
import React from 'react';
import { useCallback, useEffect, useState } from 'react';
import { useDropzone } from 'react-dropzone';
import { useDispatch } from 'react-redux';
import { ActiveState } from '../models/misc';
import { getDocs } from '../preferences/preferenceApi';
import { setSourceDocs } from '../preferences/preferenceSlice';
export default function Upload({
modalState,
setModalState,
}: {
modalState: ActiveState;
setModalState: (state: ActiveState) => void;
}) {
const [docName, setDocName] = useState('');
const [files, setfiles] = useState<File[]>([]);
const [progress, setProgress] = useState<{
type: 'UPLOAD' | 'TRAINIING';
percentage: number;
taskId?: string;
}>();
function Progress({
title,
isCancellable = false,
}: {
title: string;
isCancellable?: boolean;
}) {
return (
<div className="mt-5 flex flex-col items-center gap-2">
<p className="text-xl tracking-[0.15px]">{title}...</p>
<p className="text-sm text-gray-2000">This may take several minutes</p>
<p className="mt-10 text-2xl">{progress?.percentage || 0}%</p>
<div className="mb-10 w-[50%]">
<div className="h-1 w-[100%] bg-blue-4000"></div>
<div
className={`relative bottom-1 h-1 bg-blue-5000 transition-all`}
style={{ width: `${progress?.percentage || 0}%` }}
></div>
</div>
<button
onClick={() => {
setDocName('');
setfiles([]);
setProgress(undefined);
setModalState('INACTIVE');
}}
className={`rounded-md bg-blue-3000 px-4 py-2 text-sm font-medium text-white ${
isCancellable ? '' : 'hidden'
}`}
>
Finish
</button>
</div>
);
}
function UploadProgress() {
return <Progress title="Upload is in progress"></Progress>;
}
function TrainingProgress() {
const dispatch = useDispatch();
useEffect(() => {
(progress?.percentage ?? 0) < 100 &&
setTimeout(() => {
const apiHost = import.meta.env.VITE_API_HOST;
fetch(`${apiHost}/api/task_status?task_id=${progress?.taskId}`)
.then((data) => data.json())
.then((data) => {
if (data.status == 'SUCCESS') {
getDocs().then((data) => dispatch(setSourceDocs(data)));
setProgress(
(progress) => progress && { ...progress, percentage: 100 },
);
} else {
setProgress(
(progress) =>
progress && {
...progress,
percentage: data.result.current,
},
);
}
});
}, 5000);
}, [progress, dispatch]);
return (
<Progress
title="Training is in progress"
isCancellable={progress?.percentage === 100}
></Progress>
);
}
const onDrop = useCallback((acceptedFiles: File[]) => {
setfiles(acceptedFiles);
setDocName(acceptedFiles[0]?.name);
}, []);
const doNothing = () => undefined;
const uploadFile = () => {
const formData = new FormData();
files.forEach((file) => {
formData.append('file', file);
});
formData.append('name', docName);
formData.append('user', 'local');
const apiHost = import.meta.env.VITE_API_HOST;
const xhr = new XMLHttpRequest();
xhr.upload.addEventListener('progress', (event) => {
const progress = +((event.loaded / event.total) * 100).toFixed(2);
setProgress({ type: 'UPLOAD', percentage: progress });
});
xhr.onload = () => {
const { task_id } = JSON.parse(xhr.responseText);
setProgress({ type: 'TRAINIING', percentage: 0, taskId: task_id });
};
xhr.open('POST', `${apiHost + '/api/upload'}`);
xhr.send(formData);
};
const { getRootProps, getInputProps, isDragActive } = useDropzone({
onDrop,
multiple: true,
onDragEnter: doNothing,
onDragOver: doNothing,
onDragLeave: doNothing,
});
let view;
if (progress?.type === 'UPLOAD') {
view = <UploadProgress></UploadProgress>;
} else if (progress?.type === 'TRAINIING') {
view = <TrainingProgress></TrainingProgress>;
} else {
view = (
<>
<p className="mb-7 text-xl text-jet">Upload New Documentation</p>
<input
type="text"
className="h-10 w-[60%] rounded-md border-2 border-gray-5000 px-3 outline-none"
value={docName}
onChange={(e) => setDocName(e.target.value)}
></input>
<div className="relative bottom-12 left-2 mt-[-18.39px]">
<span className="bg-white px-2 text-xs text-gray-4000">Name</span>
</div>
<div {...getRootProps()}>
<span className="rounded-md border border-blue-2000 px-4 py-2 font-medium text-blue-2000 hover:cursor-pointer">
<input type="button" {...getInputProps()} />
Choose Files
</span>
</div>
<div className="mt-9">
<p className="mb-5 font-medium text-eerie-black">Uploaded Files</p>
{files.map((file) => (
<p key={file.name} className="text-gray-6000">
{file.name}
</p>
))}
{files.length === 0 && <p className="text-gray-6000">None</p>}
</div>
<div className="flex flex-row-reverse">
<button
onClick={uploadFile}
className="ml-6 rounded-md bg-blue-3000 py-2 px-6 text-white"
>
Train
</button>
<button
onClick={() => {
setDocName('');
setfiles([]);
setModalState('INACTIVE');
}}
className="font-medium"
>
Cancel
</button>
</div>
</>
);
}
return (
<article
className={`${
modalState === 'ACTIVE' ? 'visible' : 'hidden'
} absolute z-30 h-screen w-screen bg-gray-alpha`}
>
<article className="mx-auto mt-24 flex w-[90vw] max-w-lg flex-col gap-4 rounded-lg bg-white p-6 shadow-lg">
{view}
</article>
</article>
);
}
// TODO: sanitize all inputs

@ -16,10 +16,16 @@ module.exports = {
'gray-2000': 'rgba(0, 0, 0, 0.5)',
'gray-3000': 'rgba(243, 243, 243, 1)',
'gray-4000': '#949494',
'gray-5000': '#BBBBBB',
'gray-6000': '#757575',
'red-1000': 'rgb(254, 202, 202)',
'red-2000': '#F44336',
'red-3000': '#621B16',
'blue-1000': '#7D54D1',
'blue-2000': '#002B49',
'blue-3000': '#4B02E2',
'blue-4000': 'rgba(0, 125, 255, 0.36)',
'blue-5000': 'rgba(0, 125, 255)',
},
},
},

Loading…
Cancel
Save