Merge pull request #185 from arc53/feature/upload

application/app.py

@@ -136,6 +136,7 @@ def api_answer():
                 vectorstore = ""
         else:
             vectorstore = ""
+        print(vectorstore)
         # vectorstore = "outputs/inputs/"
         # loading the index and the store and the prompt template
         # Note if you have used other embeddings than OpenAI, you need to change the embeddings
@@ -409,8 +410,11 @@ def delete_old():
     if dirs[0] not in ['indexes', 'vectors']:
         return {"status": 'error'}
     path_clean = '/'.join(dirs)
-    shutil.rmtree(path)
     vectors_collection.delete_one({'location': path})
+    try:
+        shutil.rmtree(path_clean)
+    except FileNotFoundError:
+        pass
     return {"status": 'ok'}
 
 # handling CORS

frontend/.env.development

@@ -1,2 +1,2 @@
 # Please put appropriate value
-VITE_API_HOST = https://docsapi.arc53.com
+VITE_API_HOST = http://localhost:5001

frontend/package-lock.json

@@ -1105,6 +1105,11 @@
       "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
       "dev": true
     },
+    "attr-accept": {
+      "version": "2.2.2",
+      "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.2.tgz",
+      "integrity": "sha512-7prDjvt9HmqiZ0cl5CRjtS84sEyhsHP2coDkaZKRKVfCDo9s7iw7ChVmar78Gu9pC4SoR/28wFu/G5JJhTnqEg=="
+    },
     "autoprefixer": {
       "version": "10.4.13",
       "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz",
@@ -2168,6 +2173,21 @@
         "flat-cache": "^3.0.4"
       }
     },
+    "file-selector": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/file-selector/-/file-selector-0.6.0.tgz",
+      "integrity": "sha512-QlZ5yJC0VxHxQQsQhXvBaC7VRJ2uaxTf+Tfpu4Z/OcVQJVpZO+DGU0rkoVW5ce2SccxugvpBJoMvUs59iILYdw==",
+      "requires": {
+        "tslib": "^2.4.0"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.5.0",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
+          "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
+        }
+      }
+    },
     "fill-range": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
@@ -3088,8 +3108,7 @@
     "object-assign": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
-      "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==",
+      "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="
-      "dev": true
     },
     "object-hash": {
       "version": "3.0.0",
@@ -3395,7 +3414,6 @@
       "version": "15.8.1",
       "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
       "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==",
-      "dev": true,
       "requires": {
         "loose-envify": "^1.4.0",
         "object-assign": "^4.1.1",
@@ -3437,6 +3455,16 @@
         "scheduler": "^0.23.0"
       }
     },
+    "react-dropzone": {
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-14.2.3.tgz",
+      "integrity": "sha512-O3om8I+PkFKbxCukfIR3QAGftYXDZfOE2N1mr/7qebQJHs7U+/RSL/9xomJNpRg9kM5h9soQSdf0Gc7OHF5Fug==",
+      "requires": {
+        "attr-accept": "^2.2.2",
+        "file-selector": "^0.6.0",
+        "prop-types": "^15.8.1"
+      }
+    },
     "react-is": {
       "version": "16.13.1",
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",

frontend/package.json

@@ -23,6 +23,7 @@
     "@vercel/analytics": "^0.1.10",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
+    "react-dropzone": "^14.2.3",
     "react-redux": "^8.0.5",
     "react-router-dom": "^6.8.1"
   },

frontend/src/Navigation.tsx

@@ -2,11 +2,13 @@ import { useEffect, useRef, useState } from 'react';
 import { NavLink } from 'react-router-dom';
 import Arrow1 from './assets/arrow.svg';
 import Arrow2 from './assets/dropdown-arrow.svg';
+import Exit from './assets/exit.svg';
 import Message from './assets/message.svg';
 import Hamburger from './assets/hamburger.svg';
 import Key from './assets/key.svg';
 import Info from './assets/info.svg';
 import Link from './assets/link.svg';
+import UploadIcon from './assets/upload.svg';
 import { ActiveState } from './models/misc';
 import APIKeyModal from './preferences/APIKeyModal';
 import SelectDocsModal from './preferences/SelectDocsModal';
@@ -19,6 +21,8 @@ import {
   setSelectedDocs,
 } from './preferences/preferenceSlice';
 import { useOutsideAlerter } from './hooks';
+import Upload from './upload/Upload';
+import { Doc } from './preferences/preferenceApi';
 
 export default function Navigation({
   navState,
@@ -42,7 +46,28 @@ export default function Navigation({
   const [selectedDocsModalState, setSelectedDocsModalState] =
     useState<ActiveState>(isSelectedDocsSet ? 'INACTIVE' : 'ACTIVE');
 
+  const [uploadModalState, setUploadModalState] =
+    useState<ActiveState>('INACTIVE');
+
   const navRef = useRef(null);
+  const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
+
+  const handleDeleteClick = (index: number, doc: Doc) => {
+    const docPath = 'indexes/' + 'local' + '/' + doc.name;
+
+    fetch(`${apiHost}/api/delete_old?path=${docPath}`, {
+      method: 'GET',
+    })
+      .then(() => {
+        // remove the image element from the DOM
+        const imageElement = document.querySelector(
+          `#img-${index}`,
+        ) as HTMLElement;
+        const parentElement = imageElement.parentNode as HTMLElement;
+        parentElement.parentNode?.removeChild(parentElement);
+      })
+      .catch((error) => console.error(error));
+  };
   useOutsideAlerter(
     navRef,
     () => {
@@ -109,7 +134,7 @@ export default function Navigation({
 
         <div className="flex-grow border-b-2 border-gray-100"></div>
         <div className="flex flex-col-reverse border-b-2">
-          <div className="relative my-4 px-6">
+          <div className="relative my-4 flex gap-2 px-2">
             <div
               className="flex h-12 w-full cursor-pointer justify-between rounded-md border-2 bg-white"
               onClick={() => setIsDocsListOpen(!isDocsListOpen)}
@@ -127,8 +152,13 @@ export default function Navigation({
                 } mr-3 w-3 transition-all`}
               />
             </div>
+            <img
+              className="mt-2 h-9 w-9 hover:cursor-pointer"
+              src={UploadIcon}
+              onClick={() => setUploadModalState('ACTIVE')}
+            ></img>
             {isDocsListOpen && (
-              <div className="absolute top-12 left-0 right-0 mx-6 max-h-52 overflow-y-scroll bg-white shadow-lg">
+              <div className="absolute top-12 left-0 right-6 ml-2 mr-4 max-h-52 overflow-y-scroll bg-white shadow-lg">
                 {docs ? (
                   docs.map((doc, index) => {
                     if (doc.model) {
@@ -139,11 +169,23 @@ export default function Navigation({
                             dispatch(setSelectedDocs(doc));
                             setIsDocsListOpen(false);
                           }}
-                          className="h-10 w-full cursor-pointer border-x-2 border-b-2 hover:bg-gray-100"
+                          className="flex h-10 w-full cursor-pointer items-center justify-between border-x-2 border-b-2 hover:bg-gray-100"
                         >
-                          <p className="ml-5 py-3">
+                          <p className="ml-5 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3">
                             {doc.name} {doc.version}
                           </p>
+                          {doc.location === 'local' ? (
+                            <img
+                              src={Exit}
+                              alt="Exit"
+                              className="mr-4 h-3 w-3 cursor-pointer hover:opacity-50"
+                              id={`img-${index}`}
+                              onClick={(event) => {
+                                event.stopPropagation();
+                                handleDeleteClick(index, doc);
+                              }}
+                            />
+                          ) : null}
                         </div>
                       );
                     }
@@ -153,6 +195,7 @@ export default function Navigation({
                     <p className="ml-5 py-3">No default documentation.</p>
                   </div>
                 )}
+                )
               </div>
             )}
           </div>
@@ -222,6 +265,10 @@ export default function Navigation({
         setModalState={setApiKeyModalState}
         isCancellable={isApiKeySet}
       />
+      <Upload
+        modalState={uploadModalState}
+        setModalState={setUploadModalState}
+      ></Upload>
     </>
   );
 }

frontend/src/assets/upload.svg

@@ -0,0 +1,3 @@
+<svg width="24" height="16" viewBox="0 0 24 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M19.35 6.04C18.67 2.59 15.64 0 12 0C9.11 0 6.6 1.64 5.35 4.04C2.34 4.36 0 6.91 0 10C0 13.31 2.69 16 6 16H19C21.76 16 24 13.76 24 11C24 8.36 21.95 6.22 19.35 6.04ZM14 9V13H10V9H7L12 4L17 9H14Z" fill="black" fill-opacity="0.54"/>
+</svg>

frontend/src/conversation/conversationApi.ts

@@ -13,10 +13,12 @@ export function fetchAnswerApi(
     namePath = '.project';
   }
 
-  const docPath =
+  let docPath = 'default';
-    selectedDocs.name === 'default'
+  if (selectedDocs.location === 'local') {
-      ? 'default'
+    docPath = 'local' + '/' + selectedDocs.name + '/';
-      : selectedDocs.language +
+  } else if (selectedDocs.location === 'remote') {
+    docPath =
+      selectedDocs.language +
       '/' +
       namePath +
       '/' +
@@ -24,6 +26,7 @@ export function fetchAnswerApi(
       '/' +
       selectedDocs.model +
       '/';
+  }
 
   return fetch(apiHost + '/api/answer', {
     method: 'POST',

frontend/src/preferences/preferenceApi.ts

@@ -1,5 +1,6 @@
 // not all properties in Doc are going to be present. Make some optional
 export type Doc = {
+  location: string;
   name: string;
   language: string;
   version: string;
@@ -13,9 +14,10 @@ export type Doc = {
 //Fetches all JSON objects from the source. We only use the objects with the "model" property in SelectDocsModal.tsx. Hopefully can clean up the source file later.
 export async function getDocs(): Promise<Doc[] | null> {
   try {
-    const response = await fetch(
+    const apiHost =
-      'https://d3dg1063dc54p9.cloudfront.net/combined.json',
+      import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
-    );
+
+    const response = await fetch(apiHost + '/api/combine');
     const data = await response.json();
 
     const docs: Doc[] = [];
@@ -52,17 +54,13 @@ export function setLocalRecentDocs(doc: Doc): void {
     namePath = '.project';
   }
 
-  const docPath =
+  let docPath = 'default';
-    doc.name === 'default'
+  if (doc.location === 'local') {
-      ? 'default'
+    docPath = 'local' + '/' + doc.name + '/';
-      : doc.language +
+  } else if (doc.location === 'remote') {
-        '/' +
+    docPath =
-        namePath +
+      doc.language + '/' + namePath + '/' + doc.version + '/' + doc.model + '/';
-        '/' +
+  }
-        doc.version +
-        '/' +
-        doc.model +
-        '/';
   const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
   fetch(apiHost + '/api/docs_check', {
     method: 'POST',

frontend/src/store.ts

@@ -15,6 +15,7 @@ const store = configureStore({
       selectedDocs: doc !== null ? JSON.parse(doc) : null,
       sourceDocs: [
         {
+          location: '',
           language: '',
           name: 'default',
           version: '',

frontend/src/upload/Upload.tsx

@@ -0,0 +1,201 @@
+import React from 'react';
+import { useCallback, useEffect, useState } from 'react';
+import { useDropzone } from 'react-dropzone';
+import { useDispatch } from 'react-redux';
+import { ActiveState } from '../models/misc';
+import { getDocs } from '../preferences/preferenceApi';
+import { setSourceDocs } from '../preferences/preferenceSlice';
+
+export default function Upload({
+  modalState,
+  setModalState,
+}: {
+  modalState: ActiveState;
+  setModalState: (state: ActiveState) => void;
+}) {
+  const [docName, setDocName] = useState('');
+  const [files, setfiles] = useState<File[]>([]);
+  const [progress, setProgress] = useState<{
+    type: 'UPLOAD' | 'TRAINIING';
+    percentage: number;
+    taskId?: string;
+  }>();
+
+  function Progress({
+    title,
+    isCancellable = false,
+  }: {
+    title: string;
+    isCancellable?: boolean;
+  }) {
+    return (
+      <div className="mt-5 flex flex-col items-center gap-2">
+        <p className="text-xl tracking-[0.15px]">{title}...</p>
+        <p className="text-sm text-gray-2000">This may take several minutes</p>
+        <p className="mt-10 text-2xl">{progress?.percentage || 0}%</p>
+        <div className="mb-10 w-[50%]">
+          <div className="h-1 w-[100%] bg-blue-4000"></div>
+          <div
+            className={`relative bottom-1 h-1 bg-blue-5000 transition-all`}
+            style={{ width: `${progress?.percentage || 0}%` }}
+          ></div>
+        </div>
+        <button
+          onClick={() => {
+            setDocName('');
+            setfiles([]);
+            setProgress(undefined);
+            setModalState('INACTIVE');
+          }}
+          className={`rounded-md bg-blue-3000 px-4 py-2 text-sm font-medium text-white ${
+            isCancellable ? '' : 'hidden'
+          }`}
+        >
+          Finish
+        </button>
+      </div>
+    );
+  }
+
+  function UploadProgress() {
+    return <Progress title="Upload is in progress"></Progress>;
+  }
+
+  function TrainingProgress() {
+    const dispatch = useDispatch();
+    useEffect(() => {
+      (progress?.percentage ?? 0) < 100 &&
+        setTimeout(() => {
+          const apiHost = import.meta.env.VITE_API_HOST;
+          fetch(`${apiHost}/api/task_status?task_id=${progress?.taskId}`)
+            .then((data) => data.json())
+            .then((data) => {
+              if (data.status == 'SUCCESS') {
+                getDocs().then((data) => dispatch(setSourceDocs(data)));
+                setProgress(
+                  (progress) => progress && { ...progress, percentage: 100 },
+                );
+              } else {
+                setProgress(
+                  (progress) =>
+                    progress && {
+                      ...progress,
+                      percentage: data.result.current,
+                    },
+                );
+              }
+            });
+        }, 5000);
+    }, [progress, dispatch]);
+    return (
+      <Progress
+        title="Training is in progress"
+        isCancellable={progress?.percentage === 100}
+      ></Progress>
+    );
+  }
+
+  const onDrop = useCallback((acceptedFiles: File[]) => {
+    setfiles(acceptedFiles);
+    setDocName(acceptedFiles[0]?.name);
+  }, []);
+
+  const doNothing = () => undefined;
+
+  const uploadFile = () => {
+    const formData = new FormData();
+    files.forEach((file) => {
+      formData.append('file', file);
+    });
+    formData.append('name', docName);
+    formData.append('user', 'local');
+    const apiHost = import.meta.env.VITE_API_HOST;
+    const xhr = new XMLHttpRequest();
+    xhr.upload.addEventListener('progress', (event) => {
+      const progress = +((event.loaded / event.total) * 100).toFixed(2);
+      setProgress({ type: 'UPLOAD', percentage: progress });
+    });
+    xhr.onload = () => {
+      const { task_id } = JSON.parse(xhr.responseText);
+      setProgress({ type: 'TRAINIING', percentage: 0, taskId: task_id });
+    };
+    xhr.open('POST', `${apiHost + '/api/upload'}`);
+    xhr.send(formData);
+  };
+
+  const { getRootProps, getInputProps, isDragActive } = useDropzone({
+    onDrop,
+    multiple: true,
+    onDragEnter: doNothing,
+    onDragOver: doNothing,
+    onDragLeave: doNothing,
+  });
+
+  let view;
+  if (progress?.type === 'UPLOAD') {
+    view = <UploadProgress></UploadProgress>;
+  } else if (progress?.type === 'TRAINIING') {
+    view = <TrainingProgress></TrainingProgress>;
+  } else {
+    view = (
+      <>
+        <p className="mb-7 text-xl text-jet">Upload New Documentation</p>
+        <input
+          type="text"
+          className="h-10 w-[60%] rounded-md border-2 border-gray-5000 px-3 outline-none"
+          value={docName}
+          onChange={(e) => setDocName(e.target.value)}
+        ></input>
+        <div className="relative bottom-12 left-2 mt-[-18.39px]">
+          <span className="bg-white px-2 text-xs text-gray-4000">Name</span>
+        </div>
+        <div {...getRootProps()}>
+          <span className="rounded-md border border-blue-2000 px-4 py-2 font-medium text-blue-2000 hover:cursor-pointer">
+            <input type="button" {...getInputProps()} />
+            Choose Files
+          </span>
+        </div>
+        <div className="mt-9">
+          <p className="mb-5 font-medium text-eerie-black">Uploaded Files</p>
+          {files.map((file) => (
+            <p key={file.name} className="text-gray-6000">
+              {file.name}
+            </p>
+          ))}
+          {files.length === 0 && <p className="text-gray-6000">None</p>}
+        </div>
+        <div className="flex flex-row-reverse">
+          <button
+            onClick={uploadFile}
+            className="ml-6 rounded-md bg-blue-3000 py-2 px-6 text-white"
+          >
+            Train
+          </button>
+          <button
+            onClick={() => {
+              setDocName('');
+              setfiles([]);
+              setModalState('INACTIVE');
+            }}
+            className="font-medium"
+          >
+            Cancel
+          </button>
+        </div>
+      </>
+    );
+  }
+
+  return (
+    <article
+      className={`${
+        modalState === 'ACTIVE' ? 'visible' : 'hidden'
+      } absolute z-30  h-screen w-screen  bg-gray-alpha`}
+    >
+      <article className="mx-auto mt-24 flex w-[90vw] max-w-lg  flex-col gap-4 rounded-lg bg-white p-6 shadow-lg">
+        {view}
+      </article>
+    </article>
+  );
+}
+// TODO: sanitize all inputs

frontend/tailwind.config.cjs

@@ -16,10 +16,16 @@ module.exports = {
         'gray-2000': 'rgba(0, 0, 0, 0.5)',
         'gray-3000': 'rgba(243, 243, 243, 1)',
         'gray-4000': '#949494',
+        'gray-5000': '#BBBBBB',
+        'gray-6000': '#757575',
         'red-1000': 'rgb(254, 202, 202)',
         'red-2000': '#F44336',
         'red-3000': '#621B16',
         'blue-1000': '#7D54D1',
+        'blue-2000': '#002B49',
+        'blue-3000': '#4B02E2',
+        'blue-4000': 'rgba(0, 125, 255, 0.36)',
+        'blue-5000': 'rgba(0, 125, 255)',
       },
     },
   },