Merge pull request #830 from arc53/feature/search-endpoint

Search docs not inside the /stream in the stream request

application/api/answer/routes.py

@@ -117,12 +117,9 @@ def complete_stream(question, docsearch, chat_history, api_key, prompt_id, conve
     source_log_docs = []
     for doc in docs:
         if doc.metadata:
-            data = json.dumps({"type": "source", "doc": doc.page_content[:10], "metadata": doc.metadata})
             source_log_docs.append({"title": doc.metadata['title'].split('/')[-1], "text": doc.page_content})
         else:
-            data = json.dumps({"type": "source", "doc": doc.page_content[:10]})
             source_log_docs.append({"title": doc.page_content, "text": doc.page_content})
-        yield f"data:{data}\n\n"
 
     if len(chat_history) > 1:
         tokens_current_history = 0
@@ -343,3 +340,32 @@ def api_answer():
         traceback.print_exc()
         print(str(e))
         return bad_request(500, str(e))
+
+
+@answer.route("/api/search", methods=["POST"])
+def api_search():
+    data = request.get_json()
+    # get parameter from url question
+    question = data["question"]
+
+    if not embeddings_key_set:
+        embeddings_key = data["embeddings_key"]
+    else:
+        embeddings_key = settings.EMBEDDINGS_KEY
+    if "active_docs" in data:
+        vectorstore = get_vectorstore({"active_docs": data["active_docs"]})
+    else:
+        vectorstore = ""
+    docsearch = VectorCreator.create_vectorstore(settings.VECTOR_STORE, vectorstore, embeddings_key)
+
+    docs = docsearch.search(question, k=2)
+
+    source_log_docs = []
+    for doc in docs:
+        if doc.metadata:
+            source_log_docs.append({"title": doc.metadata['title'].split('/')[-1], "text": doc.page_content})
+        else:
+            source_log_docs.append({"title": doc.page_content, "text": doc.page_content})
+        #yield f"data:{data}\n\n"
+    return source_log_docs
+

frontend/src/conversation/conversationApi.ts

@@ -12,20 +12,20 @@ export function fetchAnswerApi(
   promptId: string | null,
 ): Promise<
   | {
-      result: any;
-      answer: any;
-      sources: any;
-      conversationId: any;
-      query: string;
-    }
+    result: any;
+    answer: any;
+    sources: any;
+    conversationId: any;
+    query: string;
+  }
   | {
-      result: any;
-      answer: any;
-      sources: any;
-      query: string;
-      conversationId: any;
-      title: any;
-    }
+    result: any;
+    answer: any;
+    sources: any;
+    query: string;
+    conversationId: any;
+    title: any;
+  }
 > {
   let namePath = selectedDocs.name;
   if (selectedDocs.language === namePath) {
@@ -128,7 +128,6 @@ export function fetchAnswerSteaming(
       conversation_id: conversationId,
       prompt_id: promptId,
     };
-
     fetch(apiHost + '/stream', {
       method: 'POST',
       headers: {
@@ -183,7 +182,58 @@ export function fetchAnswerSteaming(
       });
   });
 }
+export function searchEndpoint(
+  question: string,
+  apiKey: string,
+  selectedDocs: Doc,
+  conversation_id: string | null,
+  history: Array<any> = [],
+) {
+  /*
+  "active_docs": "default",
+  "question": "Summarise",
+  "conversation_id": null,
+  "history": "[]" */
+  let namePath = selectedDocs.name;
+  if (selectedDocs.language === namePath) {
+    namePath = '.project';
+  }
 
+  let docPath = 'default';
+  if (selectedDocs.location === 'local') {
+    docPath = 'local' + '/' + selectedDocs.name + '/';
+  } else if (selectedDocs.location === 'remote') {
+    docPath =
+      selectedDocs.language +
+      '/' +
+      namePath +
+      '/' +
+      selectedDocs.version +
+      '/' +
+      selectedDocs.model +
+      '/';
+  }
+
+  const body = {
+    question: question,
+    active_docs: docPath,
+    conversation_id,
+    history
+  };
+  return fetch(`${apiHost}/api/search`, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(
+      body
+    ),
+  }).then((response) => response.json())
+    .then((data) => {
+      return data;
+    })
+    .catch(err => console.log(err))
+}
 export function sendFeedback(
   prompt: string,
   response: string,

frontend/src/conversation/conversationSlice.ts

@@ -1,6 +1,7 @@
 import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit';
 import store from '../store';
 import { fetchAnswerApi, fetchAnswerSteaming } from './conversationApi';
+import { searchEndpoint } from './conversationApi';
 import { Answer, ConversationState, Query, Status } from './conversationModels';
 import { getConversations } from '../preferences/preferenceApi';
 import { setConversations } from '../preferences/preferenceSlice';
@@ -29,6 +30,7 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
           (event) => {
             const data = JSON.parse(event.data);
 
+
             // check if the 'end' event has been received
             if (data.type === 'end') {
               // set status to 'idle'
@@ -40,24 +42,22 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
                 .catch((error) => {
                   console.error('Failed to fetch conversations: ', error);
                 });
-            } else if (data.type === 'source') {
-              // check if data.metadata exists
-              let result;
-              if (data.metadata && data.metadata.title) {
-                const titleParts = data.metadata.title.split('/');
-                result = {
-                  title: titleParts[titleParts.length - 1],
-                  text: data.doc,
-                };
-              } else {
-                result = { title: data.doc, text: data.doc };
-              }
-              dispatch(
-                updateStreamingSource({
-                  index: state.conversation.queries.length - 1,
-                  query: { sources: [result] },
-                }),
-              );
+
+              searchEndpoint( //search for sources post streaming
+                question,
+                state.preference.apiKey,
+                state.preference.selectedDocs!,
+                state.conversation.conversationId,
+                state.conversation.queries
+              ).then(sources => {
+                //dispatch streaming sources
+                dispatch(
+                  updateStreamingSource({
+                    index: state.conversation.queries.length - 1,
+                    query: { sources },
+                  }),
+                );
+              });
             } else if (data.type === 'id') {
               dispatch(
                 updateConversationId({
@@ -165,9 +165,10 @@ export const conversationSlice = createSlice({
       state,
       action: PayloadAction<{ index: number; query: Partial<Query> }>,
     ) {
+
       const { index, query } = action.payload;
       if (!state.queries[index].sources) {
-        state.queries[index].sources = [query.sources![0]];
+        state.queries[index].sources = query?.sources;
       } else {
         state.queries[index].sources!.push(query.sources![0]);
       }