Update application files, fix LLM models, and create new retriever class

application/api/answer/routes.py

@@ -205,7 +205,12 @@ def stream():
     else:
         source = {}
 
-    retriever = RetrieverCreator.create_retriever("classic", question=question, 
+    if source["active_docs"].split("/")[0] == "default" or source["active_docs"].split("/")[0] == "local":
+        retriever_name = "classic"
+    else:
+        retriever_name = source['active_docs']
+
+    retriever = RetrieverCreator.create_retriever(retriever_name, question=question, 
         source=source, chat_history=history, prompt=prompt, chunks=chunks, gpt_model=gpt_model
         )
 
@@ -247,7 +252,12 @@ def api_answer():
         else:
             source = {data}
 
-        retriever = RetrieverCreator.create_retriever("classic", question=question, 
+        if source["active_docs"].split("/")[0] == "default" or source["active_docs"].split("/")[0] == "local":
+            retriever_name = "classic"
+        else:
+            retriever_name = source['active_docs']
+
+        retriever = RetrieverCreator.create_retriever(retriever_name, question=question, 
             source=source, chat_history=history, prompt=prompt, chunks=chunks, gpt_model=gpt_model
             )
         source_log_docs = []
@@ -290,9 +300,14 @@ def api_search():
     else:
         chunks = 2
 
-    retriever = RetrieverCreator.create_retriever("classic", question=question, 
-            source=source, chat_history=[], prompt="default", chunks=chunks, gpt_model=gpt_model
-            )
+    if source["active_docs"].split("/")[0] == "default" or source["active_docs"].split("/")[0] == "local":
+        retriever_name = "classic"
+    else:
+        retriever_name = source['active_docs']
+
+    retriever = RetrieverCreator.create_retriever(retriever_name, question=question, 
+        source=source, chat_history=[], prompt="default", chunks=chunks, gpt_model=gpt_model
+        )
     docs = retriever.search()
     return docs
 

application/api/user/routes.py

@@ -237,6 +237,20 @@ def combined_json():
         for index in data_remote:
             index["location"] = "remote"
             data.append(index)
+    if 'duckduck_search' in settings.RETRIEVERS_ENABLED:
+        data.append(
+            {
+                "name": "DuckDuckGo Search",
+                "language": "en",
+                "version": "",
+                "description": "duckduck_search",
+                "fullName": "DuckDuckGo Search",
+                "date": "duckduck_search",
+                "docLink": "duckduck_search",
+                "model": settings.EMBEDDINGS_NAME,
+                "location": "custom",
+            }
+        )
 
     return jsonify(data)
 

application/core/settings.py

@@ -18,6 +18,7 @@ class Settings(BaseSettings):
     TOKENS_MAX_HISTORY: int = 150
     UPLOAD_FOLDER: str = "inputs"
     VECTOR_STORE: str = "faiss"  # "faiss" or "elasticsearch" or "qdrant"
+    RETRIEVERS_ENABLED: list = ["classic_rag", "duckduck_search"] # also brave_search
 
     API_URL: str = "http://localhost:7091"  # backend url for celery worker
 

application/requirements.txt

@@ -3,7 +3,7 @@ boto3==1.34.6
 celery==5.3.6
 dataclasses_json==0.6.3
 docx2txt==0.8
-duckduckgo-search=5.3.0
+duckduckgo-search==5.3.0
 EbookLib==0.18
 elasticsearch==8.12.0
 escodegen==1.0.11

application/retriever/retriever_creator.py

@@ -6,7 +6,7 @@ from application.retriever.duckduck_search import DuckDuckSearch
 class RetrieverCreator:
     retievers = {
         'classic': ClassicRAG,
-        'duckduck': DuckDuckSearch
+        'duckduck_search': DuckDuckSearch
     }
 
     @classmethod

frontend/src/conversation/conversationApi.ts

@@ -3,6 +3,33 @@ import { Doc } from '../preferences/preferenceApi';
 
 const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';
 
+function getDocPath(selectedDocs: Doc | null): string {
+  let docPath = 'default';
+
+  if (selectedDocs) {
+    let namePath = selectedDocs.name;
+    if (selectedDocs.language === namePath) {
+      namePath = '.project';
+    }
+    if (selectedDocs.location === 'local') {
+      docPath = 'local' + '/' + selectedDocs.name + '/';
+    } else if (selectedDocs.location === 'remote') {
+      docPath =
+        selectedDocs.language +
+        '/' +
+        namePath +
+        '/' +
+        selectedDocs.version +
+        '/' +
+        selectedDocs.model +
+        '/';
+    } else if (selectedDocs.location === 'custom') {
+      docPath = selectedDocs.docLink;
+    }
+  }
+
+  return docPath;
+}
 export function fetchAnswerApi(
   question: string,
   signal: AbortSignal,
@@ -28,27 +55,7 @@ export function fetchAnswerApi(
       title: any;
     }
 > {
-  let docPath = 'default';
-
-  if (selectedDocs) {
-    let namePath = selectedDocs.name;
-    if (selectedDocs.language === namePath) {
-      namePath = '.project';
-    }
-    if (selectedDocs.location === 'local') {
-      docPath = 'local' + '/' + selectedDocs.name + '/';
-    } else if (selectedDocs.location === 'remote') {
-      docPath =
-        selectedDocs.language +
-        '/' +
-        namePath +
-        '/' +
-        selectedDocs.version +
-        '/' +
-        selectedDocs.model +
-        '/';
-    }
-  }
+  const docPath = getDocPath(selectedDocs);
   //in history array remove all keys except prompt and response
   history = history.map((item) => {
     return { prompt: item.prompt, response: item.response };
@@ -98,27 +105,7 @@ export function fetchAnswerSteaming(
   chunks: string,
   onEvent: (event: MessageEvent) => void,
 ): Promise<Answer> {
-  let docPath = 'default';
-
-  if (selectedDocs) {
-    let namePath = selectedDocs.name;
-    if (selectedDocs.language === namePath) {
-      namePath = '.project';
-    }
-    if (selectedDocs.location === 'local') {
-      docPath = 'local' + '/' + selectedDocs.name + '/';
-    } else if (selectedDocs.location === 'remote') {
-      docPath =
-        selectedDocs.language +
-        '/' +
-        namePath +
-        '/' +
-        selectedDocs.version +
-        '/' +
-        selectedDocs.model +
-        '/';
-    }
-  }
+  const docPath = getDocPath(selectedDocs);
 
   history = history.map((item) => {
     return { prompt: item.prompt, response: item.response };
@@ -195,31 +182,7 @@ export function searchEndpoint(
   history: Array<any> = [],
   chunks: string,
 ) {
-  /*
-  "active_docs": "default",
-  "question": "Summarise",
-  "conversation_id": null,
-  "history": "[]" */
-  let docPath = 'default';
-  if (selectedDocs) {
-    let namePath = selectedDocs.name;
-    if (selectedDocs.language === namePath) {
-      namePath = '.project';
-    }
-    if (selectedDocs.location === 'local') {
-      docPath = 'local' + '/' + selectedDocs.name + '/';
-    } else if (selectedDocs.location === 'remote') {
-      docPath =
-        selectedDocs.language +
-        '/' +
-        namePath +
-        '/' +
-        selectedDocs.version +
-        '/' +
-        selectedDocs.model +
-        '/';
-    }
-  }
+  const docPath = getDocPath(selectedDocs);
 
   const body = {
     question: question,