Merge branch 'arc53:main' into fix-celery-import-error

This commit is contained in:
xucailiang 2024-06-19 15:16:39 +08:00 committed by GitHub
commit d87b411193
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 410 additions and 186 deletions

View File

@ -8,6 +8,30 @@ from langchain_community.embeddings import (
from langchain_openai import OpenAIEmbeddings
from application.core.settings import settings
class EmbeddingsSingleton:
_instances = {}
@staticmethod
def get_instance(embeddings_name, *args, **kwargs):
if embeddings_name not in EmbeddingsSingleton._instances:
EmbeddingsSingleton._instances[embeddings_name] = EmbeddingsSingleton._create_instance(embeddings_name, *args, **kwargs)
return EmbeddingsSingleton._instances[embeddings_name]
@staticmethod
def _create_instance(embeddings_name, *args, **kwargs):
embeddings_factory = {
"openai_text-embedding-ada-002": OpenAIEmbeddings,
"huggingface_sentence-transformers/all-mpnet-base-v2": HuggingFaceEmbeddings,
"huggingface_sentence-transformers-all-mpnet-base-v2": HuggingFaceEmbeddings,
"huggingface_hkunlp/instructor-large": HuggingFaceInstructEmbeddings,
"cohere_medium": CohereEmbeddings
}
if embeddings_name not in embeddings_factory:
raise ValueError(f"Invalid embeddings_name: {embeddings_name}")
return embeddings_factory[embeddings_name](*args, **kwargs)
class BaseVectorStore(ABC):
def __init__(self):
pass
@ -20,42 +44,36 @@ class BaseVectorStore(ABC):
return settings.OPENAI_API_BASE and settings.OPENAI_API_VERSION and settings.AZURE_DEPLOYMENT_NAME
def _get_embeddings(self, embeddings_name, embeddings_key=None):
embeddings_factory = {
"openai_text-embedding-ada-002": OpenAIEmbeddings,
"huggingface_sentence-transformers/all-mpnet-base-v2": HuggingFaceEmbeddings,
"huggingface_hkunlp/instructor-large": HuggingFaceInstructEmbeddings,
"cohere_medium": CohereEmbeddings
}
if embeddings_name not in embeddings_factory:
raise ValueError(f"Invalid embeddings_name: {embeddings_name}")
if embeddings_name == "openai_text-embedding-ada-002":
if self.is_azure_configured():
os.environ["OPENAI_API_TYPE"] = "azure"
embedding_instance = embeddings_factory[embeddings_name](
embedding_instance = EmbeddingsSingleton.get_instance(
embeddings_name,
model=settings.AZURE_EMBEDDINGS_DEPLOYMENT_NAME
)
else:
embedding_instance = embeddings_factory[embeddings_name](
embedding_instance = EmbeddingsSingleton.get_instance(
embeddings_name,
openai_api_key=embeddings_key
)
elif embeddings_name == "cohere_medium":
embedding_instance = embeddings_factory[embeddings_name](
embedding_instance = EmbeddingsSingleton.get_instance(
embeddings_name,
cohere_api_key=embeddings_key
)
elif embeddings_name == "huggingface_sentence-transformers/all-mpnet-base-v2":
if os.path.exists("./model/all-mpnet-base-v2"):
embedding_instance = embeddings_factory[embeddings_name](
embedding_instance = EmbeddingsSingleton.get_instance(
embeddings_name,
model_name="./model/all-mpnet-base-v2",
model_kwargs={"device": "cpu"},
model_kwargs={"device": "cpu"}
)
else:
embedding_instance = embeddings_factory[embeddings_name](
model_kwargs={"device": "cpu"},
embedding_instance = EmbeddingsSingleton.get_instance(
embeddings_name,
model_kwargs={"device": "cpu"}
)
else:
embedding_instance = embeddings_factory[embeddings_name]()
embedding_instance = EmbeddingsSingleton.get_instance(embeddings_name)
return embedding_instance

View File

@ -1,13 +1,17 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="viewport" content="width=device-width, initial-scale=1.0,viewport-fit=cover" />
<meta name="apple-mobile-web-app-capable" content="yes">
<title>DocsGPT 🦖</title>
<link rel="shortcut icon" type="image/x-icon" href="/favicon.ico" />
</head>
<body>
<div id="root" class="h-screen"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>

View File

@ -4,7 +4,13 @@ import { useTranslation } from 'react-i18next';
export default function Hero({
handleQuestion,
}: {
handleQuestion: (question: string) => void;
handleQuestion: ({
question,
isRetry,
}: {
question: string;
isRetry?: boolean;
}) => void;
}) {
const { t } = useTranslation();
const demos = t('demo', { returnObjects: true }) as Array<{
@ -23,14 +29,14 @@ export default function Hero({
<div className="mb-4 flex flex-col items-center justify-center dark:text-white"></div>
</div>
<div className="grid w-full grid-cols-1 items-center gap-4 self-center text-xs sm:w-auto sm:gap-6 md:text-sm lg:grid-cols-2">
<div className="mb-16 grid w-full grid-cols-1 items-center gap-4 self-center text-xs sm:w-auto sm:gap-6 md:mb-0 md:text-sm lg:grid-cols-2">
{demos?.map(
(demo: { header: string; query: string }, key: number) =>
demo.header &&
demo.query && (
<Fragment key={key}>
<button
onClick={() => handleQuestion(demo.query)}
onClick={() => handleQuestion({ question: demo.query })}
className="w-full rounded-full border-2 border-silver px-6 py-4 text-left hover:border-gray-4000 dark:hover:border-gray-3000 xl:min-w-[24vw]"
>
<p className="mb-1 font-semibold text-black dark:text-silver">

View File

@ -0,0 +1,17 @@
import * as React from 'react';
import { SVGProps } from 'react';
const RetryIcon = (props: SVGProps<SVGSVGElement>) => (
<svg
xmlns="http://www.w3.org/2000/svg"
xmlSpace="preserve"
width={16}
height={16}
fill={props.fill}
stroke={props.stroke}
viewBox="0 0 383.748 383.748"
{...props}
>
<path d="M62.772 95.042C90.904 54.899 137.496 30 187.343 30c83.743 0 151.874 68.13 151.874 151.874h30C369.217 81.588 287.629 0 187.343 0c-35.038 0-69.061 9.989-98.391 28.888a182.423 182.423 0 0 0-47.731 44.705L2.081 34.641v113.365h113.91L62.772 95.042zM381.667 235.742h-113.91l53.219 52.965c-28.132 40.142-74.724 65.042-124.571 65.042-83.744 0-151.874-68.13-151.874-151.874h-30c0 100.286 81.588 181.874 181.874 181.874 35.038 0 69.062-9.989 98.391-28.888a182.443 182.443 0 0 0 47.731-44.706l39.139 38.952V235.742z" />
</svg>
);
export default RetryIcon;

View File

@ -19,6 +19,7 @@ import { FEEDBACK, Query } from './conversationModels';
import { sendFeedback } from './conversationApi';
import { useTranslation } from 'react-i18next';
import ArrowDown from './../assets/arrow-down.svg';
import RetryIcon from '../components/RetryIcon';
export default function Conversation() {
const queries = useSelector(selectQueries);
const status = useSelector(selectStatus);
@ -29,6 +30,7 @@ export default function Conversation() {
const [hasScrolledToLast, setHasScrolledToLast] = useState(true);
const fetchStream = useRef<any>(null);
const [eventInterrupt, setEventInterrupt] = useState(false);
const [lastQueryReturnedErr, setLastQueryReturnedErr] = useState(false);
const { t } = useTranslation();
const handleUserInterruption = () => {
@ -73,6 +75,13 @@ export default function Conversation() {
};
}, [endMessageRef.current]);
useEffect(() => {
if (queries.length) {
queries[queries.length - 1].error && setLastQueryReturnedErr(true);
queries[queries.length - 1].response && setLastQueryReturnedErr(false); //considering a query that initially returned error can later include a response property on retry
}
}, [queries[queries.length - 1]]);
const scrollIntoView = () => {
endMessageRef?.current?.scrollIntoView({
behavior: 'smooth',
@ -80,13 +89,20 @@ export default function Conversation() {
});
};
const handleQuestion = (question: string) => {
const handleQuestion = ({
question,
isRetry = false,
}: {
question: string;
isRetry?: boolean;
}) => {
question = question.trim();
if (question === '') return;
setEventInterrupt(false);
dispatch(addQuery({ prompt: question }));
!isRetry && dispatch(addQuery({ prompt: question })); //dispatch only new queries
fetchStream.current = dispatch(fetchAnswer({ question }));
};
const handleFeedback = (query: Query, feedback: FEEDBACK, index: number) => {
const prevFeedback = query.feedback;
dispatch(updateQuery({ index, query: { feedback } }));
@ -95,19 +111,32 @@ export default function Conversation() {
);
};
const handleQuestionSubmission = () => {
if (inputRef.current?.textContent && status !== 'loading') {
if (lastQueryReturnedErr) {
// update last failed query with new prompt
dispatch(
updateQuery({
index: queries.length - 1,
query: {
prompt: inputRef.current.textContent,
},
}),
);
handleQuestion({
question: queries[queries.length - 1].prompt,
isRetry: true,
});
} else {
handleQuestion({ question: inputRef.current.textContent });
}
inputRef.current.textContent = '';
}
};
const prepResponseView = (query: Query, index: number) => {
let responseView;
if (query.error) {
responseView = (
<ConversationBubble
ref={endMessageRef}
className={`${index === queries.length - 1 ? 'mb-32' : 'mb-7'}`}
key={`${index}ERROR`}
message={query.error}
type="ERROR"
></ConversationBubble>
);
} else if (query.response) {
if (query.response) {
responseView = (
<ConversationBubble
ref={endMessageRef}
@ -122,6 +151,35 @@ export default function Conversation() {
}
></ConversationBubble>
);
} else if (query.error) {
const retryBtn = (
<button
className="flex items-center justify-center gap-3 self-center rounded-full border border-silver py-3 px-5 text-lg text-gray-500 transition-colors delay-100 hover:border-gray-500 disabled:cursor-not-allowed dark:text-bright-gray"
disabled={status === 'loading'}
onClick={() => {
handleQuestion({
question: queries[queries.length - 1].prompt,
isRetry: true,
});
}}
>
<RetryIcon
fill={isDarkTheme ? 'rgb(236 236 241)' : 'rgb(107 114 120)'}
stroke={isDarkTheme ? 'rgb(236 236 241)' : 'rgb(107 114 120)'}
/>
Retry
</button>
);
responseView = (
<ConversationBubble
ref={endMessageRef}
className={`${index === queries.length - 1 ? 'mb-32' : 'mb-7'} `}
key={`${index}ERROR`}
message={query.error}
type="ERROR"
retryBtn={retryBtn}
></ConversationBubble>
);
}
return responseView;
};
@ -137,13 +195,13 @@ export default function Conversation() {
<div
onWheel={handleUserInterruption}
onTouchMove={handleUserInterruption}
className="flex h-[85vh] w-full justify-center overflow-y-auto p-4"
className="flex h-[90%] w-full justify-center overflow-y-auto p-4 md:h-[84vh]"
>
{queries.length > 0 && !hasScrolledToLast && (
<button
onClick={scrollIntoView}
aria-label="scroll to bottom"
className="fixed bottom-32 right-14 z-10 flex h-7 w-7 items-center justify-center rounded-full border-[0.5px] border-gray-alpha bg-gray-100 bg-opacity-50 dark:bg-purple-taupe md:h-9 md:w-9 md:bg-opacity-100 "
className="fixed bottom-40 right-14 z-10 flex h-7 w-7 items-center justify-center rounded-full border-[0.5px] border-gray-alpha bg-gray-100 bg-opacity-50 dark:bg-purple-taupe md:h-9 md:w-9 md:bg-opacity-100 "
>
<img
src={ArrowDown}
@ -165,16 +223,19 @@ export default function Conversation() {
type="QUESTION"
sources={query.sources}
></ConversationBubble>
{prepResponseView(query, index)}
</Fragment>
);
})}
</div>
)}
{queries.length === 0 && <Hero handleQuestion={handleQuestion} />}
</div>
<div className="bottom-0 flex w-11/12 flex-col items-end self-center bg-white pt-1 dark:bg-raisin-black sm:w-6/12 md:fixed">
<div className="flex h-full w-full items-center rounded-full border border-silver">
<div className="bottom-safe fixed flex w-11/12 flex-col items-end self-center rounded-2xl bg-opacity-0 pb-1 sm:w-6/12">
<div className="flex h-full w-full items-center rounded-full border border-silver bg-white dark:bg-raisin-black">
<div
id="inputbox"
ref={inputRef}
@ -186,34 +247,27 @@ export default function Conversation() {
onKeyDown={(e) => {
if (e.key === 'Enter' && !e.shiftKey) {
e.preventDefault();
if (inputRef.current?.textContent && status !== 'loading') {
handleQuestion(inputRef.current.textContent);
inputRef.current.textContent = '';
}
handleQuestionSubmission();
}
}}
></div>
{status === 'loading' ? (
<img
src={isDarkTheme ? SpinnerDark : Spinner}
className="relative right-[38px] bottom-[15px] -mr-[30px] animate-spin cursor-pointer self-end bg-transparent"
className="relative right-[38px] bottom-[24px] -mr-[30px] animate-spin cursor-pointer self-end bg-transparent"
></img>
) : (
<div className="mx-1 cursor-pointer rounded-full p-4 text-center hover:bg-gray-3000">
<img
className="w-6 text-white "
onClick={() => {
if (inputRef.current?.textContent) {
handleQuestion(inputRef.current.textContent);
inputRef.current.textContent = '';
}
}}
onClick={handleQuestionSubmission}
src={isDarkTheme ? SendDark : Send}
></img>
</div>
)}
</div>
<p className="text-gray-595959 hidden w-[100vw] self-center bg-white bg-transparent p-5 text-center text-xs dark:bg-raisin-black dark:text-bright-gray md:inline md:w-full">
<p className="text-gray-595959 hidden w-[100vw] self-center bg-white bg-transparent py-2 text-center text-xs dark:bg-raisin-black dark:text-bright-gray md:inline md:w-full">
{t('tagline')}
</p>
</div>

View File

@ -23,9 +23,10 @@ const ConversationBubble = forwardRef<
feedback?: FEEDBACK;
handleFeedback?: (feedback: FEEDBACK) => void;
sources?: { title: string; text: string; source: string }[];
retryBtn?: React.ReactElement;
}
>(function ConversationBubble(
{ message, type, className, feedback, handleFeedback, sources },
{ message, type, className, feedback, handleFeedback, sources, retryBtn },
ref,
) {
const [openSource, setOpenSource] = useState<number | null>(null);
@ -69,12 +70,17 @@ const ConversationBubble = forwardRef<
<div
className={`ml-2 mr-5 flex max-w-[90vw] rounded-3xl bg-gray-1000 p-3.5 dark:bg-gun-metal md:max-w-[70vw] lg:max-w-[50vw] ${
type === 'ERROR'
? 'flex-row items-center rounded-full border border-transparent bg-[#FFE7E7] p-2 py-5 text-sm font-normal text-red-3000 dark:border-red-2000 dark:text-white'
? 'relative flex-row items-center rounded-full border border-transparent bg-[#FFE7E7] p-2 py-5 text-sm font-normal text-red-3000 dark:border-red-2000 dark:text-white'
: 'flex-col rounded-3xl'
}`}
>
{type === 'ERROR' && (
<>
<img src={Alert} alt="alert" className="mr-2 inline" />
<div className="absolute -right-32 top-1/2 -translate-y-1/2">
{retryBtn}
</div>
</>
)}
<ReactMarkdown
className="whitespace-pre-wrap break-normal leading-normal"

View File

@ -1,6 +1,5 @@
import { Answer, FEEDBACK } from './conversationModels';
import { Doc } from '../preferences/preferenceApi';
import { selectTokenLimit } from '../preferences/preferenceSlice';
const apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';

View File

@ -408,3 +408,7 @@ template {
::-webkit-scrollbar {
width: 0;
}
.bottom-safe {
bottom: env(safe-area-inset-bottom, 0);
}

View File

@ -4,6 +4,7 @@ import { initReactI18next } from 'react-i18next';
import en from './en.json'; //English
import es from './es.json'; //Spanish
import jp from './jp.json'; //Japanese
import zh from './zh.json'; //Mandarin
i18n.use(initReactI18next).init({
resources: {
@ -16,6 +17,9 @@ i18n.use(initReactI18next).init({
jp: {
translation: jp,
},
zh: {
translation: zh,
},
},
});

View File

@ -106,4 +106,3 @@
}
}
}

109
frontend/src/locale/zh.json Normal file
View File

@ -0,0 +1,109 @@
{
"language": "普通话",
"chat": "聊天",
"chats": "聊天",
"newChat": "新聊天",
"myPlan": "我的计划",
"about": "关于",
"inputPlaceholder": "在这里输入您的消息...",
"tagline": "DocsGPT 使用 GenAI, 请使用来源审核关键信息.",
"sourceDocs": "来源文档",
"none": "无",
"cancel":"取消",
"demo": [
{
"header": "了解 DocsGPT",
"query": "DocsGPT 是什么"
},
{
"header": "总结文档",
"query": "总结当前情况"
},
{
"header": "编写代码",
"query": "为 /api/answer API 请求编写代码"
},
{
"header": "学习帮助",
"query": "为背景写出潜在问题"
}
],
"settings": {
"label": "设置",
"general": {
"label": "般",
"selectTheme": "选择主题",
"light": "浅色",
"dark": "暗色",
"selectLanguage": "选择语言",
"chunks": "每个查询处理的块",
"prompt": "提示",
"deleteAllLabel": "删除所有对话",
"deleteAllBtn": "删除所有",
"addNew": "添加新的",
"convHistory":"对话历史",
"none":"无",
"low":"低",
"medium":"中",
"high":"高",
"unlimited":"无限",
"default":"默认"
},
"documents": {
"label": "文件",
"name": "文件名称",
"date": "向量日期",
"type": "类型",
"tokenUsage": "令牌使用"
},
"apiKeys": {
"label": "API 密钥",
"name": "名称",
"key": "API 密钥",
"sourceDoc": "源文档",
"createNew": "创建新的"
}
},
"modals": {
"uploadDoc": {
"label": "上传新文档资料",
"file": "从文件",
"remote": "远程",
"name": "名称",
"choose": "选择文件",
"info": "请上传 .pdf, .txt, .rst, .docx, .md, .zip 文件,限 25MB",
"uploadedFiles": "已上传文件",
"cancel": "取消",
"train": "训练",
"link": "链接",
"urlLink": "URL 链接",
"reddit": {
"id": "客户端 ID",
"secret": "客户端密钥",
"agent": "用户代理",
"searchQueries": "搜索查询",
"numberOfPosts": "帖子数量"
}
},
"createAPIKey": {
"label": "创建新的 API 密钥",
"apiKeyName": "API 密钥名称",
"chunks": "每个查询处理的块",
"prompt": "选择活动提示",
"sourceDoc": "源文档",
"create": "创建"
},
"saveKey": {
"note": "请保存您的密钥",
"disclaimer": "这是您的密钥唯一一次展示机会。",
"copy": "复制",
"copied": "已复制",
"confirm": "我已保存密钥"
},
"deleteConv": {
"confirm": "您确定要删除所有对话吗?",
"delete": "删除"
}
}
}

View File

@ -36,6 +36,10 @@ const General: React.FC = () => {
label: 'Japanese',
value: 'jp',
},
{
label: 'Mandarin',
value: 'zh',
},
];
const chunks = ['0', '2', '4', '6', '8', '10'];
const token_limits = new Map([