From 79d6243fe1bc73f2d5af968e6b78587433d77b66 Mon Sep 17 00:00:00 2001 From: Adam Treat Date: Tue, 16 May 2023 09:48:21 -0400 Subject: [PATCH] Use the default for max_tokens to avoid errors. --- gpt4all-chat/chatgpt.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gpt4all-chat/chatgpt.cpp b/gpt4all-chat/chatgpt.cpp index 1ea16105..b2f833a4 100644 --- a/gpt4all-chat/chatgpt.cpp +++ b/gpt4all-chat/chatgpt.cpp @@ -81,12 +81,16 @@ void ChatGPT::prompt(const std::string &prompt, m_ctx = &promptCtx; m_responseCallback = responseCallback; + // FIXME: We don't set the max_tokens on purpose because in order to do so safely without encountering + // an error we need to be able to count the tokens in our prompt. The only way to do this is to use + // the OpenAI tiktokken library or to implement our own tokenization function that matches precisely + // the tokenization used by the OpenAI model we're calling. OpenAI has not introduced any means of + // using the REST API to count tokens in a prompt. QJsonObject root; root.insert("model", m_modelName); root.insert("stream", true); root.insert("temperature", promptCtx.temp); root.insert("top_p", promptCtx.top_p); - root.insert("max_tokens", 200); QJsonArray messages; for (int i = 0; i < m_context.count() && i < promptCtx.n_past; ++i) {