diff --git a/gpt4all-backend/replit.cpp b/gpt4all-backend/replit.cpp index 8689b7aa..df26aa44 100644 --- a/gpt4all-backend/replit.cpp +++ b/gpt4all-backend/replit.cpp @@ -146,14 +146,12 @@ std::vector replit_tokenizer_tokenize(replit_tokenizer & tokeniz return tokenized.first; } -std::string_view replit_tokenizer_detokenize(replit_tokenizer & tokenizer, const std::vector & tokens) { +std::string replit_tokenizer_detokenize(replit_tokenizer & tokenizer, const std::vector & tokens) { std::string text; for (auto token : tokens) { text += tokenizer.raw_vocab.id_to_token[token]; } - static std::string denormalized_text; - denormalized_text = replace_all(text, ws_symbol, " "); - return denormalized_text; + return replace_all(text, ws_symbol, " "); } // no defaults for now