Explicitly clear the kv cache each time we eval tokens to match n_past. (#1808)

9 months ago · 96cee4f9ac
parent 2d566710e5
commit 96cee4f9ac
1 changed files with 2 additions and 0 deletions
--- a/gpt4all-backend/llamamodel.cpp
+++ b/gpt4all-backend/llamamodel.cpp
@ -298,6 +298,8 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const

 bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
 {
+    llama_kv_cache_seq_rm(d_ptr->ctx, 0, ctx.n_past, -1);
+
    llama_batch batch = llama_batch_init(tokens.size(), 0, 1);

    batch.n_tokens = tokens.size();