From d3996a1ddc6433ba9f6821e31a194032301de0a9 Mon Sep 17 00:00:00 2001 From: justheuristic Date: Sun, 28 Aug 2022 18:29:37 +0300 Subject: [PATCH] remove expensive assertions in inference code --- src/server/backend.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/server/backend.py b/src/server/backend.py index 6a85b16..6a883f6 100644 --- a/src/server/backend.py +++ b/src/server/backend.py @@ -81,8 +81,6 @@ class TransformerBackend(ModuleBackend): assert new_k.shape[0] == past_k.shape[0] and new_v.shape[0] == past_v.shape[0] assert new_k.shape[1] == new_length and new_v.shape[1] == new_length assert new_k.shape[2:] == past_k.shape[2:] and new_v.shape[2:] == past_v.shape[2:] - assert torch.allclose(new_v[:, : past_v.shape[1]], past_v) - assert torch.allclose(new_k[:, : past_k.shape[1]], past_k) cache[0, :, prefix_length:new_length, :] = new_k[:, prefix_length:new_length] cache[1, :, prefix_length:new_length, :] = new_v[:, prefix_length:new_length] return (hidden_states,)