From 5883a0685ab6c833a974a007f7f5e8a12299988a Mon Sep 17 00:00:00 2001
From: sigoden <sigoden@gmail.com>
Date: Wed, 15 May 2024 00:31:36 +0000
Subject: [PATCH] refactor: improve tokenzie

---
 src/utils/mod.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/utils/mod.rs b/src/utils/mod.rs
index 6d9d295..31bb971 100644
--- a/src/utils/mod.rs
+++ b/src/utils/mod.rs
@@ -36,7 +36,8 @@ pub fn get_env_name(key: &str) -> String {
 
 pub fn tokenize(text: &str) -> Vec<&str> {
     if text.is_ascii() {
-        text.split_whitespace().collect()
+        text.split_inclusive(|c: char| c.is_ascii_whitespace())
+            .collect()
     } else {
         unicode_segmentation::UnicodeSegmentation::graphemes(text, true).collect()
     }