From e0ecbdff10129203cd203379550f189739398176 Mon Sep 17 00:00:00 2001
From: Tim Stack <timothyshanestack@gmail.com>
Date: Mon, 19 Sep 2022 20:15:35 -0700
Subject: [PATCH] [logfile] improve handling of ansi escapes

I think this covers the rest of the functionality
that needs to deal with ansi escapes.

Related to #1057
---
 src/base/ansi_scrubber.cc                     | 62 +++++++++++++++++++
 src/base/ansi_scrubber.hh                     |  2 +
 src/base/file_range.hh                        | 16 ++---
 src/base/intern_string.hh                     |  5 ++
 src/hotkeys.cc                                | 38 +++++++-----
 src/line_buffer.cc                            |  1 +
 src/log.watch.cc                              |  8 ++-
 src/log_data_helper.cc                        |  1 +
 src/log_data_table.cc                         |  2 +
 src/log_format.cc                             |  1 +
 src/log_search_table.cc                       |  5 +-
 src/log_vtab_impl.cc                          |  6 +-
 src/logfile.cc                                | 34 ++++++----
 src/logfile.hh                                | 13 +++-
 src/logfile_sub_source.cc                     | 13 ++++
 src/readline_possibilities.cc                 |  1 +
 src/shared_buffer.cc                          | 17 +++++
 src/shared_buffer.hh                          |  7 +++
 src/spectro_impls.cc                          |  1 +
 src/textfile_sub_source.cc                    |  6 +-
 test/expected/expected.am                     | 56 ++---------------
 ...02faf7817c494c33e32da7ee95f13aa9210d01.err |  0
 ...02faf7817c494c33e32da7ee95f13aa9210d01.out | 10 +++
 ...40b674cd65936a72bd64b1dac1524d16fe44c3.err |  0
 ...40b674cd65936a72bd64b1dac1524d16fe44c3.out | 11 ++++
 test/logfile_ansi.1                           | 10 +++
 test/test_ansi_scrubber.cc                    | 15 +++++
 test/test_logfile.sh                          | 10 +++
 28 files changed, 257 insertions(+), 94 deletions(-)
 create mode 100644 test/expected/test_logfile.sh_6602faf7817c494c33e32da7ee95f13aa9210d01.err
 create mode 100644 test/expected/test_logfile.sh_6602faf7817c494c33e32da7ee95f13aa9210d01.out
 create mode 100644 test/expected/test_logfile.sh_e840b674cd65936a72bd64b1dac1524d16fe44c3.err
 create mode 100644 test/expected/test_logfile.sh_e840b674cd65936a72bd64b1dac1524d16fe44c3.out
 create mode 100644 test/logfile_ansi.1

diff --git a/src/base/ansi_scrubber.cc b/src/base/ansi_scrubber.cc
index d413490d..a7bb77c7 100644
--- a/src/base/ansi_scrubber.cc
+++ b/src/base/ansi_scrubber.cc
@@ -48,6 +48,68 @@ ansi_regex()
     return retval;
 }
 
+size_t
+erase_ansi_escapes(string_fragment input)
+{
+    const auto& regex = ansi_regex();
+    auto md = regex.create_match_data();
+
+    auto matcher = regex.capture_from(input).into(md);
+    while (true) {
+        auto match_res = matcher.matches(PCRE2_NO_UTF_CHECK);
+
+        if (match_res.is<lnav::pcre2pp::matcher::not_found>()) {
+            break;
+        }
+        if (match_res.is<lnav::pcre2pp::matcher::error>()) {
+            log_error("ansi scrub regex failure");
+            break;
+        }
+
+        auto sf = md[0].value();
+        auto bs_index_res = sf.codepoint_to_byte_index(1);
+
+        if (sf.length() >= 3 && bs_index_res.isOk()
+            && sf[bs_index_res.unwrap()] == '\b')
+        {
+            static const auto OVERSTRIKE_RE
+                = lnav::pcre2pp::code::from_const(R"((\X)\x08(\X))");
+
+            size_t fill_index = 0;
+            auto loop_res = OVERSTRIKE_RE.capture_from(sf).for_each(
+                [&fill_index, &sf](lnav::pcre2pp::match_data& over_md) {
+                    auto lhs = over_md[1].value();
+                    if (lhs == "_") {
+                        auto rhs = over_md[2].value();
+                        memmove(sf.writable_data(fill_index),
+                                rhs.data(),
+                                rhs.length());
+                        fill_index += rhs.length();
+                    } else {
+                        memmove(sf.writable_data(fill_index),
+                                lhs.data(),
+                                lhs.length());
+                        fill_index += lhs.length();
+                    }
+                });
+
+            memmove(input.writable_data(sf.sf_begin + fill_index),
+                    md.remaining().data(),
+                    md.remaining().length());
+            input = input.erase(input.sf_string, sf.length() - fill_index);
+        } else {
+            memmove(const_cast<char*>(sf.data()),
+                    md.remaining().data(),
+                    md.remaining().length());
+            input = input.erase(input.sf_string, sf.length());
+        }
+
+        matcher.reload_input(input, sf.sf_begin);
+    }
+
+    return input.length();
+}
+
 void
 scrub_ansi_string(std::string& str, string_attrs_t* sa)
 {
diff --git a/src/base/ansi_scrubber.hh b/src/base/ansi_scrubber.hh
index 77382d81..b832e17c 100644
--- a/src/base/ansi_scrubber.hh
+++ b/src/base/ansi_scrubber.hh
@@ -64,6 +64,8 @@
  */
 void scrub_ansi_string(std::string& str, string_attrs_t* sa);
 
+size_t erase_ansi_escapes(string_fragment input);
+
 /**
  * Populate a variable map with strings that contain escape sequences that
  * might be useful to script writers.
diff --git a/src/base/file_range.hh b/src/base/file_range.hh
index 9d285113..d9a3de49 100644
--- a/src/base/file_range.hh
+++ b/src/base/file_range.hh
@@ -40,8 +40,14 @@ using file_ssize_t = int64_t;
 
 class file_range {
 public:
+    struct metadata {
+        bool m_valid_utf{true};
+        bool m_has_ansi{false};
+    };
+
     file_off_t fr_offset{0};
     file_ssize_t fr_size{0};
+    metadata fr_metadata;
 
     void clear()
     {
@@ -49,15 +55,9 @@ public:
         this->fr_size = 0;
     }
 
-    ssize_t next_offset() const
-    {
-        return this->fr_offset + this->fr_size;
-    }
+    ssize_t next_offset() const { return this->fr_offset + this->fr_size; }
 
-    bool empty() const
-    {
-        return this->fr_size == 0;
-    }
+    bool empty() const { return this->fr_size == 0; }
 };
 
 struct source_location {
diff --git a/src/base/intern_string.hh b/src/base/intern_string.hh
index 3be806ee..8d429a67 100644
--- a/src/base/intern_string.hh
+++ b/src/base/intern_string.hh
@@ -144,6 +144,11 @@ struct string_fragment {
         return (const unsigned char*) &this->sf_string[this->sf_begin];
     }
 
+    char* writable_data(int offset = 0)
+    {
+        return (char*) &this->sf_string[this->sf_begin + offset];
+    }
+
     char front() const { return this->sf_string[this->sf_begin]; }
 
     uint32_t front_codepoint() const
diff --git a/src/hotkeys.cc b/src/hotkeys.cc
index 3237b0a9..e8fa6dae 100644
--- a/src/hotkeys.cc
+++ b/src/hotkeys.cc
@@ -56,7 +56,7 @@ using namespace lnav::roles::literals;
 
 class logline_helper {
 public:
-    logline_helper(logfile_sub_source& lss) : lh_sub_source(lss) {}
+    explicit logline_helper(logfile_sub_source& lss) : lh_sub_source(lss) {}
 
     logline& move_to_msg_start()
     {
@@ -71,7 +71,7 @@ public:
         return (*lf)[cl];
     }
 
-    logline& current_line()
+    logline& current_line() const
     {
         content_line_t cl = this->lh_sub_source.at(this->lh_current_line);
         std::shared_ptr<logfile> lf = this->lh_sub_source.find(cl);
@@ -84,10 +84,11 @@ public:
         this->lh_string_attrs.clear();
         this->lh_line_values.clear();
         content_line_t cl = this->lh_sub_source.at(this->lh_current_line);
-        std::shared_ptr<logfile> lf = this->lh_sub_source.find(cl);
+        auto lf = this->lh_sub_source.find(cl);
         auto ll = lf->begin() + cl;
         auto format = lf->get_format();
         lf->read_full_message(ll, this->lh_line_values.lvv_sbr);
+        this->lh_line_values.lvv_sbr.erase_ansi();
         format->annotate(
             cl, this->lh_string_attrs, this->lh_line_values, false);
     }
@@ -389,7 +390,8 @@ handle_paging_key(int ch)
                     tc->shift_top(1_vl);
                 }
                 if (lnav_data.ld_last_user_mark[tc] + 1
-                    >= tc->get_inner_height()) {
+                    >= tc->get_inner_height())
+                {
                     break;
                 }
                 lnav_data.ld_last_user_mark[tc] += 1;
@@ -435,7 +437,8 @@ handle_paging_key(int ch)
 
         case 'M':
             if (lnav_data.ld_last_user_mark.find(tc)
-                == lnav_data.ld_last_user_mark.end()) {
+                == lnav_data.ld_last_user_mark.end())
+            {
                 alerter::singleton().chime("no lines have been marked");
             } else {
                 int start_line = std::min((int) tc->get_top(),
@@ -479,7 +482,8 @@ handle_paging_key(int ch)
                 while (next_top < tc->get_inner_height()) {
                     if (!lss->find_line(lss->at(next_top))->is_message()) {
                     } else if (lss->get_line_accel_direction(next_top)
-                               == log_accel::A_DECEL) {
+                               == log_accel::A_DECEL)
+                    {
                         --next_top;
                         tc->set_top(next_top);
                         break;
@@ -502,7 +506,8 @@ handle_paging_key(int ch)
                 while (0 <= next_top && next_top < tc->get_inner_height()) {
                     if (!lss->find_line(lss->at(next_top))->is_message()) {
                     } else if (lss->get_line_accel_direction(next_top)
-                               == log_accel::A_DECEL) {
+                               == log_accel::A_DECEL)
+                    {
                         --next_top;
                         tc->set_top(next_top);
                         break;
@@ -589,11 +594,11 @@ handle_paging_key(int ch)
 
         case 'o':
         case 'O':
-            if (lss) {
+            if (lss != nullptr) {
                 logline_helper start_helper(*lss);
 
                 start_helper.lh_current_line = tc->get_top();
-                logline& start_line = start_helper.move_to_msg_start();
+                auto& start_line = start_helper.move_to_msg_start();
                 start_helper.annotate();
 
                 struct line_range opid_range = find_string_attr_range(
@@ -615,7 +620,8 @@ handle_paging_key(int ch)
                     while (true) {
                         if (ch == 'o') {
                             if (++next_helper.lh_current_line
-                                >= tc->get_inner_height()) {
+                                >= tc->get_inner_height())
+                            {
                                 break;
                             }
                         } else {
@@ -771,7 +777,8 @@ handle_paging_key(int ch)
                     for (row = 0; row < dls.dls_rows.size(); row++) {
                         if (strcmp(dls.dls_rows[row][log_line_index.value()],
                                    linestr.data())
-                            == 0) {
+                            == 0)
+                        {
                             vis_line_t db_line(row);
 
                             db_tc->set_top(db_line);
@@ -809,7 +816,8 @@ handle_paging_key(int ch)
                         size_t col_len = strlen(col_value);
 
                         if (dts.scan(col_value, col_len, nullptr, &tm, tv)
-                            != nullptr) {
+                            != nullptr)
+                        {
                             lnav_data.ld_log_source.find_from_time(tv) |
                                 [tc](auto vl) {
                                     tc->set_top(vl);
@@ -890,9 +898,9 @@ handle_paging_key(int ch)
 
         case 'r':
         case 'R':
-            if (lss) {
-                auto& last_time = injector::get<const relative_time&,
-                                                last_relative_time_tag>();
+            if (lss != nullptr) {
+                const auto& last_time = injector::get<const relative_time&,
+                                                      last_relative_time_tag>();
 
                 if (last_time.empty()) {
                     lnav_data.ld_rl_view->set_value(
diff --git a/src/line_buffer.cc b/src/line_buffer.cc
index cb9bcb80..16086c3a 100644
--- a/src/line_buffer.cc
+++ b/src/line_buffer.cc
@@ -1193,6 +1193,7 @@ line_buffer::read_range(const file_range fr)
             FMT_STRING("short-read (need: {}; avail: {})"), fr.fr_size, avail));
     }
     retval.share(this->lb_share_manager, line_start, fr.fr_size);
+    retval.get_metadata() = fr.fr_metadata;
 
     return Ok(std::move(retval));
 }
diff --git a/src/log.watch.cc b/src/log.watch.cc
index a808b10c..5e4b4b94 100644
--- a/src/log.watch.cc
+++ b/src/log.watch.cc
@@ -122,6 +122,7 @@ eval_with(logfile& lf, logfile::iterator ll)
     shared_buffer_ref raw_sbr;
     logline_value_vector values;
     lf.read_full_message(ll, values.lvv_sbr);
+    values.lvv_sbr.erase_ansi();
     auto format = lf.get_format();
     string_attrs_t sa;
     auto line_number = std::distance(lf.begin(), ll);
@@ -275,8 +276,11 @@ eval_with(logfile& lf, logfile::iterator ll)
 
                     string_fragment sf = gen.to_string_fragment();
 
-                    sqlite3_bind_text(
-                        stmt, lpc + 1, sf.data(), sf.length(), SQLITE_TRANSIENT);
+                    sqlite3_bind_text(stmt,
+                                      lpc + 1,
+                                      sf.data(),
+                                      sf.length(),
+                                      SQLITE_TRANSIENT);
                 }
                 continue;
             }
diff --git a/src/log_data_helper.cc b/src/log_data_helper.cc
index 6a53d520..ade077a4 100644
--- a/src/log_data_helper.cc
+++ b/src/log_data_helper.cc
@@ -77,6 +77,7 @@ log_data_helper::parse_line(content_line_t line, bool allow_middle)
         this->ldh_line_attrs.clear();
         this->ldh_line_values.clear();
         this->ldh_file->read_full_message(ll, this->ldh_line_values.lvv_sbr);
+        this->ldh_line_values.lvv_sbr.erase_ansi();
         format->annotate(this->ldh_line_index, sa, this->ldh_line_values);
 
         body = find_string_attr_range(sa, &SA_BODY);
diff --git a/src/log_data_table.cc b/src/log_data_table.cc
index a664c9d9..7a7c5379 100644
--- a/src/log_data_table.cc
+++ b/src/log_data_table.cc
@@ -63,6 +63,7 @@ log_data_table::get_columns_int()
         this->ldt_format_impl->get_columns(cols);
     }
     lf->read_full_message(lf->begin() + cl_copy, line_values.lvv_sbr);
+    line_values.lvv_sbr.erase_ansi();
     format->annotate(cl_copy, sa, line_values, false);
     body = find_string_attr_range(sa, &SA_BODY);
     if (body.lr_end == -1) {
@@ -134,6 +135,7 @@ log_data_table::next(log_cursor& lc, logfile_sub_source& lss)
     logline_value_vector line_values;
 
     lf->read_full_message(lf_iter, line_values.lvv_sbr);
+    line_values.lvv_sbr.erase_ansi();
     lf->get_format()->annotate(cl, sa, line_values, false);
     body = find_string_attr_range(sa, &SA_BODY);
     if (body.lr_end == -1) {
diff --git a/src/log_format.cc b/src/log_format.cc
index a4b667f8..2ce78b46 100644
--- a/src/log_format.cc
+++ b/src/log_format.cc
@@ -1057,6 +1057,7 @@ external_log_format::annotate(uint64_t line_number,
     auto& line = values.lvv_sbr;
     struct line_range lr;
 
+    line.erase_ansi();
     if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
         values = this->jlf_line_values;
         sa = this->jlf_line_attrs;
diff --git a/src/log_search_table.cc b/src/log_search_table.cc
index a2bf5c87..8c9d3c9d 100644
--- a/src/log_search_table.cc
+++ b/src/log_search_table.cc
@@ -29,6 +29,7 @@
 
 #include "log_search_table.hh"
 
+#include "base/ansi_scrubber.hh"
 #include "column_namer.hh"
 #include "config.h"
 #include "sql_util.hh"
@@ -166,7 +167,9 @@ log_search_table::next(log_cursor& lc, logfile_sub_source& lss)
     }
 
     // log_debug("%d: doing message", (int) lc.lc_curr_line);
-    lf->read_full_message(lf_iter, this->lst_line_values_cache.lvv_sbr);
+    auto& sbr = this->lst_line_values_cache.lvv_sbr;
+    lf->read_full_message(lf_iter, sbr);
+    sbr.erase_ansi();
     lf->get_format()->annotate(
         cl, this->vi_attrs, this->lst_line_values_cache, false);
     this->lst_content
diff --git a/src/log_vtab_impl.cc b/src/log_vtab_impl.cc
index 3d38d633..8d2f4337 100644
--- a/src/log_vtab_impl.cc
+++ b/src/log_vtab_impl.cc
@@ -29,6 +29,7 @@
 
 #include "log_vtab_impl.hh"
 
+#include "base/ansi_scrubber.hh"
 #include "base/itertools.hh"
 #include "base/lnav_log.hh"
 #include "base/string_util.hh"
@@ -298,7 +299,9 @@ struct vtab_cursor {
         if (this->log_msg_line == this->log_cursor.lc_curr_line) {
             return;
         }
-        lf->read_full_message(ll, this->line_values.lvv_sbr);
+        auto& sbr = this->line_values.lvv_sbr;
+        lf->read_full_message(ll, sbr);
+        sbr.erase_ansi();
         this->log_msg_line = this->log_cursor.lc_curr_line;
     }
 
@@ -871,6 +874,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
                         shared_buffer_ref line;
 
                         lf->read_full_message(ll, line);
+                        line.erase_ansi();
                         sqlite3_result_text(ctx,
                                             line.get_data(),
                                             line.length(),
diff --git a/src/logfile.cc b/src/logfile.cc
index da752a07..63e3ec30 100644
--- a/src/logfile.cc
+++ b/src/logfile.cc
@@ -464,12 +464,15 @@ logfile::rebuild_index(nonstd::optional<ui_clock::time_point> deadline)
                 auto last_line = this->lf_index.end();
                 --last_line;
                 auto check_line_off = last_line->get_offset();
-                auto last_length = ssize_t(this->line_length(last_line, false));
+                auto last_length_res
+                    = this->message_byte_length(last_line, false);
                 log_debug("flushing at %d", check_line_off);
                 this->lf_line_buffer.flush_at(check_line_off);
 
-                auto read_result = this->lf_line_buffer.read_range(
-                    {check_line_off, last_length});
+                auto read_result = this->lf_line_buffer.read_range({
+                    check_line_off,
+                    last_length_res.mlr_length,
+                });
 
                 if (read_result.isErr()) {
                     log_info("overwritten file detected, closing -- %s (%s)",
@@ -785,11 +788,13 @@ Result<shared_buffer_ref, std::string>
 logfile::read_line(logfile::iterator ll)
 {
     try {
-        return this->lf_line_buffer.read_range(this->get_file_range(ll, false))
-            .map([&ll, this](auto sbr) {
+        auto get_range_res = this->get_file_range(ll, false);
+        return this->lf_line_buffer.read_range(get_range_res)
+            .map([&ll, &get_range_res, this](auto sbr) {
                 sbr.rtrim(is_line_ending);
-                if (!ll->is_valid_utf()) {
+                if (!get_range_res.fr_metadata.m_valid_utf) {
                     scrub_to_utf8(sbr.get_writable_data(), sbr.length());
+                    sbr.get_metadata().m_valid_utf = true;
                 }
 
                 if (this->lf_format != nullptr) {
@@ -798,7 +803,7 @@ logfile::read_line(logfile::iterator ll)
 
                 return sbr;
             });
-    } catch (line_buffer::error& e) {
+    } catch (const line_buffer::error& e) {
         return Err(std::string(strerror(e.e_err)));
     }
 }
@@ -840,6 +845,7 @@ logfile::read_full_message(logfile::const_iterator ll,
             return;
         }
         msg_out = read_result.unwrap();
+        msg_out.get_metadata() = range_for_line.fr_metadata;
         if (this->lf_format.get() != nullptr) {
             this->lf_format->get_subline(*ll, msg_out, true);
         }
@@ -898,19 +904,25 @@ logfile::get_path() const
     return this->lf_filename;
 }
 
-size_t
-logfile::line_length(logfile::const_iterator ll, bool include_continues)
+logfile::message_length_result
+logfile::message_byte_length(logfile::const_iterator ll, bool include_continues)
 {
     auto next_line = ll;
+    file_range::metadata meta;
     size_t retval;
 
     if (!include_continues && this->lf_next_line_cache) {
         if (ll->get_offset() == (*this->lf_next_line_cache).first) {
-            return this->lf_next_line_cache->second;
+            return {
+                (file_ssize_t) this->lf_next_line_cache->second,
+                {ll->is_valid_utf(), ll->has_ansi()},
+            };
         }
     }
 
     do {
+        meta.m_has_ansi = meta.m_has_ansi || next_line->has_ansi();
+        meta.m_valid_utf = meta.m_valid_utf && next_line->is_valid_utf();
         ++next_line;
     } while ((next_line != this->end())
              && ((ll->get_offset() == next_line->get_offset())
@@ -932,7 +944,7 @@ logfile::line_length(logfile::const_iterator ll, bool include_continues)
         }
     }
 
-    return retval;
+    return {(file_ssize_t) retval, meta};
 }
 
 Result<shared_buffer_ref, std::string>
diff --git a/src/logfile.hh b/src/logfile.hh
index 264b63e3..61aa0726 100644
--- a/src/logfile.hh
+++ b/src/logfile.hh
@@ -264,13 +264,22 @@ public:
         return retval;
     }
 
-    size_t line_length(const_iterator ll, bool include_continues = true);
+    struct message_length_result {
+        file_ssize_t mlr_length;
+        file_range::metadata mlr_metadata;
+    };
+
+    message_length_result message_byte_length(const_iterator ll,
+                                              bool include_continues = true);
 
     file_range get_file_range(const_iterator ll, bool include_continues = true)
     {
+        auto mlr = this->message_byte_length(ll, include_continues);
+
         return {
             ll->get_offset(),
-            (file_ssize_t) this->line_length(ll, include_continues),
+            mlr.mlr_length,
+            mlr.mlr_metadata,
         };
     }
 
diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc
index bb4f0d07..b4807389 100644
--- a/src/logfile_sub_source.cc
+++ b/src/logfile_sub_source.cc
@@ -192,6 +192,10 @@ logfile_sub_source::text_value_for_line(textview_curses& tc,
 
         this->lss_token_file->read_full_message(this->lss_token_line, sbr);
         this->lss_token_value = to_string(sbr);
+        if (sbr.get_metadata().m_has_ansi) {
+            scrub_ansi_string(this->lss_token_value, &this->lss_token_attrs);
+            sbr.get_metadata().m_has_ansi = false;
+        }
     } else {
         this->lss_token_value
             = this->lss_token_file->read_line(this->lss_token_line)
@@ -1407,6 +1411,7 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
     logline_value_vector values;
     auto& sbr = values.lvv_sbr;
     lf->read_full_message(ll, sbr);
+    sbr.erase_ansi();
     auto format = lf->get_format();
     string_attrs_t sa;
     auto line_number = std::distance(lf->cbegin(), ll);
@@ -2061,6 +2066,13 @@ logline_window::logmsg_info::load_msg() const
     auto format = this->li_file->get_format();
     this->li_file->read_full_message(this->li_logline,
                                      this->li_line_values.lvv_sbr);
+    if (this->li_line_values.lvv_sbr.get_metadata().m_has_ansi) {
+        auto* writable_data = this->li_line_values.lvv_sbr.get_writable_data();
+        auto str
+            = std::string{writable_data, this->li_line_values.lvv_sbr.length()};
+        scrub_ansi_string(str, &this->li_string_attrs);
+        this->li_line_values.lvv_sbr.get_metadata().m_has_ansi = false;
+    }
     format->annotate(std::distance(this->li_file->cbegin(), this->li_logline),
                      this->li_string_attrs,
                      this->li_line_values,
@@ -2204,6 +2216,7 @@ logfile_sub_source::text_crumbs_for_line(int line,
     auto& sbr = values.lvv_sbr;
 
     lf->read_full_message(msg_start_iter, sbr);
+    sbr.erase_ansi();
     attr_line_t al(to_string(sbr));
     format->annotate(file_line_number, al.get_attrs(), values);
 
diff --git a/src/readline_possibilities.cc b/src/readline_possibilities.cc
index 68cab7de..66698f19 100644
--- a/src/readline_possibilities.cc
+++ b/src/readline_possibilities.cc
@@ -258,6 +258,7 @@ add_filter_expr_possibilities(readline_curses* rlc,
         logline_value_vector values;
 
         lf->read_full_message(ll, values.lvv_sbr);
+        values.lvv_sbr.erase_ansi();
         format->annotate(cl, sa, values);
         for (auto& lv : values.lvv_values) {
             if (!lv.lv_meta.lvm_struct_name.empty()) {
diff --git a/src/shared_buffer.cc b/src/shared_buffer.cc
index 7e23a657..7ad168ac 100644
--- a/src/shared_buffer.cc
+++ b/src/shared_buffer.cc
@@ -37,6 +37,7 @@
 
 #include <algorithm>
 
+#include "base/ansi_scrubber.hh"
 #include "shared_buffer.hh"
 
 static const bool DEBUG_TRACE = false;
@@ -163,6 +164,7 @@ shared_buffer_ref::copy_ref(const shared_buffer_ref& other)
             const_cast<char*>(this->sb_data), other.sb_data, other.sb_length);
         this->sb_length = other.sb_length;
     }
+    this->sb_metadata = other.sb_metadata;
 }
 
 shared_buffer_ref::narrow_result
@@ -179,3 +181,18 @@ shared_buffer_ref::widen(narrow_result old_data_length)
     this->sb_data = old_data_length.first;
     this->sb_length = old_data_length.second;
 }
+
+void
+shared_buffer_ref::erase_ansi()
+{
+    if (!this->sb_metadata.m_has_ansi) {
+        return;
+    }
+
+    auto* writable_data = this->get_writable_data();
+    auto new_len = erase_ansi_escapes(
+        string_fragment::from_bytes(writable_data, this->sb_length));
+
+    this->sb_length = new_len;
+    this->sb_metadata.m_has_ansi = false;
+}
diff --git a/src/shared_buffer.hh b/src/shared_buffer.hh
index c71bab2d..61aa5e0d 100644
--- a/src/shared_buffer.hh
+++ b/src/shared_buffer.hh
@@ -41,6 +41,7 @@
 
 #include "base/attr_line.hh"
 #include "base/auto_mem.hh"
+#include "base/file_range.hh"
 #include "base/intern_string.hh"
 #include "base/lnav_log.hh"
 #include "scn/util/string_view.h"
@@ -61,6 +62,7 @@ public:
         this->sb_owner = nullptr;
         this->sb_data = nullptr;
         this->sb_length = 0;
+        this->sb_metadata = file_range::metadata{};
 
         this->copy_ref(other);
     }
@@ -108,6 +110,8 @@ public:
         return (this->sb_data <= ptr && ptr < buffer_end);
     }
 
+    file_range::metadata& get_metadata() { return this->sb_metadata; }
+
     char* get_writable_data()
     {
         if (this->take_ownership()) {
@@ -145,6 +149,8 @@ public:
 
     bool subset(shared_buffer_ref& other, off_t offset, size_t len);
 
+    void erase_ansi();
+
     bool take_ownership();
 
     void disown();
@@ -153,6 +159,7 @@ private:
     void copy_ref(const shared_buffer_ref& other);
 
     auto_mem<char*> sb_backtrace;
+    file_range::metadata sb_metadata;
     shared_buffer* sb_owner;
     const char* sb_data;
     size_t sb_length;
diff --git a/src/spectro_impls.cc b/src/spectro_impls.cc
index a12b3dee..c9f94126 100644
--- a/src/spectro_impls.cc
+++ b/src/spectro_impls.cc
@@ -301,6 +301,7 @@ log_spectro_value_source::spectro_mark(textview_curses& tc,
 
         values.clear();
         lf->read_full_message(ll, values.lvv_sbr);
+        values.lvv_sbr.erase_ansi();
         sa.clear();
         format->annotate(cl, sa, values, false);
 
diff --git a/src/textfile_sub_source.cc b/src/textfile_sub_source.cc
index 43e4d58c..6031c4f2 100644
--- a/src/textfile_sub_source.cc
+++ b/src/textfile_sub_source.cc
@@ -128,8 +128,10 @@ textfile_sub_source::text_size_for_line(textview_curses& tc,
                 lf->get_logline_observer());
             if (line < 0 || line >= lfo->lfo_filter_state.tfs_index.size()) {
             } else {
-                retval = lf->line_length(
-                    lf->begin() + lfo->lfo_filter_state.tfs_index[line]);
+                retval
+                    = lf->message_byte_length(
+                            lf->begin() + lfo->lfo_filter_state.tfs_index[line])
+                          .mlr_length;
             }
         } else {
             retval = rend_iter->second.rf_text_source->text_size_for_line(
diff --git a/test/expected/expected.am b/test/expected/expected.am
index 7a833519..4068e539 100644
--- a/test/expected/expected.am
+++ b/test/expected/expected.am
@@ -296,12 +296,16 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_logfile.sh_3fc6bfd8a6160817211f3e14fde957af75b9dbe7.out \
     $(srcdir)/%reldir%/test_logfile.sh_4a2a907fcb069b8d6e65961a7b2e796d6c3a87b1.err \
     $(srcdir)/%reldir%/test_logfile.sh_4a2a907fcb069b8d6e65961a7b2e796d6c3a87b1.out \
+    $(srcdir)/%reldir%/test_logfile.sh_6602faf7817c494c33e32da7ee95f13aa9210d01.err \
+    $(srcdir)/%reldir%/test_logfile.sh_6602faf7817c494c33e32da7ee95f13aa9210d01.out \
     $(srcdir)/%reldir%/test_logfile.sh_7c2e11488bccc59458b5775db4b90de964858259.err \
     $(srcdir)/%reldir%/test_logfile.sh_7c2e11488bccc59458b5775db4b90de964858259.out \
     $(srcdir)/%reldir%/test_logfile.sh_a7037efd0c4bbf51940137a44e57d94e9307e83e.err \
     $(srcdir)/%reldir%/test_logfile.sh_a7037efd0c4bbf51940137a44e57d94e9307e83e.out \
     $(srcdir)/%reldir%/test_logfile.sh_c18e14a26d8261c9f72747118a469266121d5459.err \
     $(srcdir)/%reldir%/test_logfile.sh_c18e14a26d8261c9f72747118a469266121d5459.out \
+    $(srcdir)/%reldir%/test_logfile.sh_e840b674cd65936a72bd64b1dac1524d16fe44c3.err \
+    $(srcdir)/%reldir%/test_logfile.sh_e840b674cd65936a72bd64b1dac1524d16fe44c3.out \
     $(srcdir)/%reldir%/test_meta.sh_154047fb52e4831aabf7d36512247bad6a6a2cf7.err \
     $(srcdir)/%reldir%/test_meta.sh_154047fb52e4831aabf7d36512247bad6a6a2cf7.out \
     $(srcdir)/%reldir%/test_meta.sh_3c9b5940f7533c5fc3d4956a6efce50a9e7132d4.err \
@@ -350,58 +354,6 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_pretty_print.sh_cd361eeca7e91bfab942b75d6c3422c7a456a111.out \
     $(srcdir)/%reldir%/test_pretty_print.sh_f8feb52a321026d9562b271eb37a2c56dfaed329.err \
     $(srcdir)/%reldir%/test_pretty_print.sh_f8feb52a321026d9562b271eb37a2c56dfaed329.out \
-    $(srcdir)/%reldir%/test_regex101.sh_0fa3663a45aca6a328cb728872af7ed7ee896f1c.err \
-    $(srcdir)/%reldir%/test_regex101.sh_0fa3663a45aca6a328cb728872af7ed7ee896f1c.out \
-    $(srcdir)/%reldir%/test_regex101.sh_182ae9244db314a953af2bee969726e381bc5a32.err \
-    $(srcdir)/%reldir%/test_regex101.sh_182ae9244db314a953af2bee969726e381bc5a32.out \
-    $(srcdir)/%reldir%/test_regex101.sh_2158f1f011ba8e1b152396c072790c076fdb8ce8.err \
-    $(srcdir)/%reldir%/test_regex101.sh_2158f1f011ba8e1b152396c072790c076fdb8ce8.out \
-    $(srcdir)/%reldir%/test_regex101.sh_281af24141680330791db7f7c5fa70833ce08a6b.err \
-    $(srcdir)/%reldir%/test_regex101.sh_281af24141680330791db7f7c5fa70833ce08a6b.out \
-    $(srcdir)/%reldir%/test_regex101.sh_35703b13990785632cca82123fb3883797959c0b.err \
-    $(srcdir)/%reldir%/test_regex101.sh_35703b13990785632cca82123fb3883797959c0b.out \
-    $(srcdir)/%reldir%/test_regex101.sh_366730cac50b4a09b7de4b84641791470b1cb9a3.err \
-    $(srcdir)/%reldir%/test_regex101.sh_366730cac50b4a09b7de4b84641791470b1cb9a3.out \
-    $(srcdir)/%reldir%/test_regex101.sh_3d18474a3e472fff6e23e0c41337ec9188fee591.err \
-    $(srcdir)/%reldir%/test_regex101.sh_3d18474a3e472fff6e23e0c41337ec9188fee591.out \
-    $(srcdir)/%reldir%/test_regex101.sh_442cc58676590a3604d5c2183f5fe0a75c98351a.err \
-    $(srcdir)/%reldir%/test_regex101.sh_442cc58676590a3604d5c2183f5fe0a75c98351a.out \
-    $(srcdir)/%reldir%/test_regex101.sh_566fd88d216a44bc1c6e23f2d6f2d0caf99d42f9.err \
-    $(srcdir)/%reldir%/test_regex101.sh_566fd88d216a44bc1c6e23f2d6f2d0caf99d42f9.out \
-    $(srcdir)/%reldir%/test_regex101.sh_5f2f7ecb6ab9cbec4b41385b91bd038906b8a7b2.err \
-    $(srcdir)/%reldir%/test_regex101.sh_5f2f7ecb6ab9cbec4b41385b91bd038906b8a7b2.out \
-    $(srcdir)/%reldir%/test_regex101.sh_629bde30483e0a6461076e9058f3a5eb81ae0425.err \
-    $(srcdir)/%reldir%/test_regex101.sh_629bde30483e0a6461076e9058f3a5eb81ae0425.out \
-    $(srcdir)/%reldir%/test_regex101.sh_630db454054cf92ec9bd0f4e3e83300047f583ff.err \
-    $(srcdir)/%reldir%/test_regex101.sh_630db454054cf92ec9bd0f4e3e83300047f583ff.out \
-    $(srcdir)/%reldir%/test_regex101.sh_771af6f3d29b8350542d5c6e98bdbf4c223cd531.err \
-    $(srcdir)/%reldir%/test_regex101.sh_771af6f3d29b8350542d5c6e98bdbf4c223cd531.out \
-    $(srcdir)/%reldir%/test_regex101.sh_7991a5b617867cf37c9f7baa85ffa425f7d455a2.err \
-    $(srcdir)/%reldir%/test_regex101.sh_7991a5b617867cf37c9f7baa85ffa425f7d455a2.out \
-    $(srcdir)/%reldir%/test_regex101.sh_79ee3f5fe71ccec97b2619d8c1f74ca97ffd2243.err \
-    $(srcdir)/%reldir%/test_regex101.sh_79ee3f5fe71ccec97b2619d8c1f74ca97ffd2243.out \
-    $(srcdir)/%reldir%/test_regex101.sh_7de76c174c58d67bf93e8f01d6d55ebb6a023f10.err \
-    $(srcdir)/%reldir%/test_regex101.sh_7de76c174c58d67bf93e8f01d6d55ebb6a023f10.out \
-    $(srcdir)/%reldir%/test_regex101.sh_8a43e6657d4f60e68d31eb8302542ca28e80d077.err \
-    $(srcdir)/%reldir%/test_regex101.sh_8a43e6657d4f60e68d31eb8302542ca28e80d077.out \
-    $(srcdir)/%reldir%/test_regex101.sh_8e93a3b6b941847c71409a297779fbb0a6666a51.err \
-    $(srcdir)/%reldir%/test_regex101.sh_8e93a3b6b941847c71409a297779fbb0a6666a51.out \
-    $(srcdir)/%reldir%/test_regex101.sh_95c56a9d146ec9a7c2196559d316f928b2ae6ae9.err \
-    $(srcdir)/%reldir%/test_regex101.sh_95c56a9d146ec9a7c2196559d316f928b2ae6ae9.out \
-    $(srcdir)/%reldir%/test_regex101.sh_9d101ee29c45cdb8c0f117ad736c9a5dd5da5839.err \
-    $(srcdir)/%reldir%/test_regex101.sh_9d101ee29c45cdb8c0f117ad736c9a5dd5da5839.out \
-    $(srcdir)/%reldir%/test_regex101.sh_c43e07df9b3068696fdc8759c7561135db981b38.err \
-    $(srcdir)/%reldir%/test_regex101.sh_c43e07df9b3068696fdc8759c7561135db981b38.out \
-    $(srcdir)/%reldir%/test_regex101.sh_cbd859487e4ea011cd6e0f0f114d70158bfd8b43.err \
-    $(srcdir)/%reldir%/test_regex101.sh_cbd859487e4ea011cd6e0f0f114d70158bfd8b43.out \
-    $(srcdir)/%reldir%/test_regex101.sh_cf6c0a9f0f04e24ce1fae7a0a434830b14447f83.err \
-    $(srcdir)/%reldir%/test_regex101.sh_cf6c0a9f0f04e24ce1fae7a0a434830b14447f83.out \
-    $(srcdir)/%reldir%/test_regex101.sh_d84597760285c3964b258726341e018f6cd49954.err \
-    $(srcdir)/%reldir%/test_regex101.sh_d84597760285c3964b258726341e018f6cd49954.out \
-    $(srcdir)/%reldir%/test_regex101.sh_f23e393dbf23d0d8e276e9b7610c7b74d79980f8.err \
-    $(srcdir)/%reldir%/test_regex101.sh_f23e393dbf23d0d8e276e9b7610c7b74d79980f8.out \
-    $(srcdir)/%reldir%/test_regex101.sh_fc41b6ee90cbf038620151f16d164b361acf82dd.err \
-    $(srcdir)/%reldir%/test_regex101.sh_fc41b6ee90cbf038620151f16d164b361acf82dd.out \
     $(srcdir)/%reldir%/test_sessions.sh_0300a1391c33b1c45ddfa90198a6bd0a5404a77f.err \
     $(srcdir)/%reldir%/test_sessions.sh_0300a1391c33b1c45ddfa90198a6bd0a5404a77f.out \
     $(srcdir)/%reldir%/test_sessions.sh_17b85654b929b2a8fc1705a170ced544783292fa.err \
diff --git a/test/expected/test_logfile.sh_6602faf7817c494c33e32da7ee95f13aa9210d01.err b/test/expected/test_logfile.sh_6602faf7817c494c33e32da7ee95f13aa9210d01.err
new file mode 100644
index 00000000..e69de29b
diff --git a/test/expected/test_logfile.sh_6602faf7817c494c33e32da7ee95f13aa9210d01.out b/test/expected/test_logfile.sh_6602faf7817c494c33e32da7ee95f13aa9210d01.out
new file mode 100644
index 00000000..372e078a
--- /dev/null
+++ b/test/expected/test_logfile.sh_6602faf7817c494c33e32da7ee95f13aa9210d01.out
@@ -0,0 +1,10 @@
+Sep 19 09:24:04 Tims-MacBook-Air AMPDeviceDiscoveryAgent[17600]: tid:1d1f - Mux ID not found in mapping dictionary
+Sep 19 09:24:04 Tims-MacBook-Air AMPDeviceDiscoveryAgent[17600]: tid:1d1f - Can't handle disconnect with invalid ecid
+Sep 19 09:24:20 Tims-MacBook-Air MobileDeviceUpdater[17530]: Entered:_AMMuxedDeviceDisconnected, mux-device:1003
+Sep 19 09:24:20 Tims-MacBook-Air AMPDeviceDiscoveryAgent[17600]: Entered:_AMMuxedDeviceDisconnected, mux-device:1003
+Sep 19 09:24:20 Tims-MacBook-Air MobileDeviceUpdater[17530]: Entered:__thr_AMMuxedDeviceDisconnected, mux-device:1003
+Sep 19 09:24:20 Tims-MacBook-Air AMPDeviceDiscoveryAgent[17600]: Entered:__thr_AMMuxedDeviceDisconnected, mux-device:1003
+Sep 19 09:24:20 Tims-MacBook-Air MobileDeviceUpdater[17530]: tid:191f - Mux ID not found in mapping dictionary
+Sep 19 09:24:20 Tims-MacBook-Air AMPDeviceDiscoveryAgent[17600]: tid:1d1f - Mux ID not found in mapping dictionary
+Sep 19 09:24:20 Tims-MacBook-Air MobileDeviceUpdater[17530]: tid:191f - Can't handle disconnect with invalid ecid
+Sep 19 09:24:20 Tims-MacBook-Air AMPDeviceDiscoveryAgent[17600]: tid:1d1f - Can't handle disconnect with invalid ecid
diff --git a/test/expected/test_logfile.sh_e840b674cd65936a72bd64b1dac1524d16fe44c3.err b/test/expected/test_logfile.sh_e840b674cd65936a72bd64b1dac1524d16fe44c3.err
new file mode 100644
index 00000000..e69de29b
diff --git a/test/expected/test_logfile.sh_e840b674cd65936a72bd64b1dac1524d16fe44c3.out b/test/expected/test_logfile.sh_e840b674cd65936a72bd64b1dac1524d16fe44c3.out
new file mode 100644
index 00000000..9de987f4
--- /dev/null
+++ b/test/expected/test_logfile.sh_e840b674cd65936a72bd64b1dac1524d16fe44c3.out
@@ -0,0 +1,11 @@
+       log_time                                 log_body                         
+2022-09-19 09:24:04.000 tid:1d1f - Mux ID not found in mapping dictionary        
+2022-09-19 09:24:04.000 tid:1d1f - Can't handle disconnect with invalid ecid     
+2022-09-19 09:24:20.000 Entered:_AMMuxedDeviceDisconnected, mux-device:1003      
+2022-09-19 09:24:20.000 Entered:_AMMuxedDeviceDisconnected, mux-device:1003      
+2022-09-19 09:24:20.000 Entered:__thr_AMMuxedDeviceDisconnected, mux-device:1003 
+2022-09-19 09:24:20.000 Entered:__thr_AMMuxedDeviceDisconnected, mux-device:1003 
+2022-09-19 09:24:20.000 tid:191f - Mux ID not found in mapping dictionary        
+2022-09-19 09:24:20.000 tid:1d1f - Mux ID not found in mapping dictionary        
+2022-09-19 09:24:20.000 tid:191f - Can't handle disconnect with invalid ecid     
+2022-09-19 09:24:20.000 tid:1d1f - Can't handle disconnect with invalid ecid     
diff --git a/test/logfile_ansi.1 b/test/logfile_ansi.1
new file mode 100644
index 00000000..51fc1bf5
--- /dev/null
+++ b/test/logfile_ansi.1
@@ -0,0 +1,10 @@
+Sep 19 09:24:04 Tims-MacBook-Air AMPDeviceDiscoveryAgent[17600]: tid:1d1f - Mux ID not found in mapping dictionary
+Sep 19 09:24:04 Tims-MacBook-Air AMPDeviceDiscoveryAgent[17600]: tid:1d1f - Can't handle disconnect with invalid ecid
+Sep 19 09:24:20 Tims-MacBook-Air MobileDeviceUpdater[17530]: Entered:_AMMuxedDeviceDisconnected, mux-device:1003
+Sep 19 09:24:20 Tims-MacBook-Air AMPDeviceDiscoveryAgent[17600]: Entered:_AMMuxedDeviceDisconnected, mux-device:1003
+Sep 19 09:24:20 Tims-MacBook-Air MobileDeviceUpdater[17530]: Entered:__thr_AMMuxedDeviceDisconnected, mux-device:1003
+Sep 19 09:24:20 Tims-MacBook-Air AMPDeviceDiscoveryAgent[17600]: Entered:__thr_AMMuxedDeviceDisconnected, mux-device:1003
+Sep 19 09:24:20 Tims-MacBook-Air MobileDeviceUpdater[17530]: tid:191f - Mux ID not found in mapping dictionary
+Sep 19 09:24:20 Tims-MacBook-Air AMPDeviceDiscoveryAgent[17600]: tid:1d1f - Mux ID not found in mapping dictionary
+Sep 19 09:24:20 Tims-MacBook-Air MobileDeviceUpdater[17530]: tid:191f - Can't handle disconnect with invalid ecid
+Sep 19 09:24:20 Tims-MacBook-Air AMPDeviceDiscoveryAgent[17600]: tid:1d1f - Can't handle disconnect with invalid ecid
diff --git a/test/test_ansi_scrubber.cc b/test/test_ansi_scrubber.cc
index 179bc4f0..584273e0 100644
--- a/test/test_ansi_scrubber.cc
+++ b/test/test_ansi_scrubber.cc
@@ -45,16 +45,31 @@ using namespace std;
 int
 main(int argc, char* argv[])
 {
+    {
+        char input[] = "Hello, \x1b[33;mWorld\x1b[0;m!";
+
+        auto new_len = erase_ansi_escapes(string_fragment::from_const(input));
+
+        assert(new_len == 13);
+    }
+
     {
         std::string boldish
             = "\u2022\b\u2022\u2023\b\u2023 h\bhe\bel\blo\bo _\ba_\bb_\bc a\b_ "
               "b";
+        auto boldish2 = boldish;
         string_attrs_t sa;
 
         sa.clear();
         scrub_ansi_string(boldish, &sa);
         printf("boldish %s\n", boldish.c_str());
         assert(boldish == "\u2022\u2023 helo abc a b");
+
+        auto new_len = erase_ansi_escapes(boldish2);
+        boldish2.resize(new_len);
+        printf("boldish2 %s\n", boldish2.c_str());
+        assert(boldish2 == "\u2022\u2023 helo abc a b");
+
         for (const auto& attr : sa) {
             printf("attr %d:%d %s\n",
                    attr.sa_range.lr_start,
diff --git a/test/test_logfile.sh b/test/test_logfile.sh
index 27ea3cf8..6c4253f1 100644
--- a/test/test_logfile.sh
+++ b/test/test_logfile.sh
@@ -685,3 +685,13 @@ EOF
 # 192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkboot.gz HTTP/1.0" 404 46210 "-" "gPXE/0.9.7"
 # 192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkernel.gz HTTP/1.0" 200 78929 "-" "gPXE/0.9.7"
 # EOF
+
+export YES_COLOR=1
+
+run_cap_test ${lnav_test} -n \
+    -c ';SELECT log_time, log_body FROM syslog_log' \
+    ${test_dir}/logfile_ansi.1
+
+run_cap_test ${lnav_test} -n \
+    -c ':switch-to-view pretty' \
+    ${test_dir}/logfile_ansi.1