[security] improve handling of file names with escape sequences

NEWS.md

@@ -1,3 +1,15 @@
+## lnav v0.12.2
+
+Bug Fixes:
+* With the recent xz backdoor shenanigans, it seems like a good
+  time to add some checks for data being hidden by escape codes:
+  - File names with escape sequences are now displayed in quotes
+    with backslash escapes.
+  - Text that has the same foreground and background colors will
+    have the background set to a contrasting color.
+* A crash during initialization on Apple Silicon and MacOS 12
+  has been fixed.
+
 ## lnav v0.12.1
 
 Features:

src/CMakeLists.txt

@@ -118,7 +118,7 @@ add_custom_command(OUTPUT time_fmts.cc COMMAND ptimec ${TIME_FORMATS} >
         time_fmts.cc)
 
 add_library(lnavdt STATIC config.h.in ptimec.hh ptimec_rt.cc time_fmts.cc)
-target_include_directories(lnavdt PUBLIC . ${CMAKE_CURRENT_BINARY_DIR})
+target_include_directories(lnavdt PUBLIC . ${CMAKE_CURRENT_BINARY_DIR} third-party/date/include)
 
 function(bin2c)
     cmake_parse_arguments(BIN2C_ "" "VARNAME" "" ${ARGN})
@@ -191,6 +191,20 @@ add_custom_command(
         DEPENDS bin2c ${FORMAT_FILES})
 list(APPEND GEN_SRCS default-formats.h default-formats.cc)
 
+set(PRQL_FILES
+        prql/stats.prql
+        prql/utils.prql)
+
+set(PRQL_FILE_PATHS ${PRQL_FILES})
+
+list(TRANSFORM PRQL_FILE_PATHS PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
+
+add_custom_command(
+        OUTPUT prql-modules.h prql-modules.cc
+        COMMAND bin2c -n lnav_prql_modules prql-modules ${PRQL_FILE_PATHS}
+        DEPENDS bin2c ${PRQL_FILES})
+list(APPEND GEN_SRCS prql-modules.h prql-modules.cc)
+
 set(CONFIG_FILES
         root-config.json
         keymaps/de-keymap.json
@@ -660,7 +674,8 @@ add_library(
 
         third-party/robin_hood/robin_hood.h
 
-        third-party/prqlc-c/prqlc.hpp
+        third-party/prqlc-c/prqlc.cxx.hh
+        third-party/prqlc-c/prqlc.cxx.cc
 )
 
 set(lnav_SRCS lnav.cc)

src/base/CMakeLists.txt

@@ -75,6 +75,7 @@ add_library(
 )
 
 target_include_directories(base PUBLIC . .. ../third-party
+        ../third-party/date/include
         ${CMAKE_CURRENT_BINARY_DIR}/..)
 target_link_libraries(base cppfmt cppscnlib pcrepp ncurses::libcurses pthread lnavdt datepp)
 

src/base/ansi_scrubber.cc

@@ -153,6 +153,7 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
             line_range bold_range;
             line_range ul_range;
             auto sub_sf = sf;
+            auto mid_sf = string_fragment();
 
             while (!sub_sf.empty()) {
                 auto lhs_opt = sub_sf.consume_codepoint();
@@ -170,7 +171,6 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
                     return;
                 }
                 auto rhs_pair = rhs_opt.value();
-                sub_sf = rhs_pair.second;
 
                 if (lhs_pair.first == '_' || rhs_pair.first == '_') {
                     if (sa != nullptr && bold_range.is_valid()) {
@@ -191,7 +191,9 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
                     ww898::utf::utf8::write(cp, [&str, &fill_index](auto ch) {
                         str[fill_index++] = ch;
                     });
-                } else {
+                } else if (lhs_pair.first == rhs_pair.first
+                           && !fmt::v10::detail::needs_escape(lhs_pair.first))
+                {
                     if (sa != nullptr && ul_range.is_valid()) {
                         shift_string_attrs(
                             *sa, ul_range.lr_start, -ul_range.length() * 2);
@@ -214,11 +216,15 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
                         log_error("invalid UTF-8 at %d", sf.sf_begin);
                         return;
                     }
+                } else {
+                    mid_sf = mid_pair.second;
+                    break;
                 }
+                sub_sf = rhs_pair.second;
             }
 
             auto output_size = fill_index - sf.sf_begin;
-            auto erased_size = sf.length() - output_size;
+            auto erased_size = sub_sf.sf_begin - fill_index;
 
             if (sa != nullptr && ul_range.is_valid()) {
                 shift_string_attrs(
@@ -234,14 +240,18 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
                                  VC_STYLE.value(text_attrs{A_BOLD}));
                 bold_range.clear();
             }
-            if (sa != nullptr) {
+            if (sa != nullptr && output_size > 0) {
                 sa->emplace_back(line_range{last_origin_offset_end,
                                             sf.sf_begin + (int) output_size},
                                  SA_ORIGIN_OFFSET.value(origin_offset));
             }
 
-            str.erase(str.begin() + fill_index, str.begin() + sf.sf_end);
-            last_origin_offset_end = sf.sf_begin + output_size;
+            str.erase(str.begin() + fill_index, str.begin() + sub_sf.sf_begin);
+            if (!mid_sf.empty()) {
+                last_origin_offset_end = mid_sf.sf_begin;
+            } else {
+                last_origin_offset_end = sf.sf_begin + output_size;
+            }
             origin_offset += erased_size;
             matcher.reload_input(str, last_origin_offset_end);
             continue;

src/base/attr_line.hh

@@ -313,15 +313,6 @@ public:
         return *this;
     }
 
-    attr_line_t& append_quoted(const attr_line_t& al)
-    {
-        this->al_string.append("\u201c");
-        this->append(al);
-        this->al_string.append("\u201d");
-
-        return *this;
-    }
-
     template<typename S>
     attr_line_t& append_quoted(S s)
     {
@@ -344,13 +335,25 @@ public:
         return *this;
     }
 
-    template<typename S>
-    attr_line_t& append(S str)
+    attr_line_t& append(const std::string& str)
     {
         this->al_string.append(str);
         return *this;
     }
 
+    attr_line_t& append(const char* str)
+    {
+        this->al_string.append(str);
+        return *this;
+    }
+
+    template<typename V>
+    attr_line_t& append(const V& v)
+    {
+        this->al_string.append(fmt::to_string(v));
+        return *this;
+    }
+
     template<typename... Args>
     attr_line_t& appendf(fmt::format_string<Args...> fstr, Args&&... args)
     {

src/base/fs_util.cc

@@ -34,6 +34,7 @@
 #include "config.h"
 #include "fmt/format.h"
 #include "itertools.hh"
+#include "lnav_log.hh"
 #include "opt_util.hh"
 
 namespace lnav {
@@ -217,3 +218,20 @@ file_lock::file_lock(const ghc::filesystem::path& archive_path)
 
 }  // namespace filesystem
 }  // namespace lnav
+
+namespace fmt {
+
+auto
+formatter<ghc::filesystem::path>::format(const ghc::filesystem::path& p,
+                                         format_context& ctx)
+    -> decltype(ctx.out()) const
+{
+    auto esc_res = fmt::v10::detail::find_escape(&(*p.native().begin()),
+                                                 &(*p.native().end()));
+    if (esc_res.end == nullptr) {
+        return formatter<string_view>::format(p.native(), ctx);
+    }
+
+    return format_to(ctx.out(), FMT_STRING("{:?}"), p.native());
+}
+}  // namespace fmt

src/base/fs_util.hh

@@ -141,4 +141,12 @@ public:
 }  // namespace filesystem
 }  // namespace lnav
 
+namespace fmt {
+template<>
+struct formatter<ghc::filesystem::path> : formatter<string_view> {
+    auto format(const ghc::filesystem::path& p, format_context& ctx)
+        -> decltype(ctx.out()) const;
+};
+}  // namespace fmt
+
 #endif

src/base/lnav.console.cc

@@ -514,15 +514,66 @@ println(FILE* file, const attr_line_t& al)
             line_style |= default_bg_style;
         }
 
+        if (line_style.has_foreground() && line_style.has_background()
+            && !line_style.get_foreground().is_rgb
+            && !line_style.get_background().is_rgb
+            && line_style.get_foreground().value.term_color
+                == line_style.get_background().value.term_color)
+        {
+            auto new_style = fmt::text_style{};
+
+            if (line_style.has_emphasis()) {
+                new_style |= line_style.get_emphasis();
+            }
+            new_style |= fmt::fg(line_style.get_foreground());
+            if (line_style.get_background().value.term_color
+                == lnav::enums::to_underlying(fmt::terminal_color::black))
+            {
+                new_style |= fmt::bg(fmt::terminal_color::white);
+            } else {
+                new_style |= fmt::bg(fmt::terminal_color::black);
+            }
+            line_style = new_style;
+        }
+
         if (href) {
             fmt::print(file, FMT_STRING("\x1b]8;;{}\x1b\\"), href.value());
         }
         if (start < str.size()) {
             auto actual_end = std::min(str.size(), static_cast<size_t>(point));
-            fmt::print(file,
-                       line_style,
-                       FMT_STRING("{}"),
-                       str.substr(start, actual_end - start));
+            auto sub = std::string{};
+
+            for (auto lpc = start; lpc < actual_end;) {
+                auto cp_start = lpc;
+                auto read_res = ww898::utf::utf8::read(
+                    [&str, &lpc]() { return str[lpc++]; });
+
+                if (read_res.isErr()) {
+                    sub.append(fmt::format(
+                        FMT_STRING("{:?}"),
+                        fmt::string_view{&str[cp_start], lpc - cp_start}));
+                    continue;
+                }
+
+                auto ch = read_res.unwrap();
+                switch (ch) {
+                    case '\b':
+                        sub.append("\u232b");
+                        break;
+                    case '\x1b':
+                        sub.append("\u238b");
+                        break;
+                    case '\x07':
+                        sub.append("\U0001F514");
+                        break;
+
+                    default:
+                        sub.append(&str[cp_start], lpc - cp_start);
+                        break;
+                }
+            }
+
+            fmt::print(file, line_style, FMT_STRING("{}"), sub);
         }
         if (href) {
             fmt::print(file, FMT_STRING("\x1b]8;;\x1b\\"));

src/base/string_util.cc

@@ -304,3 +304,19 @@ scrub_ws(const char* in, ssize_t len)
 
     return retval;
 }
+
+namespace fmt {
+auto
+formatter<lnav::tainted_string>::format(const lnav::tainted_string& ts,
+                                        format_context& ctx)
+    -> decltype(ctx.out()) const
+{
+    auto esc_res = fmt::v10::detail::find_escape(&(*ts.ts_str.begin()),
+                                                 &(*ts.ts_str.end()));
+    if (esc_res.end == nullptr) {
+        return formatter<string_view>::format(ts.ts_str, ctx);
+    }
+
+    return format_to(ctx.out(), FMT_STRING("{:?}"), ts.ts_str);
+}
+}  // namespace fmt

src/base/string_util.hh

@@ -244,4 +244,45 @@ on_blank(const std::string& str, const std::string& def)
     return str;
 }
 
+namespace lnav {
+class tainted_string {
+public:
+    explicit tainted_string(std::string s) : ts_str(std::move(s)) {}
+
+    bool operator==(const tainted_string& other) const
+    {
+        return this->ts_str == other.ts_str;
+    }
+
+    bool operator!=(const tainted_string& other) const
+    {
+        return this->ts_str != other.ts_str;
+    }
+
+    bool operator<(const tainted_string& other) const
+    {
+        return this->ts_str < other.ts_str;
+    }
+
+    bool empty() const { return this->ts_str.empty(); }
+
+    size_t length() const { return this->ts_str.length(); }
+
+    size_t size() const { return this->ts_str.size(); }
+
+    friend fmt::formatter<lnav::tainted_string>;
+
+private:
+    const std::string ts_str;
+};
+}  // namespace lnav
+
+namespace fmt {
+template<>
+struct formatter<lnav::tainted_string> : formatter<string_view> {
+    auto format(const lnav::tainted_string& ts, format_context& ctx)
+        -> decltype(ctx.out()) const;
+};
+}  // namespace fmt
+
 #endif

src/base/time_util.hh

@@ -32,7 +32,6 @@
 
 #include <chrono>
 
-#include <date/date.h>
 #include <inttypes.h>
 #include <string.h>
 #include <sys/time.h>
@@ -40,6 +39,7 @@
 #include <time.h>
 
 #include "config.h"
+#include "date/date.h"
 
 namespace lnav {
 

src/config.cmake.h.in

@@ -26,8 +26,6 @@
 
 #define HAVE_SQLITE3_DROP_MODULES
 
-#define HAVE_RUST_DEPS 1
-
 #define _XOPEN_SOURCE_EXTENDED 1
 
 #define PACKAGE_BUGREPORT "lnav@googlegroups.com"

src/document.sections.cc

@@ -51,6 +51,7 @@ hier_node::lookup_child(section_key_t key) const
             if (iter != this->hn_named_children.end()) {
                 return iter->second;
             }
+
             return nullptr;
         },
         [this](size_t index) -> hier_node* {
@@ -522,6 +523,14 @@ public:
                         }
                         file2 = file2.consume_n(2).value();
                     }
+                    this->sw_line.get_attrs().emplace_back(
+                        line_range{
+                            this->sw_range.lr_start
+                                + tokenize_res->tr_capture.c_begin,
+                            this->sw_range.lr_start
+                                + tokenize_res->tr_capture.c_begin,
+                        },
+                        VC_ROLE.value(role_t::VCR_H1));
                     if (file1 == "/dev/null" || file1 == file2) {
                         this->sw_line.get_attrs().emplace_back(
                             line_range{
@@ -541,6 +550,14 @@ public:
                     break;
                 }
                 case DT_DIFF_HUNK_HEADING: {
+                    this->sw_line.get_attrs().emplace_back(
+                        line_range{
+                            this->sw_range.lr_start
+                                + tokenize_res->tr_capture.c_begin,
+                            this->sw_range.lr_start
+                                + tokenize_res->tr_capture.c_begin,
+                        },
+                        VC_ROLE.value(role_t::VCR_H2));
                     this->sw_line.get_attrs().emplace_back(
                         line_range{
                             this->sw_range.lr_start + inner_cap.c_begin,
@@ -878,3 +895,31 @@ metadata::possibility_provider(const std::vector<section_key_t>& path)
 
 }  // namespace document
 }  // namespace lnav
+
+namespace fmt {
+auto
+formatter<lnav::document::section_key_t>::format(
+    const lnav::document::section_key_t& key, fmt::format_context& ctx)
+    -> decltype(ctx.out()) const
+{
+    return key.match(
+        [this, &ctx](const std::string& str) {
+            return formatter<string_view>::format(str, ctx);
+        },
+        [&ctx](size_t index) {
+            return format_to(ctx.out(), FMT_STRING("{}"), index);
+        });
+}
+
+auto
+formatter<std::vector<lnav::document::section_key_t>>::format(
+    const std::vector<lnav::document::section_key_t>& path,
+    fmt::format_context& ctx) -> decltype(ctx.out()) const
+{
+    for (const auto& part : path) {
+        format_to(ctx.out(), FMT_STRING("\uff1a"));
+        format_to(ctx.out(), FMT_STRING("{}"), part);
+    }
+    return ctx.out();
+}
+}  // namespace fmt

src/document.sections.hh

@@ -144,4 +144,19 @@ metadata discover_structure(attr_line_t& al,
 }  // namespace document
 }  // namespace lnav
 
+namespace fmt {
+template<>
+struct formatter<lnav::document::section_key_t> : formatter<string_view> {
+    auto format(const lnav::document::section_key_t& p, format_context& ctx)
+        -> decltype(ctx.out()) const;
+};
+
+template<>
+struct formatter<std::vector<lnav::document::section_key_t>>
+    : formatter<string_view> {
+    auto format(const std::vector<lnav::document::section_key_t>& p,
+                format_context& ctx) -> decltype(ctx.out()) const;
+};
+}  // namespace fmt
+
 #endif

src/file_collection.cc

@@ -170,8 +170,8 @@ file_collection::regenerate_unique_file_names()
     for (const auto& lf : this->fc_files) {
         const auto& path = lf->get_unique_path();
 
-        if (path.length() > this->fc_largest_path_length) {
-            this->fc_largest_path_length = path.length();
+        if (path.native().length() > this->fc_largest_path_length) {
+            this->fc_largest_path_length = path.native().length();
         }
     }
     for (const auto& pair : this->fc_other_files) {

src/files_sub_source.cc

@@ -30,6 +30,7 @@
 #include "files_sub_source.hh"
 
 #include "base/ansi_scrubber.hh"
+#include "base/fs_util.hh"
 #include "base/humanize.hh"
 #include "base/humanize.network.hh"
 #include "base/opt_util.hh"
@@ -255,7 +256,7 @@ files_sub_source::text_value_for_line(textview_curses& tc,
             auto iter = errs->begin();
             std::advance(iter, line);
             auto path = ghc::filesystem::path(iter->first);
-            auto fn = path.filename().string();
+            auto fn = fmt::to_string(path.filename());
 
             truncate_to(fn, filename_width);
             value_out = fmt::format(FMT_STRING("    {:<{}}   {}"),
@@ -272,7 +273,7 @@ files_sub_source::text_value_for_line(textview_curses& tc,
         auto iter = fc.fc_other_files.begin();
         std::advance(iter, line);
         auto path = ghc::filesystem::path(iter->first);
-        auto fn = path.string();
+        auto fn = fmt::to_string(path);
 
         truncate_to(fn, filename_width);
         value_out = fmt::format(FMT_STRING("    {:<{}}   {:14}  {}"),
@@ -286,7 +287,7 @@ files_sub_source::text_value_for_line(textview_curses& tc,
     line -= fc.fc_other_files.size();
 
     const auto& lf = fc.fc_files[line];
-    auto fn = lf->get_unique_path();
+    auto fn = fmt::to_string(ghc::filesystem::path(lf->get_unique_path()));
     char start_time[64] = "", end_time[64] = "";
     std::vector<std::string> file_notes;
 

src/line_buffer.hh

@@ -260,7 +260,7 @@ public:
 
     void enable_cache();
 
-    file_size_t get_piper_header_size() const
+    file_ssize_t get_piper_header_size() const
     {
         return this->lb_piper_header_size;
     }
@@ -342,7 +342,7 @@ private:
     safe_gz_indexed lb_gz_file; /*< File reader for gzipped files. */
     bool lb_bz_file{false}; /*< Flag set for bzip2 compressed files. */
     bool lb_line_metadata{false};
-    file_size_t lb_piper_header_size{0};
+    file_ssize_t lb_piper_header_size{0};
 
     auto_buffer lb_buffer{auto_buffer::alloc(DEFAULT_LINE_BUFFER_SIZE)};
     nonstd::optional<auto_buffer> lb_alt_buffer;

src/lnav.cc

@@ -2577,7 +2577,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
             return EXIT_FAILURE;
         }
 
-        for (auto& file_path : file_args) {
+        for (const auto& file_path : file_args) {
             if (endswith(file_path, ".git")) {
                 if (!install_from_git(file_path)) {
                     return EXIT_FAILURE;
@@ -2963,62 +2963,63 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         load_stdin = true;
     }
 
-    for (auto& file_path : file_args) {
-        scrub_ansi_string(file_path, nullptr);
-        auto file_path_without_trailer = file_path;
+    for (const auto& file_path_str : file_args) {
+        auto file_path_without_trailer = file_path_str;
         auto file_loc = file_location_t{mapbox::util::no_init{}};
         auto_mem<char> abspath;
         struct stat st;
 
-        auto colon_index = file_path.rfind(':');
+        auto colon_index = file_path_str.rfind(':');
         if (colon_index != std::string::npos) {
-            auto top_range = scn::string_view{&file_path[colon_index + 1],
-                                              &(*file_path.cend())};
+            auto top_range = scn::string_view{&file_path_str[colon_index + 1],
+                                              &(*file_path_str.cend())};
             auto scan_res = scn::scan_value<int>(top_range);
 
             if (scan_res) {
-                file_path_without_trailer = file_path.substr(0, colon_index);
+                file_path_without_trailer
+                    = file_path_str.substr(0, colon_index);
                 file_loc = vis_line_t(scan_res.value());
             } else {
                 log_warning(
                     "failed to parse line number from file path "
                     "with colon: %s",
-                    file_path.c_str());
+                    file_path_str.c_str());
             }
         }
-        auto hash_index = file_path.rfind('#');
+        auto hash_index = file_path_str.rfind('#');
         if (hash_index != std::string::npos) {
-            file_loc = file_path.substr(hash_index);
-            file_path_without_trailer = file_path.substr(0, hash_index);
-        }
-        if (stat(file_path_without_trailer.c_str(), &st) == 0) {
-            file_path = file_path_without_trailer;
+            file_loc = file_path_str.substr(hash_index);
+            file_path_without_trailer = file_path_str.substr(0, hash_index);
         }
+        auto file_path = ghc::filesystem::path(
+            stat(file_path_without_trailer.c_str(), &st) == 0
+                ? file_path_without_trailer
+                : file_path_str);
 
-        if (file_path == "-") {
+        if (file_path_str == "-") {
             load_stdin = true;
         }
 #ifdef HAVE_LIBCURL
-        else if (is_url(file_path))
+        else if (is_url(file_path_str))
         {
-            auto ul = std::make_shared<url_loader>(file_path);
+            auto ul = std::make_shared<url_loader>(file_path_str);
 
             lnav_data.ld_active_files.fc_file_names[ul->get_path()]
                 .with_filename(file_path);
             isc::to<curl_looper&, services::curl_streamer_t>().send(
                 [ul](auto& clooper) { clooper.add_request(ul); });
-        } else if (file_path.find("://") != std::string::npos) {
+        } else if (file_path_str.find("://") != std::string::npos) {
             lnav_data.ld_commands.insert(
                 lnav_data.ld_commands.begin(),
-                fmt::format(FMT_STRING(":open {}"), file_path));
+                fmt::format(FMT_STRING(":open {}"), file_path_str));
         }
 #endif
         else if (lnav::filesystem::is_glob(file_path))
         {
             lnav_data.ld_active_files.fc_file_names[file_path].with_tail(
                 !(lnav_data.ld_flags & LNF_HEADLESS));
-        } else if (stat(file_path.c_str(), &st) == -1) {
-            if (file_path.find(':') != std::string::npos) {
+        } else if (lnav::filesystem::statp(file_path, &st) == -1) {
+            if (file_path_str.find(':') != std::string::npos) {
                 lnav_data.ld_active_files.fc_file_names[file_path].with_tail(
                     !(lnav_data.ld_flags & LNF_HEADLESS));
             } else {
@@ -3041,7 +3042,8 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         } else if (S_ISFIFO(st.st_mode)) {
             auto_fd fifo_fd;
 
-            if ((fifo_fd = open(file_path.c_str(), O_RDONLY)) == -1) {
+            if ((fifo_fd = lnav::filesystem::openp(file_path, O_RDONLY)) == -1)
+            {
                 lnav::console::print(
                     stderr,
                     lnav::console::user_message::error(

src/lnav_commands.cc

@@ -385,7 +385,7 @@ com_set_file_timezone(exec_context& ec,
             const auto* tz = date::locate_zone(split_args[1]);
             auto pattern = split_args.size() == 2
                 ? line_pair->first->get_filename()
-                : split_args[2];
+                : ghc::filesystem::path(split_args[2]);
 
             if (!ec.ec_dry_run) {
                 static auto& safe_options_hier

src/log.annotate.cc

@@ -190,7 +190,7 @@ apply(vis_line_t vl, std::vector<intern_string_t> annos)
             }
         }
         root.gen("log_path");
-        root.gen(lf->get_filename());
+        root.gen(lf->get_filename().native());
         root.gen("log_format");
         root.gen(lf->get_format_name());
         root.gen("log_format_regex");

src/log.watch.cc

@@ -193,7 +193,7 @@ eval_with(logfile& lf, logfile::iterator ll)
                 sqlite3_bind_text(stmt,
                                   lpc + 1,
                                   filename.c_str(),
-                                  filename.length(),
+                                  filename.native().length(),
                                   SQLITE_STATIC);
                 continue;
             }
@@ -202,7 +202,7 @@ eval_with(logfile& lf, logfile::iterator ll)
                 sqlite3_bind_text(stmt,
                                   lpc + 1,
                                   filename.c_str(),
-                                  filename.length(),
+                                  filename.native().length(),
                                   SQLITE_STATIC);
                 continue;
             }

src/log_format.cc

@@ -1612,6 +1612,9 @@ external_log_format::annotate(uint64_t line_number,
             for (const auto& attr : this->jlf_line_attrs) {
                 if (this->jlf_cached_sub_range.contains(attr.sa_range)) {
                     sa.emplace_back(attr);
+                    sa.back().sa_range.shift(
+                        this->jlf_cached_sub_range.lr_start,
+                        -this->jlf_cached_sub_range.lr_start);
                 }
             }
         }

src/log_vtab_impl.cc

@@ -897,15 +897,19 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
                     case log_footer_columns::path: {
                         const auto& fn = lf->get_filename();
 
-                        sqlite3_result_text(
-                            ctx, fn.c_str(), fn.length(), SQLITE_STATIC);
+                        sqlite3_result_text(ctx,
+                                            fn.c_str(),
+                                            fn.native().length(),
+                                            SQLITE_STATIC);
                         break;
                     }
                     case log_footer_columns::unique_path: {
                         const auto& fn = lf->get_unique_path();
 
-                        sqlite3_result_text(
-                            ctx, fn.c_str(), fn.length(), SQLITE_STATIC);
+                        sqlite3_result_text(ctx,
+                                            fn.c_str(),
+                                            fn.native().length(),
+                                            SQLITE_STATIC);
                         break;
                     }
                     case log_footer_columns::text: {

src/logfile.cc

@@ -70,23 +70,27 @@ static const typed_json_path_container<lnav::gzip::header> file_header_handlers
 };
 
 Result<std::shared_ptr<logfile>, std::string>
-logfile::open(std::string filename, const logfile_open_options& loo, auto_fd fd)
+logfile::open(ghc::filesystem::path filename,
+              const logfile_open_options& loo,
+              auto_fd fd)
 {
     require(!filename.empty());
 
     auto lf = std::shared_ptr<logfile>(new logfile(std::move(filename), loo));
 
     memset(&lf->lf_stat, 0, sizeof(lf->lf_stat));
-    char resolved_path[PATH_MAX] = "";
+    ghc::filesystem::path resolved_path;
 
     if (!fd.has_value()) {
-        if (realpath(lf->lf_filename.c_str(), resolved_path) == nullptr) {
+        auto rp_res = lnav::filesystem::realpath(lf->lf_filename);
+        if (rp_res.isErr()) {
             return Err(fmt::format(FMT_STRING("realpath({}) failed with: {}"),
                                    lf->lf_filename,
-                                   strerror(errno)));
+                                   rp_res.unwrapErr()));
         }
 
-        if (stat(resolved_path, &lf->lf_stat) == -1) {
+        resolved_path = rp_res.unwrap();
+        if (lnav::filesystem::statp(resolved_path, &lf->lf_stat) == -1) {
             return Err(fmt::format(FMT_STRING("stat({}) failed with: {}"),
                                    lf->lf_filename,
                                    strerror(errno)));
@@ -94,15 +98,17 @@ logfile::open(std::string filename, const logfile_open_options& loo, auto_fd fd)
 
         if (!S_ISREG(lf->lf_stat.st_mode)) {
             return Err(fmt::format(FMT_STRING("{} is not a regular file"),
-                                   lf->lf_filename,
-                                   strerror(errno)));
+                                   lf->lf_filename));
         }
     }
 
     auto_fd lf_fd;
     if (fd.has_value()) {
         lf_fd = std::move(fd);
-    } else if ((lf_fd = ::open(resolved_path, O_RDONLY | O_CLOEXEC)) == -1) {
+    } else if ((lf_fd
+                = lnav::filesystem::openp(resolved_path, O_RDONLY | O_CLOEXEC))
+               == -1)
+    {
         return Err(fmt::format(FMT_STRING("open({}) failed with: {}"),
                                lf->lf_filename,
                                strerror(errno)));
@@ -166,7 +172,8 @@ logfile::open(std::string filename, const logfile_open_options& loo, auto_fd fd)
     return Ok(lf);
 }
 
-logfile::logfile(std::string filename, const logfile_open_options& loo)
+logfile::logfile(ghc::filesystem::path filename,
+                 const logfile_open_options& loo)
     : lf_filename(std::move(filename)), lf_options(loo)
 {
     this->lf_opids.writeAccess()->los_opid_ranges.reserve(64);
@@ -794,16 +801,23 @@ logfile::rebuild_index(nonstd::optional<ui_clock::time_point> deadline)
                                       = string_fragment::from_str(sbr_str)
                                             .split_lines();
                                   for (auto line_iter = lines.rbegin();
-                                       line_iter != lines.rend();
+                                       // XXX rejigger read_range() for
+                                       // multi-line reads
+                                       std::next(line_iter) != lines.rend();
                                        ++line_iter)
                                   {
                                       sbr_str.erase(line_iter->sf_begin, 22);
                                   }
                               }
+                              if (is_utf8(sbr_str).is_valid()) {
+                                  auto new_size = erase_ansi_escapes(sbr_str);
+                                  sbr_str.resize(new_size);
+                              }
                               return detect_text_format(sbr_str, path);
                           })
                           .unwrapOr(text_format_t::TF_UNKNOWN);
-                log_debug("setting text format to %d", this->lf_text_format);
+                log_debug("setting text format to %s",
+                          fmt::to_string(this->lf_text_format).c_str());
             }
             if (!li.li_utf8_scan_result.is_valid()
                 && this->lf_text_format != text_format_t::TF_MARKDOWN

src/logfile.hh

@@ -114,7 +114,7 @@ public:
      * descriptor needs to be seekable.
      */
     static Result<std::shared_ptr<logfile>, std::string> open(
-        std::string filename,
+        ghc::filesystem::path filename,
         const logfile_open_options& loo,
         auto_fd fd = auto_fd{});
 
@@ -128,7 +128,10 @@ public:
     }
 
     /** @return The filename as given in the constructor. */
-    const std::string& get_filename() const { return this->lf_filename; }
+    const ghc::filesystem::path& get_filename() const
+    {
+        return this->lf_filename;
+    }
 
     /** @return The filename as given in the constructor, excluding the path
      * prefix. */
@@ -428,11 +431,11 @@ protected:
     void set_format_base_time(log_format* lf);
 
 private:
-    logfile(std::string filename, const logfile_open_options& loo);
+    logfile(ghc::filesystem::path filename, const logfile_open_options& loo);
 
     bool file_options_have_changed();
 
-    std::string lf_filename;
+    ghc::filesystem::path lf_filename;
     logfile_open_options lf_options;
     logfile_activity lf_activity;
     bool lf_named_file{true};

src/logfile_sub_source.cc

@@ -36,6 +36,7 @@
 
 #include "base/ansi_scrubber.hh"
 #include "base/ansi_vars.hh"
+#include "base/fs_util.hh"
 #include "base/itertools.hh"
 #include "base/string_util.hh"
 #include "bookmarks.json.hh"
@@ -404,14 +405,14 @@ logfile_sub_source::text_value_for_line(textview_curses& tc,
         std::string name;
         if (this->lss_flags & F_FILENAME) {
             file_offset_end = this->lss_filename_width;
-            name = this->lss_token_file->get_filename();
+            name = fmt::to_string(this->lss_token_file->get_filename());
             if (file_offset_end < name.size()) {
                 file_offset_end = name.size();
                 this->lss_filename_width = name.size();
             }
         } else {
             file_offset_end = this->lss_basename_width;
-            name = this->lss_token_file->get_unique_path();
+            name = fmt::to_string(this->lss_token_file->get_unique_path());
             if (file_offset_end < name.size()) {
                 file_offset_end = name.size();
                 this->lss_basename_width = name.size();
@@ -992,10 +993,11 @@ logfile_sub_source::rebuild_index(
             }
             this->lss_longest_line = std::max(this->lss_longest_line,
                                               lf->get_longest_line_length());
-            this->lss_basename_width = std::max(this->lss_basename_width,
-                                                lf->get_unique_path().size());
-            this->lss_filename_width
-                = std::max(this->lss_filename_width, lf->get_filename().size());
+            this->lss_basename_width
+                = std::max(this->lss_basename_width,
+                           lf->get_unique_path().native().size());
+            this->lss_filename_width = std::max(
+                this->lss_filename_width, lf->get_filename().native().size());
         }
 
         if (full_sort) {
@@ -1721,7 +1723,7 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
             sqlite3_bind_text(stmt,
                               lpc + 1,
                               filename.c_str(),
-                              filename.length(),
+                              filename.native().length(),
                               SQLITE_STATIC);
             continue;
         }
@@ -1730,7 +1732,7 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt,
             sqlite3_bind_text(stmt,
                               lpc + 1,
                               filename.c_str(),
-                              filename.length(),
+                              filename.native().length(),
                               SQLITE_STATIC);
             continue;
         }

src/readline_possibilities.cc

@@ -32,6 +32,7 @@
 
 #include "readline_possibilities.hh"
 
+#include "base/fs_util.hh"
 #include "base/isc.hh"
 #include "base/opt_util.hh"
 #include "config.h"
@@ -425,8 +426,7 @@ add_file_possibilities()
             continue;
         }
 
-        auto escaped_fn
-            = std::regex_replace(lf->get_filename(), sh_escape, R"(\\\1)");
+        auto escaped_fn = fmt::to_string(lf->get_filename());
 
         rc->add_possibility(ln_mode_t::COMMAND, "loaded-files", escaped_fn);
 

src/session_data.cc

@@ -1509,7 +1509,7 @@ save_session_with_id(const std::string& session_id)
                 for (auto& lf : lnav_data.ld_active_files.fc_files) {
                     auto ld_opt = lnav_data.ld_log_source.find_data(lf);
 
-                    file_states.gen(lf->get_filename());
+                    file_states.gen(lf->get_filename().native());
 
                     {
                         yajlpp_map file_state(handle);

src/term_extra.hh

@@ -69,8 +69,6 @@ public:
 
     void update_title(listview_curses* lc)
     {
-        static const char* xterm_title_fmt = "\033]0;%s\007";
-
         if (!this->te_enabled) {
             return;
         }
@@ -84,12 +82,12 @@ public:
             auto line_attr_opt = get_string_attr(sa, logline::L_FILE);
             if (line_attr_opt) {
                 auto lf = line_attr_opt.value().get();
-                const std::string& filename = lf->get_unique_path();
+                const auto& filename = lf->get_unique_path();
 
                 if (filename != this->te_last_title) {
-                    std::string title = this->te_prefix + filename;
-
-                    printf(xterm_title_fmt, title.c_str());
+                    fmt::print(FMT_STRING("\033]0;{}{}\007"),
+                               this->te_prefix,
+                               filename);
                     fflush(stdout);
 
                     this->te_last_title = filename;
@@ -101,9 +99,8 @@ public:
         const std::string& view_title = lc->get_title();
 
         if (view_title != this->te_last_title) {
-            std::string title = this->te_prefix + view_title;
-
-            printf(xterm_title_fmt, title.c_str());
+            fmt::print(
+                FMT_STRING("\033]0;{}{}\007"), this->te_prefix, view_title);
             fflush(stdout);
 
             this->te_last_title = view_title;

src/textfile_sub_source.cc

@@ -1267,12 +1267,15 @@ textfile_sub_source::adjacent_anchor(vis_line_t vl, text_anchors::direction dir)
         return nonstd::nullopt;
     }
 
+    log_debug("  path for line: %s", fmt::to_string(path_for_line).c_str());
     auto last_key = path_for_line.back();
     path_for_line.pop_back();
 
     auto parent_opt = lnav::document::hier_node::lookup_path(
         md.m_sections_root.get(), path_for_line);
     if (!parent_opt) {
+        log_debug("  no parent for path: %s",
+                  fmt::to_string(path_for_line).c_str());
         return nonstd::nullopt;
     }
     auto parent = parent_opt.value();
@@ -1280,6 +1283,7 @@ textfile_sub_source::adjacent_anchor(vis_line_t vl, text_anchors::direction dir)
     auto child_hn = parent->lookup_child(last_key);
     if (!child_hn) {
         // XXX "should not happen"
+        log_debug("  child not found");
         return nonstd::nullopt;
     }
 

src/unique_path.cc

@@ -30,6 +30,7 @@
 #include "unique_path.hh"
 
 #include "config.h"
+#include "fmt/format.h"
 
 void
 unique_path_generator::add_source(
@@ -54,13 +55,14 @@ unique_path_generator::generate()
             if (pair.second.size() == 1) {
                 if (loop_count > 0) {
                     const auto src = pair.second[0];
-
-                    src->set_unique_path("[" + src->get_unique_path());
+                    auto lsquare = fmt::format(FMT_STRING("[{}"),
+                                               src->get_unique_path().native());
+                    src->set_unique_path(lsquare);
                 }
 
-                this->upg_max_len
-                    = std::max(this->upg_max_len,
-                               pair.second[0]->get_unique_path().size());
+                this->upg_max_len = std::max(
+                    this->upg_max_len,
+                    pair.second[0]->get_unique_path().native().size());
             } else {
                 bool all_common = true;
 
@@ -107,10 +109,10 @@ unique_path_generator::generate()
 
             if (loop_count == 0) {
                 src->set_unique_path(prefix.filename().string() + "]/"
-                                     + unique_path);
+                                     + unique_path.string());
             } else {
                 src->set_unique_path(prefix.filename().string() + "/"
-                                     + unique_path);
+                                     + unique_path.string());
             }
 
             const auto parent = prefix.parent_path();
@@ -118,7 +120,9 @@ unique_path_generator::generate()
             src->set_path_prefix(parent);
 
             if (parent.empty() || parent == prefix) {
-                src->set_unique_path("[" + src->get_unique_path());
+                auto lsquare = fmt::format(FMT_STRING("[{}"),
+                                           src->get_unique_path().native());
+                src->set_unique_path(lsquare);
             } else {
                 this->upg_unique_paths[src->get_unique_path()].push_back(src);
             }

src/unique_path.hh

@@ -51,11 +51,14 @@ public:
         this->ups_unique_path = path;
     }
 
-    const std::string& get_unique_path() const { return this->ups_unique_path; }
+    const ghc::filesystem::path& get_unique_path() const
+    {
+        return this->ups_unique_path;
+    }
 
     virtual ghc::filesystem::path get_path() const = 0;
 
-    ghc::filesystem::path& get_path_prefix()
+    const ghc::filesystem::path& get_path_prefix() const
     {
         return this->ups_prefix;
     }
@@ -67,7 +70,7 @@ public:
 
 private:
     ghc::filesystem::path ups_prefix;
-    std::string ups_unique_path;
+    ghc::filesystem::path ups_unique_path;
 };
 
 /**

src/view_curses.cc

@@ -196,6 +196,13 @@ view_curses::mvwattrline(WINDOW* window,
                 lpc += 1;
                 break;
 
+            case '\x07':
+                expanded_line.append("\U0001F514");
+                utf_adjustments.emplace_back(lpc, -1);
+                char_index += 1;
+                lpc += 1;
+                break;
+
             case '\r':
             case '\n':
                 expanded_line.push_back(' ');
@@ -430,11 +437,31 @@ view_curses::mvwattrline(WINDOW* window,
         short cur_fg, cur_bg;
         pair_content(cur_pair, &cur_fg, &cur_bg);
 
-        if (fg_color[lpc] >= 0
-            && fg_color[lpc] < view_colors::vc_active_palette->tc_palette.size()
-            && bg_color[lpc] == -1 && base_attrs.ta_bg_color.value_or(0) >= 0
-            && base_attrs.ta_bg_color.value_or(0)
-                < view_colors::vc_active_palette->tc_palette.size())
+        auto desired_fg = fg_color[lpc] != -1 ? fg_color[lpc] : cur_fg;
+        auto desired_bg = bg_color[lpc] != -1 ? bg_color[lpc] : cur_bg;
+        if (desired_fg == desired_bg) {
+            if (desired_bg >= 0
+                && desired_bg
+                    < view_colors::vc_active_palette->tc_palette.size())
+            {
+                auto adjusted_color
+                    = view_colors::vc_active_palette->tc_palette[desired_bg]
+                          .xc_lab_color;
+                if (adjusted_color.lc_l < 50.0) {
+                    adjusted_color.lc_l += 50.0;
+                } else {
+                    adjusted_color.lc_l -= 50.0;
+                }
+                bg_color[lpc] = view_colors::vc_active_palette->match_color(
+                    adjusted_color);
+            }
+        } else if (fg_color[lpc] >= 0
+                   && fg_color[lpc]
+                       < view_colors::vc_active_palette->tc_palette.size()
+                   && bg_color[lpc] == -1
+                   && base_attrs.ta_bg_color.value_or(0) >= 0
+                   && base_attrs.ta_bg_color.value_or(0)
+                       < view_colors::vc_active_palette->tc_palette.size())
         {
             const auto& fg_color_info
                 = view_colors::vc_active_palette->tc_palette.at(fg_color[lpc]);
@@ -446,7 +473,7 @@ view_curses::mvwattrline(WINDOW* window,
                     bg_color_info.xc_lab_color))
             {
                 auto adjusted_color = bg_color_info.xc_lab_color;
-                adjusted_color.lc_l -= 40.0;
+                adjusted_color.lc_l = std::max(0.0, adjusted_color.lc_l - 40.0);
                 auto new_bg = view_colors::vc_active_palette->match_color(
                     adjusted_color);
                 for (int lpc2 = lpc; lpc2 < line_width_chars; lpc2++) {

test/CMakeLists.txt

@@ -2,6 +2,7 @@ enable_testing()
 
 include_directories(
         . ${CMAKE_SOURCE_DIR}/src ${CMAKE_SOURCE_DIR}/src/fmtlib
+        ${CMAKE_SOURCE_DIR}/src/third-party/date/include
         ${CMAKE_CURRENT_BINARY_DIR}/../src ${CMAKE_CURRENT_BINARY_DIR})
 
 add_executable(test_abbrev test_abbrev.cc test_stubs.cc)

test/Makefile.am

@@ -460,6 +460,7 @@ dist_noinst_DATA = \
     log-samples/sample-f2fba0d0b1e57f9a707ea96a8a4efcdc.txt \
     log-samples/sample-f5afbee90a8c054061c4e9ffe673293cce7761de.txt \
     log-samples/sample-fc8923633e57bacd641d80dde3ff878212230552.txt \
+    naughty_files.py \
 	remote-log-dir/logfile_access_log.0 \
 	remote-log-dir/logfile_access_log.1 \
     tui-captures/tui_echo.0 \

test/drive_doc_discovery.cc

@@ -81,8 +81,10 @@ main(int argc, char* argv[])
                         fmt::print(FMT_STRING("{}^"),
                                    std::string(indent_diff, ' '));
                         if (iv.stop >= line_sf.sf_end + 1) {
-                            fmt::print(
-                                FMT_STRING("  [{}:{})"), iv.start, iv.stop);
+                            fmt::print(FMT_STRING("  [{}:{}) - {}"),
+                                       iv.start,
+                                       iv.stop,
+                                       iv.value);
                             return;
                         }
                         auto dot_len = iv.stop - iv.start - 1;
@@ -90,7 +92,9 @@ main(int argc, char* argv[])
                                    std::string(dot_len, '-'));
                         fmt::print(FMT_STRING("  [{}:{})"), iv.start, iv.stop);
                     });
-                fmt::print(FMT_STRING("\n"));
+                fmt::print(FMT_STRING("\nPath for line[{}:{}): "),
+                           line_sf.sf_begin,
+                           line_sf.sf_end);
                 meta.m_sections_tree.visit_overlapping(
                     line_sf.sf_begin,
                     line_sf.sf_end,
@@ -99,11 +103,11 @@ main(int argc, char* argv[])
                             fmt::fg(iv.start < line_sf.sf_begin
                                         ? fmt::terminal_color::yellow
                                         : fmt::terminal_color::green),
-                            FMT_STRING("/{}"),
+                            FMT_STRING("\uff1a{}"),
                             iv.value.match(
                                 [](const std::string& str) { return str; },
                                 [](size_t ind) {
-                                    return fmt::to_string(ind);
+                                    return fmt::format(FMT_STRING("[{}]"), ind);
                                 }));
                     });
                 fmt::print(FMT_STRING("\n"));

test/expected/expected.am

@@ -2,6 +2,8 @@
 EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.err \
     $(srcdir)/%reldir%/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out \
+    $(srcdir)/%reldir%/test_cli.sh_108c6922cde063429f76abc3fdb8a81ad9a2f671.err \
+    $(srcdir)/%reldir%/test_cli.sh_108c6922cde063429f76abc3fdb8a81ad9a2f671.out \
     $(srcdir)/%reldir%/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.err \
     $(srcdir)/%reldir%/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.out \
     $(srcdir)/%reldir%/test_cli.sh_17a68b798354f9a6cdfab372006caeb74038d15c.err \
@@ -16,6 +18,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_cli.sh_76aa57821598962e59063a40c20171040c95a731.out \
     $(srcdir)/%reldir%/test_cli.sh_97e19b9ff3775d84074455a2e8993a0611b1c269.err \
     $(srcdir)/%reldir%/test_cli.sh_97e19b9ff3775d84074455a2e8993a0611b1c269.out \
+    $(srcdir)/%reldir%/test_cli.sh_af3ace7762b4cc150fcdcac86083b379bded7b32.err \
+    $(srcdir)/%reldir%/test_cli.sh_af3ace7762b4cc150fcdcac86083b379bded7b32.out \
     $(srcdir)/%reldir%/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.err \
     $(srcdir)/%reldir%/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.out \
     $(srcdir)/%reldir%/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.err \
@@ -1250,6 +1254,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_sql_xml_func.sh_fefeb387ae14d4171225ea06cbbff3ec43990cf0.out \
     $(srcdir)/%reldir%/test_sql_yaml_func.sh_dc189d02e8979b7ed245d5d750f68b9965984699.err \
     $(srcdir)/%reldir%/test_sql_yaml_func.sh_dc189d02e8979b7ed245d5d750f68b9965984699.out \
+    $(srcdir)/%reldir%/test_text_file.sh_02a0514e0e384e5511ae202ea519552ba04030ed.err \
+    $(srcdir)/%reldir%/test_text_file.sh_02a0514e0e384e5511ae202ea519552ba04030ed.out \
     $(srcdir)/%reldir%/test_text_file.sh_0bba304f34ae07c4fa9e91e0b42f5fe98654a6a8.err \
     $(srcdir)/%reldir%/test_text_file.sh_0bba304f34ae07c4fa9e91e0b42f5fe98654a6a8.out \
     $(srcdir)/%reldir%/test_text_file.sh_11fd274911e45a743b4de616888a64183d07cb76.err \
@@ -1288,6 +1294,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_text_file.sh_8a4954af3e536b3789b1fd5b33519e9d444cc933.out \
     $(srcdir)/%reldir%/test_text_file.sh_8b2cd055e6a1db2ed9b2af2a917f8556395fa653.err \
     $(srcdir)/%reldir%/test_text_file.sh_8b2cd055e6a1db2ed9b2af2a917f8556395fa653.out \
+    $(srcdir)/%reldir%/test_text_file.sh_a87c18c643994c3dbbbbb619a06a601d3668ea71.err \
+    $(srcdir)/%reldir%/test_text_file.sh_a87c18c643994c3dbbbbb619a06a601d3668ea71.out \
     $(srcdir)/%reldir%/test_text_file.sh_ac486314c4e02e480d829ea2f077b86c49fedcec.err \
     $(srcdir)/%reldir%/test_text_file.sh_ac486314c4e02e480d829ea2f077b86c49fedcec.out \
     $(srcdir)/%reldir%/test_text_file.sh_ac872aadda29b9a824361a2c711d62ec1c75d40f.err \

test/expected/test_cli.sh_108c6922cde063429f76abc3fdb8a81ad9a2f671.out

@@ -0,0 +1 @@
+hi

test/expected/test_cli.sh_af3ace7762b4cc150fcdcac86083b379bded7b32.out

@@ -0,0 +1,9 @@
+[
+    {
+        "filepath": "[0] echo hi",
+        "descriptor": "org.lnav.piper.header",
+        "mimetype": "application/json",
+        "ctime": "2013-06-06T19:13:20.000",
+        "cwd": "{test_dir}"
+    }
+]

test/expected/test_json_format.sh_469f005b0708d629bc95f0c48a5e390f440c1fef.out

@@ -18,12 +18,12 @@
 
 [2013-09-06T22:01:49.124] ⋮ 1 beat per second
 
-[2013-09-06T22:01:49.124] ⋮ not looking good
+[2013-09-06T22:01:49.124] ⋮ not looking good
 
-[2013-09-06T22:01:49.124] ⋮ looking bad
+[2013-09-06T22:01:49.124] ⋮ looking bad
 
-[2013-09-06T22:01:49.124] ⋮ sooo bad
+[2013-09-06T22:01:49.124] ⋮ sooo bad
 
-[2013-09-06T22:01:49.124] ⋮ shoot
+[2013-09-06T22:01:49.124] ⋮ shoot
   obj: { "field1" : "hi", "field2": 2 }
   arr: ["hi", {"sub1": true}]

test/expected/test_text_file.sh_02a0514e0e384e5511ae202ea519552ba04030ed.out

@@ -0,0 +1,3 @@
+Hello, World!!
+Goodbye, World!!
+That is not⌫⌫⌫all

test/expected/test_text_file.sh_a87c18c643994c3dbbbbb619a06a601d3668ea71.out

@@ -0,0 +1,2 @@
+time for a reset ⎋ckapow
+ding dong! 🔔

test/naughty_files.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+import os
+
+os.makedirs("naughty", 0o700, True)
+with open('naughty/file-with-escape-sequences-\x1b[31mred\x1b[m-color.txt', 'w+') as fi:
+    fi.write('boo')
+
+with open('naughty/file-with-escape-sequences-\x1b[31mred\x1b[m-color.log', 'w+') as fi:
+    fi.write('2012-07-02 10:22:40,672:DEBUG:foo bar baz\n')
+
+try:
+    with open(b'naughty/text-file-with-invalid-utf-\xc3\x28', 'w+') as fi:
+        fi.write('boo')
+
+    with open(b'naughty/log-file-with-invalid-utf-\xc3\x28', 'w+') as fi:
+        fi.write('2015-04-24T21:09:39.296 25376]ERROR:somemodule:Something very INFOrmative.')
+except OSError as e:
+    pass
+
+with open('naughty/file-with-hidden-text.txt', 'w+') as fi:
+    fi.write('Hello, \x1b[30;40mWorld!\x1b[m!\n')
+    fi.write('Goodbye, \x1b[37;47mWorld!\x1b[m!\n')
+    fi.write('That is not\b\b\ball\n')
+
+with open('naughty/file-with-terminal-controls.txt', 'w+') as fi:
+    fi.write('time for a reset \x1bckapow\n')
+    fi.write('ding dong! \x07\n')

test/test_ansi_scrubber.cc

@@ -45,6 +45,26 @@ using namespace std;
 int
 main(int argc, char* argv[])
 {
+    printf("BEGIN test\n");
+    {
+        std::string bad_bold = "That is not\b\b\ball\n";
+        string_attrs_t sa;
+
+        scrub_ansi_string(bad_bold, &sa);
+        printf("bad bold1: '%s'\n",
+               fmt::format(FMT_STRING("{:?}"), bad_bold).c_str());
+        assert(bad_bold == "That is not\b\b\ball\n");
+    }
+    {
+        std::string bad_bold = "test r\bra\bc not\b\b\ball \x16";
+        string_attrs_t sa;
+
+        scrub_ansi_string(bad_bold, &sa);
+        printf("bad bold2: '%s'\n",
+               fmt::format(FMT_STRING("{:?}"), bad_bold).c_str());
+        assert(bad_bold == "test ra\bc not\b\b\ball ");
+    }
+
     {
         char input[] = "Hello, \x1b[33;mWorld\x1b[0;m!";
 

test/test_cli.sh

@@ -52,7 +52,7 @@ run_cap_test ${lnav_test} -n -e 'echo hi'
 
 run_cap_test ${lnav_test} -m piper list
 
-PIPER_URL=$(${lnav_test} -m -q piper list | tail -1 | sed -r -e 's;.*(piper://[^ ]+).*;\1;g')
+PIPER_URL=$(env NO_COLOR=1 ${lnav_test} -m -q piper list | tail -1 | sed -r -e 's;.*(piper://[^ ]+).*;\1;g')
 
 run_cap_test ${lnav_test} -n $PIPER_URL
 

test/test_text_file.sh

@@ -117,3 +117,8 @@ echo "Hello, World!" | run_cap_test \
 echo "Hello, World!" | run_cap_test \
     env TEST_COMMENT="piper crumbs" TZ=America/Los_Angeles \
     ${lnav_test} -nt
+
+${test_dir}/naughty_files.py
+run_cap_test ${lnav_test} -n naughty/file-with-hidden-text.txt
+
+run_cap_test ${lnav_test} -n naughty/file-with-terminal-controls.txt