[line_buffer] cache compressed files

src/CMakeLists.txt

@@ -409,6 +409,7 @@ add_library(
         spectro_source.cc
         sql_commands.cc
         sql_util.cc
+        sqlitepp.cc
         state-extension-functions.cc
         styling.cc
         text_format.cc

src/Makefile.am

@@ -437,6 +437,7 @@ libdiag_a_SOURCES = \
 	shlex.cc \
 	spectro_impls.cc \
 	spectro_source.cc \
+	sqlitepp.cc \
 	sqlite-extension-func.cc \
 	statusview_curses.cc \
 	string-extension-functions.cc \

src/archive_manager.cc

@@ -54,50 +54,6 @@ namespace fs = ghc::filesystem;
 
 namespace archive_manager {
 
-class archive_lock {
-public:
-    class guard {
-    public:
-        explicit guard(archive_lock& arc_lock) : g_lock(arc_lock)
-        {
-            this->g_lock.lock();
-        };
-
-        ~guard()
-        {
-            this->g_lock.unlock();
-        };
-
-    private:
-        archive_lock& g_lock;
-    };
-
-    void lock() const
-    {
-        lockf(this->lh_fd, F_LOCK, 0);
-    };
-
-    void unlock() const
-    {
-        lockf(this->lh_fd, F_ULOCK, 0);
-    };
-
-    explicit archive_lock(const fs::path& archive_path)
-    {
-        auto lock_path = archive_path;
-
-        lock_path += ".lck";
-        auto open_res = lnav::filesystem::create_file(lock_path, O_RDWR, 0600);
-        if (open_res.isErr()) {
-            throw std::runtime_error(open_res.unwrapErr());
-        }
-        this->lh_fd = open_res.unwrap();
-        this->lh_fd.close_on_exec();
-    };
-
-    auto_fd lh_fd;
-};
-
 #if HAVE_ARCHIVE_H
 /**
  * Enables a subset of the supported archive formats to speed up detection,
@@ -275,8 +231,8 @@ extract(const std::string& filename, const extract_cb& cb)
                                ec.message()));
     }
 
-    auto arc_lock = archive_lock(tmp_path);
-    auto lock_guard = archive_lock::guard(arc_lock);
+    auto arc_lock = lnav::filesystem::file_lock(tmp_path);
+    auto lock_guard = lnav::filesystem::file_lock::guard(arc_lock);
     auto done_path = tmp_path;
 
     done_path += ".done";

src/base/fs_util.hh

@@ -78,6 +78,41 @@ Result<void, std::string> write_file(const ghc::filesystem::path& path,
 
 std::string build_path(const std::vector<ghc::filesystem::path>& paths);
 
+class file_lock {
+public:
+    class guard {
+    public:
+        explicit guard(file_lock& arc_lock) : g_lock(arc_lock)
+        {
+            this->g_lock.lock();
+        };
+
+        ~guard() { this->g_lock.unlock(); };
+
+    private:
+        file_lock& g_lock;
+    };
+
+    void lock() const { lockf(this->lh_fd, F_LOCK, 0); }
+
+    void unlock() const { lockf(this->lh_fd, F_ULOCK, 0); }
+
+    explicit file_lock(const ghc::filesystem::path& archive_path)
+    {
+        auto lock_path = archive_path;
+
+        lock_path += ".lck";
+        auto open_res = lnav::filesystem::create_file(
+            lock_path, O_RDWR | O_CLOEXEC, 0600);
+        if (open_res.isErr()) {
+            throw std::runtime_error(open_res.unwrapErr());
+        }
+        this->lh_fd = open_res.unwrap();
+    }
+
+    auto_fd lh_fd;
+};
+
 }  // namespace filesystem
 }  // namespace lnav
 

src/base/snippet_highlighters.cc

@@ -39,6 +39,10 @@ namespace snippets {
 static bool
 is_bracket(const std::string& str, int index, bool is_lit)
 {
+    if (index == 0) {
+        return true;
+    }
+
     if (is_lit && str[index - 1] == '\\') {
         return true;
     }

src/line_buffer.cc

@@ -49,13 +49,18 @@
 #endif
 
 #include "base/auto_pid.hh"
+#include "base/fs_util.hh"
 #include "base/injector.bind.hh"
 #include "base/injector.hh"
 #include "base/is_utf8.hh"
 #include "base/isc.hh"
 #include "base/math_util.hh"
+#include "base/paths.hh"
 #include "fmtlib/fmt/format.h"
 #include "line_buffer.hh"
+#include "lnav_util.hh"
+
+using namespace std::chrono_literals;
 
 static const ssize_t INITIAL_REQUEST_SIZE = 16 * 1024;
 static const ssize_t DEFAULT_INCREMENT = 128 * 1024;
@@ -438,7 +443,7 @@ line_buffer::ensure_available(file_off_t start, ssize_t max_length)
         if ((this->lb_file_size != (ssize_t) -1)
             && (start + this->lb_buffer.capacity() > this->lb_file_size))
         {
-            require(start < this->lb_file_size);
+            require(start <= this->lb_file_size);
             /*
              * If the start is near the end of the file, move the offset back a
              * bit so we can get more of the file in the cache.
@@ -496,7 +501,7 @@ line_buffer::load_next_buffer()
 
     // log_debug("BEGIN preload read");
     /* ... read in the new data. */
-    if (*gi) {
+    if (!this->lb_cached_fd && *gi) {
         if (this->lb_file_size != (ssize_t) -1 && this->in_range(start)
             && this->in_range(this->lb_file_size - 1))
         {
@@ -523,7 +528,7 @@ line_buffer::load_next_buffer()
         }
     }
 #ifdef HAVE_BZLIB_H
-    else if (this->lb_bz_file)
+    else if (!this->lb_cached_fd && this->lb_bz_file)
     {
         if (this->lb_file_size != (ssize_t) -1
             && (((ssize_t) start >= this->lb_file_size)
@@ -547,7 +552,7 @@ line_buffer::load_next_buffer()
                 close(bzfd);
                 throw error(errno);
             }
-            if ((bz_file = BZ2_bzdopen(bzfd, "r")) == NULL) {
+            if ((bz_file = BZ2_bzdopen(bzfd, "r")) == nullptr) {
                 close(bzfd);
                 if (errno == 0) {
                     throw std::bad_alloc();
@@ -571,8 +576,10 @@ line_buffer::load_next_buffer()
             this->lb_compressed_offset = lseek(bzfd, 0, SEEK_SET);
             BZ2_bzclose(bz_file);
 
-            if (rc != -1 && (rc < (this->lb_alt_buffer.value().available())) &&
-                (start + this->lb_alt_buffer.value().size() + rc > this->lb_file_size)) {
+            if (rc != -1 && (rc < (this->lb_alt_buffer.value().available()))
+                && (start + this->lb_alt_buffer.value().size() + rc
+                    > this->lb_file_size))
+            {
                 this->lb_file_size
                     = (start + this->lb_alt_buffer.value().size() + rc);
             }
@@ -581,7 +588,8 @@ line_buffer::load_next_buffer()
 #endif
     else
     {
-        rc = pread(this->lb_fd,
+        rc = pread(this->lb_cached_fd ? this->lb_cached_fd.value().get()
+                                      : this->lb_fd.get(),
                    this->lb_alt_buffer.value().end(),
                    this->lb_alt_buffer.value().available(),
                    start + this->lb_alt_buffer.value().size());
@@ -763,7 +771,7 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length)
         safe::WriteAccess<safe_gz_indexed> gi(this->lb_gz_file);
 
         /* ... read in the new data. */
-        if (*gi) {
+        if (!this->lb_cached_fd && *gi) {
             // log_debug("old decomp start");
             if (this->lb_file_size != (ssize_t) -1 && this->in_range(start)
                 && this->in_range(this->lb_file_size - 1))
@@ -771,7 +779,7 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length)
                 rc = 0;
             } else {
                 this->lb_stats.s_decompressions += 1;
-                if (this->lb_last_line_offset > 0) {
+                if (false && this->lb_last_line_offset > 0) {
                     this->lb_stats.s_hist[(this->lb_file_offset * 10)
                                           / this->lb_last_line_offset]
                         += 1;
@@ -793,7 +801,7 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length)
 #endif
         }
 #ifdef HAVE_BZLIB_H
-        else if (this->lb_bz_file)
+        else if (!this->lb_cached_fd && this->lb_bz_file)
         {
             if (this->lb_file_size != (ssize_t) -1
                 && (((ssize_t) start >= this->lb_file_size)
@@ -852,7 +860,7 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length)
         else if (this->lb_seekable)
         {
             this->lb_stats.s_preads += 1;
-            if (this->lb_last_line_offset > 0) {
+            if (false && this->lb_last_line_offset > 0) {
                 this->lb_stats.s_hist[(this->lb_file_offset * 10)
                                       / this->lb_last_line_offset]
                     += 1;
@@ -862,7 +870,8 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length)
                       this->lb_fd.get(),
                       this->lb_file_offset + this->lb_buffer.size());
 #endif
-            rc = pread(this->lb_fd,
+            rc = pread(this->lb_cached_fd ? this->lb_cached_fd.value().get()
+                                          : this->lb_fd.get(),
                        this->lb_buffer.end(),
                        this->lb_buffer.available(),
                        this->lb_file_offset + this->lb_buffer.size());
@@ -993,7 +1002,7 @@ line_buffer::load_next_line(file_range prev_line)
     }
     while (!done) {
         auto old_retval_size = retval.li_file_range.fr_size;
-        char *line_start, *lf;
+        const char *line_start, *lf;
 
         /* Find the data in the cache and */
         line_start = this->get_range(offset, retval.li_file_range.fr_size);
@@ -1059,7 +1068,8 @@ line_buffer::load_next_line(file_range prev_line)
                 && (!this->is_pipe() || request_size > DEFAULT_INCREMENT)))
         {
             if ((lf != nullptr)
-                && ((size_t) (lf - line_start) >= MAX_LINE_BUFFER_SIZE - 1)) {
+                && ((size_t) (lf - line_start) >= MAX_LINE_BUFFER_SIZE - 1))
+            {
                 lf = nullptr;
             }
             if (lf != nullptr) {
@@ -1134,11 +1144,12 @@ Result<shared_buffer_ref, std::string>
 line_buffer::read_range(const file_range fr)
 {
     shared_buffer_ref retval;
-    char* line_start;
+    const char* line_start;
     file_ssize_t avail;
 
     if (this->lb_last_line_offset != -1
-        && fr.fr_offset > this->lb_last_line_offset) {
+        && fr.fr_offset > this->lb_last_line_offset)
+    {
         /*
          * Don't return anything past the last known line.  The caller needs
          * to try reading at the offset of the last line again.
@@ -1232,3 +1243,126 @@ line_buffer::quiesce()
         this->lb_loader_future.wait();
     }
 }
+
+static ghc::filesystem::path
+line_buffer_cache_path()
+{
+    return lnav::paths::workdir() / "buffer-cache";
+}
+
+void
+line_buffer::enable_cache()
+{
+    if (!this->lb_compressed || this->lb_cached_fd) {
+        log_info("%d: skipping cache request (compressed=%d already-cached=%d)",
+                 this->lb_fd.get(),
+                 this->lb_compressed,
+                 (bool) this->lb_cached_fd);
+        return;
+    }
+
+    struct stat st;
+
+    if (fstat(this->lb_fd, &st) == -1) {
+        log_error("failed to fstat(%d) - %d", this->lb_fd.get(), errno);
+        return;
+    }
+
+    auto cached_base_name = hasher()
+                                .update(st.st_dev)
+                                .update(st.st_ino)
+                                .update(st.st_size)
+                                .to_string();
+    auto cache_dir = line_buffer_cache_path() / cached_base_name.substr(0, 2);
+
+    ghc::filesystem::create_directories(cache_dir);
+
+    auto cached_file_name = fmt::format(FMT_STRING("{}.bin"), cached_base_name);
+    auto cached_file_path = cache_dir / cached_file_name;
+    auto cached_done_path
+        = cache_dir / fmt::format(FMT_STRING("{}.done"), cached_base_name);
+
+    log_info(
+        "%d:cache file path: %s", this->lb_fd.get(), cached_file_path.c_str());
+
+    auto fl = lnav::filesystem::file_lock(cached_file_path);
+    auto guard = lnav::filesystem::file_lock::guard(fl);
+
+    if (ghc::filesystem::exists(cached_done_path)) {
+        log_info("%d:using existing cache file");
+        auto open_res = lnav::filesystem::open_file(cached_file_path, O_RDWR);
+        if (open_res.isOk()) {
+            this->lb_cached_fd = open_res.unwrap();
+            return;
+        }
+        ghc::filesystem::remove(cached_done_path);
+    }
+
+    auto create_res = lnav::filesystem::create_file(
+        cached_file_path, O_RDWR | O_TRUNC, 0600);
+    if (create_res.isErr()) {
+        log_error("failed to create cache file: %s -- %s",
+                  cached_file_path.c_str(),
+                  create_res.unwrapErr().c_str());
+        return;
+    }
+
+    auto write_fd = create_res.unwrap();
+    auto done = false;
+
+    static const ssize_t FILL_LENGTH = 1024 * 1024;
+    auto off = file_off_t{0};
+    while (!done) {
+        log_debug("%d: caching file content at %d", this->lb_fd.get(), off);
+        if (!this->fill_range(off, FILL_LENGTH)) {
+            log_debug("%d: caching finished", this->lb_fd.get());
+            done = true;
+        } else {
+            file_ssize_t avail;
+
+            const auto* data = this->get_range(off, avail);
+            auto rc = write(write_fd, data, avail);
+            if (rc != avail) {
+                log_error("%d: short write!", this->lb_fd.get());
+                return;
+            }
+
+            off += avail;
+        }
+    }
+
+    lnav::filesystem::create_file(cached_done_path, O_WRONLY, 0600);
+
+    this->lb_cached_fd = std::move(write_fd);
+}
+
+void
+line_buffer::cleanup_cache()
+{
+    (void) std::async(std::launch::async, []() {
+        auto now = std::chrono::system_clock::now();
+        auto cache_path = line_buffer_cache_path();
+        std::vector<ghc::filesystem::path> to_remove;
+
+        for (const auto& cache_subdir :
+             ghc::filesystem::directory_iterator(cache_path))
+        {
+            for (const auto& entry :
+                 ghc::filesystem::directory_iterator(cache_subdir))
+            {
+                auto mtime = ghc::filesystem::last_write_time(entry.path());
+                auto exp_time = mtime + 1h;
+                if (now < exp_time) {
+                    continue;
+                }
+
+                to_remove.emplace_back(entry.path());
+            }
+        }
+
+        for (auto& entry : to_remove) {
+            log_debug("removing compressed file cache: %s", entry.c_str());
+            ghc::filesystem::remove_all(entry);
+        }
+    });
+}

src/line_buffer.hh

@@ -92,15 +92,9 @@ public:
     public:
         gz_indexed();
         gz_indexed(gz_indexed&& other) = default;
-        ~gz_indexed()
-        {
-            this->close();
-        }
+        ~gz_indexed() { this->close(); }
 
-        inline operator bool() const
-        {
-            return this->gz_fd != -1;
-        }
+        inline operator bool() const { return this->gz_fd != -1; }
 
         uLong get_source_offset() const
         {
@@ -150,38 +144,23 @@ public:
     void set_fd(auto_fd& fd);
 
     /** @return The file descriptor that data should be pulled from. */
-    int get_fd() const
-    {
-        return this->lb_fd;
-    }
+    int get_fd() const { return this->lb_fd; }
 
-    time_t get_file_time() const
-    {
-        return this->lb_file_time;
-    }
+    time_t get_file_time() const { return this->lb_file_time; }
 
     /**
      * @return The size of the file or the amount of data pulled from a pipe.
      */
-    file_ssize_t get_file_size() const
-    {
-        return this->lb_file_size;
-    }
+    file_ssize_t get_file_size() const { return this->lb_file_size; }
 
-    bool is_pipe() const
-    {
-        return !this->lb_seekable;
-    }
+    bool is_pipe() const { return !this->lb_seekable; }
 
     bool is_pipe_closed() const
     {
         return !this->lb_seekable && (this->lb_file_size != -1);
     }
 
-    bool is_compressed() const
-    {
-        return this->lb_compressed;
-    }
+    bool is_compressed() const { return this->lb_compressed; }
 
     file_off_t get_read_offset(file_off_t off) const
     {
@@ -259,15 +238,13 @@ public:
         std::array<uint32_t, 10> s_hist{};
     };
 
-    struct stats consume_stats()
-    {
-        return std::exchange(this->lb_stats, {});
-    }
+    struct stats consume_stats() { return std::exchange(this->lb_stats, {}); }
 
-    size_t get_buffer_size() const
-    {
-        return this->lb_buffer.size();
-    }
+    size_t get_buffer_size() const { return this->lb_buffer.size(); }
+
+    void enable_cache();
+
+    static void cleanup_cache();
 
 private:
     /**
@@ -316,15 +293,14 @@ private:
      * @return A pointer to the start of the cached data in the internal
      * buffer.
      */
-    char* get_range(file_off_t start, file_ssize_t& avail_out)
+    const char* get_range(file_off_t start, file_ssize_t& avail_out) const
     {
         auto buffer_offset = start - this->lb_file_offset;
-        char* retval;
 
         require(buffer_offset >= 0);
         require(this->lb_buffer.size() >= buffer_offset);
 
-        retval = this->lb_buffer.at(buffer_offset);
+        const auto* retval = this->lb_buffer.at(buffer_offset);
         avail_out = this->lb_buffer.size() - buffer_offset;
 
         return retval;
@@ -367,6 +343,8 @@ private:
     std::vector<uint32_t> lb_line_starts;
     std::vector<bool> lb_line_is_utf;
     stats lb_stats;
+
+    nonstd::optional<auto_fd> lb_cached_fd;
 };
 
 #endif

src/lnav.cc

@@ -1167,8 +1167,11 @@ looper()
         rlc->set_display_next_action(rl_display_next);
         rlc->set_blur_action(rl_blur);
         rlc->set_completion_request_action(rl_completion_request);
-        rlc->set_alt_value(HELP_MSG_2(
-            e, E, "to move forward/backward through error messages"));
+        rlc->set_alt_value(
+            HELP_MSG_2(e,
+                       E,
+                       "to move forward/backward through " ANSI_COLOR(
+                           COLOR_RED) "error" ANSI_NORM " messages"));
 
         (void) curs_set(0);
 
@@ -1775,6 +1778,7 @@ UPDATE lnav_views_echo
                     }
 
                     if (!ran_cleanup) {
+                        line_buffer::cleanup_cache();
                         archive_manager::cleanup_cache();
                         tailer::cleanup_cache();
                         ran_cleanup = true;
@@ -2949,6 +2953,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                 execute_init_commands(lnav_data.ld_exec_context, cmd_results);
                 archive_manager::cleanup_cache();
                 tailer::cleanup_cache();
+                line_buffer::cleanup_cache();
                 wait_for_pipers();
                 isc::to<curl_looper&, services::curl_streamer_t>()
                     .send_and_wait(

src/log_format.cc

@@ -2166,10 +2166,12 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
                 max_name_width = std::max(max_name_width, pat.p_name.size());
             }
             for (const auto& line_frag : sample_lines) {
-                auto src_line = line_frag.to_string();
-                if (!endswith(src_line, "\n")) {
+                auto src_line = attr_line_t(line_frag.to_string());
+                if (!line_frag.endswith("\n")) {
                     src_line.append("\n");
                 }
+                src_line.with_attr_for_all(
+                    VC_ROLE.value(role_t::VCR_QUOTED_CODE));
                 notes.append("   ").append(src_line);
                 for (auto& part_pair : partial_indexes) {
                     if (part_pair.first >= 0

src/log_vtab_impl.cc

@@ -394,6 +394,16 @@ vt_open(sqlite3_vtab* p_svt, sqlite3_vtab_cursor** pp_cursor)
     p_cur->log_cursor.lc_end_line = vis_line_t(p_vt->lss->text_line_count());
     p_cur->log_cursor.lc_sub_index = 0;
 
+    for (auto& ld : *p_vt->lss) {
+        auto *lf = ld->get_file_ptr();
+
+        if (lf == nullptr) {
+            continue;
+        }
+
+        lf->enable_cache();
+    }
+
     return SQLITE_OK;
 }
 

src/logfile.hh

@@ -373,6 +373,8 @@ public:
 
     void quiesce() { this->lf_line_buffer.quiesce(); }
 
+    void enable_cache() { this->lf_line_buffer.enable_cache(); }
+
     void dump_stats();
 
 protected:

src/logfile_sub_source.cc

@@ -212,13 +212,7 @@ logfile_sub_source::text_value_for_line(textview_curses& tc,
     sbr.share(this->lss_share_manager,
               (char*) this->lss_token_value.c_str(),
               this->lss_token_value.size());
-    if (this->lss_token_line->is_continued()) {
-        this->lss_token_attrs.emplace_back(
-            line_range{0, (int) this->lss_token_value.length()},
-            SA_BODY.value());
-    } else {
-        format->annotate(line, this->lss_token_attrs, this->lss_token_values);
-    }
+    format->annotate(line, this->lss_token_attrs, this->lss_token_values);
     if (this->lss_token_line->get_sub_offset() != 0) {
         this->lss_token_attrs.clear();
     }

src/shared_buffer.cc

@@ -42,7 +42,7 @@
 static const bool DEBUG_TRACE = false;
 
 void
-shared_buffer_ref::share(shared_buffer& sb, char* data, size_t len)
+shared_buffer_ref::share(shared_buffer& sb, const char* data, size_t len)
 {
 #ifdef HAVE_EXECINFO_H
     if (DEBUG_TRACE) {
@@ -77,7 +77,8 @@ shared_buffer_ref::subset(shared_buffer_ref& other, off_t offset, size_t len)
                 return false;
             }
 
-            memcpy(this->sb_data, &other.sb_data[offset], len);
+            memcpy(
+                const_cast<char*>(this->sb_data), &other.sb_data[offset], len);
         } else {
             this->sb_owner->add_ref(*this);
             this->sb_data = &other.sb_data[offset];
@@ -134,7 +135,7 @@ shared_buffer_ref::disown()
 {
     if (this->sb_owner == nullptr) {
         if (this->sb_data != nullptr) {
-            free(this->sb_data);
+            free(const_cast<char*>(this->sb_data));
         }
     } else {
         this->sb_owner->sb_refs.erase(find(this->sb_owner->sb_refs.begin(),
@@ -158,7 +159,8 @@ shared_buffer_ref::copy_ref(const shared_buffer_ref& other)
     } else {
         this->sb_owner = nullptr;
         this->sb_data = (char*) malloc(other.sb_length);
-        memcpy(this->sb_data, other.sb_data, other.sb_length);
+        memcpy(
+            const_cast<char*>(this->sb_data), other.sb_data, other.sb_length);
         this->sb_length = other.sb_length;
     }
 }

src/shared_buffer.hh

@@ -111,7 +111,7 @@ public:
     char* get_writable_data()
     {
         if (this->take_ownership()) {
-            return this->sb_data;
+            return const_cast<char*>(this->sb_data);
         }
 
         return nullptr;
@@ -136,12 +136,12 @@ public:
         };
     }
 
-    using narrow_result = std::pair<char*, size_t>;
+    using narrow_result = std::pair<const char*, size_t>;
     narrow_result narrow(size_t new_data, size_t new_length);
 
     void widen(narrow_result old_data_length);
 
-    void share(shared_buffer& sb, char* data, size_t len);
+    void share(shared_buffer& sb, const char* data, size_t len);
 
     bool subset(shared_buffer_ref& other, off_t offset, size_t len);
 
@@ -154,7 +154,7 @@ private:
 
     auto_mem<char*> sb_backtrace;
     shared_buffer* sb_owner;
-    char* sb_data;
+    const char* sb_data;
     size_t sb_length;
 };
 

src/sql_util.cc

@@ -46,6 +46,7 @@
 #include "base/time_util.hh"
 #include "bound_tags.hh"
 #include "config.h"
+#include "lnav_util.hh"
 #include "pcrepp/pcrepp.hh"
 #include "readline_context.hh"
 #include "readline_highlighters.hh"
@@ -345,7 +346,8 @@ walk_sqlite_metadata(sqlite3* db, struct sqlite_metadata_callbacks& smc)
     }
 
     for (auto iter = smc.smc_db_list.begin(); iter != smc.smc_db_list.end();
-         ++iter) {
+         ++iter)
+    {
         struct table_list_data tld = {&smc, &iter};
         auto_mem<char, sqlite3_free> query;
 
@@ -836,18 +838,14 @@ sql_execute_script(sqlite3* db,
                 }
 
                 default: {
-                    attr_line_t sql_content(sqlite3_sql(stmt));
-                    const char* errmsg;
-
-                    errmsg = sqlite3_errmsg(db);
-                    sql_content.with_attr_for_all(
-                        VC_ROLE.value(role_t::VCR_QUOTED_CODE));
-                    readline_sqlite_highlighter(sql_content,
-                                                sql_content.length());
+                    const auto* sql_str = sqlite3_sql(stmt);
+                    auto sql_content
+                        = annotate_sql_with_error(db, sql_str, nullptr);
+
                     errors.emplace_back(
                         lnav::console::user_message::error(
                             "failed to execute SQL statement")
-                            .with_reason(errmsg)
+                            .with_reason(sqlite3_errmsg_to_attr_line(db))
                             .with_snippet(lnav::console::snippet::from(
                                 intern_string::lookup(src_name), sql_content)));
                     done = true;
@@ -971,6 +969,24 @@ sqlite_authorizer(void* pUserData,
     return SQLITE_OK;
 }
 
+attr_line_t
+sqlite3_errmsg_to_attr_line(sqlite3* db)
+{
+    const auto* errmsg = sqlite3_errmsg(db);
+    if (startswith(errmsg, sqlitepp::ERROR_PREFIX)) {
+        auto from_res = lnav::from_json<lnav::console::user_message>(
+            &errmsg[strlen(sqlitepp::ERROR_PREFIX)]);
+
+        if (from_res.isOk()) {
+            return from_res.unwrap().to_attr_line();
+        }
+
+        return from_res.unwrapErr()[0].um_message.get_string();
+    }
+
+    return attr_line_t(errmsg);
+}
+
 std::string
 sql_keyword_re()
 {
@@ -1064,7 +1080,8 @@ annotate_sql_statement(attr_line_t& al)
     int start = 0;
 
     while ((iter = find_string_attr(sa, &SQL_IDENTIFIER_ATTR, start))
-           != sa.end()) {
+           != sa.end())
+    {
         string_attrs_t::const_iterator piter;
         bool found_open = false;
         ssize_t lpc;
@@ -1177,7 +1194,8 @@ find_sql_help_for_line(const attr_line_t& al, size_t x)
         if (help_count > 1 && name != func_pair.first->second->ht_name) {
             while (func_pair.first != func_pair.second) {
                 if (find(kw.begin(), kw.end(), func_pair.first->second->ht_name)
-                    == kw.end()) {
+                    == kw.end())
+                {
                     ++func_pair.first;
                 } else {
                     func_pair.second = next(func_pair.first);

src/sql_util.hh

@@ -120,6 +120,8 @@ int guess_type_from_pcre(const std::string& pattern, std::string& collator);
 
 const char* sqlite3_type_to_string(int type);
 
+attr_line_t sqlite3_errmsg_to_attr_line(sqlite3* db);
+
 attr_line_t annotate_sql_with_error(sqlite3* db,
                                     const char* sql,
                                     const char* tail);

src/sqlitepp.cc

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) 2022, Timothy Stack
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of Timothy Stack nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "sqlitepp.hh"
+
+namespace sqlitepp {
+
+const char* ERROR_PREFIX = "lnav-error:";
+
+}

src/sqlitepp.hh

@@ -34,7 +34,10 @@
 
 #include <string>
 
+#include <sqlite3.h>
+
 #include "base/auto_mem.hh"
+#include "optional.hpp"
 
 /* XXX figure out how to do this with the template */
 void sqlite_close_wrapper(void* mem);
@@ -55,6 +58,8 @@ quote(const nonstd::optional<std::string>& str)
     return retval;
 }
 
+extern const char* ERROR_PREFIX;
+
 }  // namespace sqlitepp
 
 #endif

src/tailer/tailer.looper.cc

@@ -598,7 +598,8 @@ tailer::looper::host_tailer::loop_body()
 
                 auto finished_child = std::move(conn).close();
                 if (finished_child.exit_status() != 0
-                    && !this->ht_error_queue.empty()) {
+                    && !this->ht_error_queue.empty())
+                {
                     report_error(this->ht_netloc, this->ht_error_queue.back());
                 }
 
@@ -632,7 +633,8 @@ tailer::looper::host_tailer::loop_body()
                     auto child_iter = conn.c_child_paths.find(pe.pe_path);
 
                     if (child_iter != conn.c_child_paths.end()
-                        && !child_iter->second.loo_tail) {
+                        && !child_iter->second.loo_tail)
+                    {
                         conn.c_child_paths.erase(child_iter);
                     }
                 }
@@ -1029,7 +1031,8 @@ tailer::looper::child_finished(std::shared_ptr<service_base> child)
     auto child_tailer = std::static_pointer_cast<host_tailer>(child);
 
     for (auto iter = this->l_remotes.begin(); iter != this->l_remotes.end();
-         ++iter) {
+         ++iter)
+    {
         if (iter->second != child_tailer) {
             continue;
         }
@@ -1101,12 +1104,13 @@ tailer::cleanup_cache()
     (void) std::async(std::launch::async, []() {
         auto now = std::chrono::system_clock::now();
         auto cache_path = remote_cache_path();
-        auto& cfg = injector::get<const config&>();
+        const auto& cfg = injector::get<const config&>();
         std::vector<ghc::filesystem::path> to_remove;
 
         log_debug("cache-ttl %d", cfg.c_cache_ttl.count());
         for (const auto& entry :
-             ghc::filesystem::directory_iterator(cache_path)) {
+             ghc::filesystem::directory_iterator(cache_path))
+        {
             auto mtime = ghc::filesystem::last_write_time(entry.path());
             auto exp_time = mtime + cfg.c_cache_ttl;
             if (now < exp_time) {

src/textview_curses.hh

@@ -577,7 +577,11 @@ public:
         return retval;
     }
 
-    void grep_quiesce() { this->tc_sub_source->quiesce(); }
+    void grep_quiesce() {
+        if (this->tc_sub_source != nullptr) {
+            this->tc_sub_source->quiesce();
+        }
+    }
 
     void grep_begin(grep_proc<vis_line_t>& gp,
                     vis_line_t start,

src/view_helpers.cc

@@ -155,7 +155,8 @@ public:
 
                         if (parent_node != nullptr) {
                             for (const auto& sibling :
-                                 parent_node->hn_named_children) {
+                                 parent_node->hn_named_children)
+                            {
                                 retval.template emplace_back(sibling.first);
                             }
                         }
@@ -306,7 +307,8 @@ open_pretty_view()
         bool first_line = true;
 
         for (vis_line_t vl = log_tc->get_top(); vl <= log_tc->get_bottom();
-             ++vl) {
+             ++vl)
+        {
             content_line_t cl = lss.at(vl);
             auto lf = lss.find(cl);
             auto ll = lf->begin() + cl;
@@ -409,7 +411,8 @@ open_pretty_view()
 
             for (vis_line_t vl = text_tc->get_top();
                  vl <= text_tc->get_bottom();
-                 ++vl) {
+                 ++vl)
+            {
                 auto ll = lf->begin() + vl;
                 shared_buffer_ref sbr;
 
@@ -796,7 +799,8 @@ execute_examples()
                         dos.list_value_for_overlay(db_tc, 0, 1, 0_vl, al);
                         result.append(al);
                         for (int lpc = 0; lpc < (int) dls.text_line_count();
-                             lpc++) {
+                             lpc++)
+                        {
                             al.clear();
                             dls.text_value_for_line(
                                 db_tc, lpc, al.get_string(), false);
@@ -864,6 +868,10 @@ toggle_view(textview_curses* toggle_tc)
             rebuild_hist();
         } else if (toggle_tc == &lnav_data.ld_views[LNV_HELP]) {
             build_all_help_text();
+            if (lnav_data.ld_rl_view != nullptr) {
+                lnav_data.ld_rl_view->set_alt_value(
+                    HELP_MSG_1(q, "to return to the previous view"));
+            }
         }
         lnav_data.ld_last_view = nullptr;
         lnav_data.ld_view_stack.push_back(toggle_tc);

src/views_vtab.cc

@@ -194,15 +194,9 @@ CREATE TABLE lnav_views (
 
     using iterator = textview_curses*;
 
-    iterator begin()
-    {
-        return std::begin(lnav_data.ld_views);
-    }
+    iterator begin() { return std::begin(lnav_data.ld_views); }
 
-    iterator end()
-    {
-        return std::end(lnav_data.ld_views);
-    }
+    iterator end() { return std::end(lnav_data.ld_views); }
 
     int get_column(cursor& vc, sqlite3_context* ctx, int col)
     {
@@ -417,15 +411,9 @@ CREATE TABLE lnav_view_stack (
 );
 )";
 
-    iterator begin()
-    {
-        return lnav_data.ld_view_stack.begin();
-    }
+    iterator begin() { return lnav_data.ld_view_stack.begin(); }
 
-    iterator end()
-    {
-        return lnav_data.ld_view_stack.end();
-    }
+    iterator end() { return lnav_data.ld_view_stack.end(); }
 
     int get_column(cursor& vc, sqlite3_context* ctx, int col)
     {
@@ -535,10 +523,7 @@ struct lnav_view_filter_base {
         return ++retval;
     }
 
-    iterator end()
-    {
-        return {LNV__MAX, -1};
-    }
+    iterator end() { return {LNV__MAX, -1}; }
 
     sqlite_int64 get_rowid(iterator iter)
     {
@@ -731,7 +716,7 @@ CREATE TABLE lnav_view_filters (
                 if (set_res.isErr()) {
                     tab->zErrMsg = sqlite3_mprintf(
                         "%s%s",
-                        LNAV_SQLITE_ERROR_PREFIX,
+                        sqlitepp::ERROR_PREFIX,
                         lnav::to_json(set_res.unwrapErr()).c_str());
                     return SQLITE_ERROR;
                 }
@@ -832,7 +817,7 @@ CREATE TABLE lnav_view_filters (
             if (set_res.isErr()) {
                 tab->zErrMsg = sqlite3_mprintf(
                     "%s%s",
-                    LNAV_SQLITE_ERROR_PREFIX,
+                    sqlitepp::ERROR_PREFIX,
                     lnav::to_json(set_res.unwrapErr()).c_str());
                 return SQLITE_ERROR;
             }

src/vtab_module.cc

@@ -31,18 +31,17 @@
 
 #include "config.h"
 #include "lnav_util.hh"
+#include "sqlitepp.hh"
 
 std::string vtab_module_schemas;
 
 std::map<intern_string_t, std::string> vtab_module_ddls;
 
-const char* LNAV_SQLITE_ERROR_PREFIX = "lnav-error:";
-
 void
 to_sqlite(sqlite3_context* ctx, const lnav::console::user_message& um)
 {
     auto errmsg = fmt::format(
-        FMT_STRING("{}{}"), LNAV_SQLITE_ERROR_PREFIX, lnav::to_json(um));
+        FMT_STRING("{}{}"), sqlitepp::ERROR_PREFIX, lnav::to_json(um));
 
     sqlite3_result_error(ctx, errmsg.c_str(), errmsg.size());
 }
@@ -51,9 +50,9 @@ lnav::console::user_message
 sqlite3_error_to_user_message(sqlite3* db)
 {
     const auto* errmsg = sqlite3_errmsg(db);
-    if (startswith(errmsg, LNAV_SQLITE_ERROR_PREFIX)) {
+    if (startswith(errmsg, sqlitepp::ERROR_PREFIX)) {
         auto from_res = lnav::from_json<lnav::console::user_message>(
-            &errmsg[strlen(LNAV_SQLITE_ERROR_PREFIX)]);
+            &errmsg[strlen(sqlitepp::ERROR_PREFIX)]);
 
         if (from_res.isOk()) {
             return from_res.unwrap();

src/vtab_module.hh

@@ -48,8 +48,6 @@
 #include "shlex.resolver.hh"
 #include "sqlite-extension-func.hh"
 
-extern const char* LNAV_SQLITE_ERROR_PREFIX;
-
 lnav::console::user_message sqlite3_error_to_user_message(sqlite3*);
 
 struct from_sqlite_conversion_error : std::exception {

test/expected/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err

@@ -88,7 +88,7 @@
  reason: sample does not match any patterns
  --> {test_dir}/bad-config/formats/invalid-sample/format.json:41
  = note: the following shows how each pattern matched this sample:
-            1428634687123; foo bar
+            1428634687123; foo bar
                          ^ bad-time matched up to here
                               ^ semi matched up to here
                          ^ std matched up to here
@@ -125,7 +125,8 @@
  | CREATE TALE invalid (x y z);            
  |        ^ near "TALE": syntax error      
 ✘ error: failed to execute SQL statement
- reason: lnav-error:{"level":"error","message":{"str":"call to regexp_match(re, str) failed","attrs":[{"start":8,"end":20,"type":"role","value":46},{"start":21,"end":23,"type":"role","value":45},{"start":25,"end":28,"type":"role","value":45},{"start":8,"end":29,"type":"role","value":59}]},"reason":{"str":"missing )","attrs":[]},"snippets":[],"help":{"str":"","attrs":[]}}
+ reason: ✘ error: call to regexp_match(re, str) failed
+ |        reason: missing )
  --> {test_dir}/bad-config/formats/invalid-sql/init2.sql
  | SELECT regexp_match('abc(', '123')      
  | FROM sqlite_master;                     

test/expected/test_json_format.sh_c60050b3469f37c5b0864e1dc7eb354e91d6ec81.out

@@ -38,13 +38,13 @@
 	... 33 common frames omitted
 
   @version: 1
-  logger_name: org.apache.jasper.runtime.JspFactoryImpl
+  logger_name: org.apache.jasper.runtime.JspFactoryImpl
   thread_name: http-bio-0.0.0.0-8081-exec-198
   level: ERROR
   customer: foobaz
 2016-08-03T12:06:31.009 - ;Exception initializing page context; 
   @version: 1
-  logger_name: org.apache.jasper.runtime.JspFactoryImpl
+  logger_name: org.apache.jasper.runtime.JspFactoryImpl
   thread_name: http-bio-0.0.0.0-8081-exec-198
   level: ERROR
   customer: foobaz