[piper] replace piper_proc with a thread

Related to #1029

NEWS.md

@@ -1,3 +1,39 @@
+## lnav v0.12.0
+
+Features:
+* Added the `:sh` command and `-e` option to execute a shell
+  command-line and display its output within **lnav**.   The
+  captured output will be displayed in the TEXT view.  The
+  lines from stdout and stderr are recorded separately so
+  that the lines from stderr can be shown in the theme's
+  "error" highlight.  The time that the lines were received
+  are also recorded internally so that the "time-offset"
+  display (enabled by pressing `Shift` + `T`) can be shown
+  and the "jump to slow-down" hotkeys (`s`/`Shift` + `S`)
+  work.  Since the line-by-line timestamps are recorded
+  internally, they will not interfere with timestamps that
+  are in the commands output.
+* Added a `:cd` command to change **lnav**'s current directory.
+
+Bug Fixes:
+* When piping data into **lnav**'s stdin, the input used to
+  only be written to a single file without any rotation.
+  Now, the input is written to a directory of rotating files.
+  The same is true for the command-lines executed through the
+  new `:sh` command.
+* Binary data piped into stdin should now be treated the same
+  as if it was in a file that was passed on the command-line.
+
+Breaking changes:
+* Removed the `-w` command-line option.  This option was
+  useful when stdin was not automatically preserved.  Since
+  the data is now stored (and cleaned up) as well as being
+  spread across multiple files, this option doesn't make
+  sense anymore.
+* Removed the `-t` command-line flag.  Text data fed in
+  on stdin and captured from a `:sh` execution is
+  automatically timestamped.
+
 ## lnav v0.11.2
 
 Features:
@@ -12,7 +48,7 @@ Features:
   field should automatically be determined by the observed
   values.
 * Added bunyan log format from Tobias Gruetzmacher.
-* Added cloudlare log format from @minusf.
+* Added cloudflare log format from @minusf.
 * Number fields used in a JSON log format `line-format`
   array now default to being right-aligned.  Also, added
   `prefix` and `suffix` to `line-format` elements so a

docs/schemas/config-v1.schema.json

@@ -39,6 +39,26 @@
                     },
                     "additionalProperties": false
                 },
+                "piper": {
+                    "description": "Settings related to capturing piped data",
+                    "title": "/tuning/piper",
+                    "type": "object",
+                    "properties": {
+                        "max-size": {
+                            "title": "/tuning/piper/max-size",
+                            "description": "The maximum size of a capture file",
+                            "type": "integer",
+                            "minimum": 128
+                        },
+                        "rotations": {
+                            "title": "/tuning/piper/rotations",
+                            "description": "The number of rotated files to keep",
+                            "type": "integer",
+                            "minimum": 2
+                        }
+                    },
+                    "additionalProperties": false
+                },
                 "file-vtab": {
                     "description": "Settings related to the lnav_file virtual-table",
                     "title": "/tuning/file-vtab",

docs/source/cli.rst

@@ -45,6 +45,12 @@ Options
 
    Execute the given command file.  This option can be given multiple times.
 
+.. option:: -e <command-line>
+
+   Execute the given shell command-line and display its output.  This is
+   equivalent to executing the :code:`:sh` command and passing the
+   :option:`-N` flag. This option can be given multiple times.
+
 .. option:: -I <path>
 
    Add a configuration directory.
@@ -76,14 +82,6 @@ Options
 
    Recursively load files from the given base directories.
 
-.. option:: -t
-
-   Prepend timestamps to the lines of data being read in on the standard input.
-
-.. option:: -w <path>
-
-   Write the contents of the standard input to this file.
-
 .. option:: -V
 
    Print the version of lnav.
@@ -161,8 +159,8 @@ Examples
 
     lnav /var/log
 
-  To watch the output of make with timestamps prepended:
+  To watch the output of make:
 
   .. prompt:: bash
 
-    make 2>&1 | lnav -t
+    lnav -e 'make -j4'

docs/source/config.rst

@@ -1,4 +1,3 @@
-
 .. _Configuration:
 
 Configuration
@@ -261,6 +260,8 @@ command.
 
 .. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/clipboard
 
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/piper
+
 .. jsonschema:: ../schemas/config-v1.schema.json#/definitions/clip-commands
 
 .. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/file-vtab

docs/source/hotkeys.rst

@@ -223,7 +223,9 @@ Display
    * - :kbd:`Shift` + :kbd:`p`
      - Switch to/from the pretty-printed view of the displayed log or text files
    * - :kbd:`Shift` + :kbd:`t`
-     - Display elapsed time between lines
+     - Display the elapsed time from a bookmark to a given line.  In the TEXT view,
+       this only works for content that was captured from stdin or a :code:`:sh`
+       command.
    * - :kbd:`t`
      - Switch to/from the text file view
    * - :kbd:`i`

lnav.1

@@ -94,9 +94,6 @@ Load older rotated log files as well.
 \fB\-t\fR
 Prepend timestamps to the lines of data being read in
 on the standard input.
-.TP
-\fB\-w\fR file
-Write the contents of the standard input to this file.
 .SS "Optional arguments:"
 .TP
 logfile1

src/CMakeLists.txt

@@ -412,7 +412,7 @@ add_library(
         statusview_curses.cc
         string-extension-functions.cc
         sysclip.cc
-        piper_proc.cc
+        piper.looper.cc
         spectro_impls.cc
         spectro_source.cc
         sql_commands.cc
@@ -503,6 +503,8 @@ add_library(
         md4cpp.hh
         optional.hpp
         pcap_manager.hh
+        piper.looper.hh
+        piper.looper.cfg.hh
         plain_text_source.hh
         pretty_printer.hh
         preview_status_source.hh

src/Makefile.am

@@ -259,7 +259,8 @@ noinst_HEADERS = \
 	md4cpp.hh \
 	optional.hpp \
 	pcap_manager.hh \
-	piper_proc.hh \
+	piper.looper.hh \
+	piper.looper.cfg.hh \
 	plain_text_source.hh \
 	pollable.hh \
 	pretty_printer.hh \
@@ -428,6 +429,7 @@ libdiag_a_SOURCES = \
 	network-extension-functions.cc \
 	data_parser.cc \
 	pcap_manager.cc \
+	piper.looper.cc \
 	plain_text_source.cc \
 	pollable.cc \
 	pretty_printer.cc \
@@ -456,7 +458,6 @@ libdiag_a_SOURCES = \
 	text_format.cc \
 	textfile_sub_source.cc \
 	timer.cc \
-	piper_proc.cc \
 	sql_commands.cc \
 	sql_util.cc \
 	state-extension-functions.cc \

src/base/auto_fd.hh

@@ -194,6 +194,8 @@ public:
      */
     int get() const { return this->af_fd; }
 
+    bool has_value() const { return this->af_fd != -1; }
+
     /**
      * Closes the current file descriptor and replaces its value with the given
      * one.

src/command_executor.cc

@@ -35,6 +35,7 @@
 #include "base/fs_util.hh"
 #include "base/injector.hh"
 #include "base/itertools.hh"
+#include "base/paths.hh"
 #include "base/string_util.hh"
 #include "bound_tags.hh"
 #include "config.h"
@@ -842,15 +843,19 @@ execute_init_commands(
     if (ec_out && fstat(fd_copy, &st) != -1 && st.st_size > 0) {
         static const auto OUTPUT_NAME = std::string("Initial command output");
 
-        lnav_data.ld_active_files.fc_file_names[OUTPUT_NAME]
+        auto create_piper_res = lnav::piper::create_looper(
-            .with_fd(std::move(fd_copy))
+            OUTPUT_NAME, std::move(fd_copy), auto_fd{});
-            .with_include_in_session(false)
+        if (create_piper_res.isOk()) {
-            .with_detect_format(false);
+            lnav_data.ld_active_files.fc_file_names[OUTPUT_NAME]
-        lnav_data.ld_files_to_front.emplace_back(OUTPUT_NAME, 0_vl);
+                .with_piper(create_piper_res.unwrap())
-
+                .with_include_in_session(false)
-        if (lnav_data.ld_rl_view != nullptr) {
+                .with_detect_format(false);
-            lnav_data.ld_rl_view->set_alt_value(
+            lnav_data.ld_files_to_front.emplace_back(OUTPUT_NAME, 0_vl);
-                HELP_MSG_1(X, "to close the file"));
+
+            if (lnav_data.ld_rl_view != nullptr) {
+                lnav_data.ld_rl_view->set_alt_value(
+                    HELP_MSG_1(X, "to close the file"));
+            }
         }
     }
 
@@ -978,24 +983,21 @@ pipe_callback(exec_context& ec, const std::string& cmdline, auto_fd& fd)
             return std::string();
         });
     }
-    auto tmp_fd
+    auto open_temp_res = lnav::filesystem::open_temp_file(lnav::paths::workdir()
-        = lnav::filesystem::open_temp_file(
+                                                          / "exec.XXXXXX");
-              ghc::filesystem::temp_directory_path() / "lnav.out.XXXXXX")
+    if (open_temp_res.isErr()) {
-              .map([](auto pair) {
+        return lnav::futures::make_ready_future(
-                  ghc::filesystem::remove(pair.first);
+            fmt::format(FMT_STRING("error: cannot open temp file -- {}"),
-
+                        open_temp_res.unwrapErr()));
-                  return std::move(pair.second);
+    }
-              })
+
-              .expect("Cannot create temporary file for callback");
+    auto tmp_pair = open_temp_res.unwrap();
-    auto pp
-        = std::make_shared<piper_proc>(std::move(fd), false, std::move(tmp_fd));
-    static int exec_count = 0;
 
-    lnav_data.ld_pipers.push_back(pp);
+    static int exec_count = 0;
     auto desc
         = fmt::format(FMT_STRING("[{}] Output of {}"), exec_count++, cmdline);
-    lnav_data.ld_active_files.fc_file_names[desc]
+    lnav_data.ld_active_files.fc_file_names[tmp_pair.first]
-        .with_fd(pp->get_fd())
+        .with_filename(desc)
         .with_include_in_session(false)
         .with_detect_format(false);
     lnav_data.ld_files_to_front.emplace_back(desc, 0_vl);

src/command_executor.hh

@@ -72,15 +72,9 @@ struct exec_context {
                  sql_callback_t sql_callback = ::sql_callback,
                  pipe_callback_t pipe_callback = nullptr);
 
-    bool is_read_write() const
+    bool is_read_write() const { return this->ec_perms == perm_t::READ_WRITE; }
-    {
-        return this->ec_perms == perm_t::READ_WRITE;
-    }
 
-    bool is_read_only() const
+    bool is_read_only() const { return this->ec_perms == perm_t::READ_ONLY; }
-    {
-        return this->ec_perms == perm_t::READ_ONLY;
-    }
 
     exec_context& with_perms(perm_t perms)
     {
@@ -91,15 +85,22 @@ struct exec_context {
     void add_error_context(lnav::console::user_message& um);
 
     template<typename... Args>
-    Result<std::string, lnav::console::user_message> make_error(
+    lnav::console::user_message make_error_msg(fmt::string_view format_str,
-        fmt::string_view format_str, const Args&... args)
+                                               const Args&... args)
     {
         auto retval = lnav::console::user_message::error(
             fmt::vformat(format_str, fmt::make_format_args(args...)));
 
         this->add_error_context(retval);
 
-        return Err(retval);
+        return retval;
+    }
+
+    template<typename... Args>
+    Result<std::string, lnav::console::user_message> make_error(
+        fmt::string_view format_str, const Args&... args)
+    {
+        return Err(this->make_error_msg(format_str, args...));
     }
 
     nonstd::optional<FILE*> get_output()

src/file_collection.cc

@@ -162,9 +162,8 @@ file_collection::merge(file_collection& other)
                                  other.fc_synced_files.end());
     this->fc_name_to_errors.insert(other.fc_name_to_errors.begin(),
                                    other.fc_name_to_errors.end());
-    this->fc_file_names.insert(
+    this->fc_file_names.insert(other.fc_file_names.begin(),
-        std::make_move_iterator(other.fc_file_names.begin()),
+                               other.fc_file_names.end());
-        std::make_move_iterator(other.fc_file_names.end()));
     if (!other.fc_files.empty()) {
         for (const auto& lf : other.fc_files) {
             this->fc_name_to_errors.erase(lf->get_filename());
@@ -193,7 +192,7 @@ file_collection::merge(file_collection& other)
  * Functor used to compare files based on their device and inode number.
  */
 struct same_file {
-    explicit same_file(const struct stat& stat) : sf_stat(stat){};
+    explicit same_file(const struct stat& stat) : sf_stat(stat) {}
 
     /**
      * Compare the given log file against the 'stat' given in the constructor.
@@ -203,7 +202,20 @@ struct same_file {
      */
     bool operator()(const std::shared_ptr<logfile>& lf) const
     {
-        return !lf->is_closed() && this->sf_stat.st_dev == lf->get_stat().st_dev
+        if (lf->is_closed()) {
+            return false;
+        }
+
+        const auto& lf_loo = lf->get_open_options();
+
+        if (lf_loo.loo_temp_dev != 0
+            && this->sf_stat.st_dev == lf_loo.loo_temp_dev
+            && this->sf_stat.st_ino == lf_loo.loo_temp_ino)
+        {
+            return true;
+        }
+
+        return this->sf_stat.st_dev == lf->get_stat().st_dev
             && this->sf_stat.st_ino == lf->get_stat().st_ino;
     }
 
@@ -228,16 +240,12 @@ file_collection::watch_logfile(const std::string& filename,
     struct stat st;
     int rc;
 
+    auto filename_key = loo.loo_filename.empty() ? filename : loo.loo_filename;
     if (this->fc_closed_files.count(filename)) {
         return lnav::futures::make_ready_future(std::move(retval));
     }
 
-    if (loo.loo_fd != -1) {
+    if (loo.loo_temp_file) {
-        rc = fstat(loo.loo_fd, &st);
-        if (rc == 0) {
-            loo.with_stat_for_temp(st);
-        }
-    } else if (loo.loo_temp_file) {
         memset(&st, 0, sizeof(st));
         st.st_dev = loo.loo_temp_dev;
         st.st_ino = loo.loo_temp_ino;
@@ -265,7 +273,7 @@ file_collection::watch_logfile(const std::string& filename,
                 return lnav::futures::make_ready_future(std::move(retval));
             }
         }
-        auto err_iter = this->fc_name_to_errors.find(filename);
+        auto err_iter = this->fc_name_to_errors.find(filename_key);
         if (err_iter != this->fc_name_to_errors.end()) {
             if (err_iter->second.fei_mtime != st.st_mtime) {
                 this->fc_name_to_errors.erase(err_iter);
@@ -274,6 +282,9 @@ file_collection::watch_logfile(const std::string& filename,
     }
     if (rc == -1) {
         if (required) {
+            log_error("failed to open required file: %s -- %s",
+                      filename.c_str(),
+                      strerror(errno));
             retval.fc_name_to_errors.emplace(filename,
                                              file_error_info{
                                                  time(nullptr),
@@ -306,7 +317,7 @@ file_collection::watch_logfile(const std::string& filename,
 
         auto func = [filename,
                      st,
-                     loo2 = std::move(loo),
+                     loo,
                      prog = this->fc_progress,
                      errs = this->fc_name_to_errors]() mutable {
             file_collection retval;
@@ -316,10 +327,10 @@ file_collection::watch_logfile(const std::string& filename,
                 return retval;
             }
 
-            auto ff = loo2.loo_temp_file ? file_format_t::UNKNOWN
+            auto ff = loo.loo_temp_file ? file_format_t::UNKNOWN
-                                         : detect_file_format(filename);
+                                        : detect_file_format(filename);
 
-            loo2.loo_file_format = ff;
+            loo.loo_file_format = ff;
             switch (ff) {
                 case file_format_t::SQLITE_DB:
                     retval.fc_other_files[filename].ofd_format = ff;
@@ -330,8 +341,6 @@ file_collection::watch_logfile(const std::string& filename,
 
                     if (res.isOk()) {
                         auto convert_res = res.unwrap();
-
-                        loo2.with_fd(std::move(convert_res.cr_destination));
                         retval.fc_child_pollers.emplace_back(child_poller{
                             std::move(convert_res.cr_child),
                             [filename,
@@ -358,10 +367,15 @@ file_collection::watch_logfile(const std::string& filename,
                                     });
                             },
                         });
-                        auto open_res = logfile::open(filename, loo2);
+                        loo.with_stat_for_temp(st);
+                        auto open_res
+                            = logfile::open(convert_res.cr_destination, loo);
                         if (open_res.isOk()) {
                             retval.fc_files.push_back(open_res.unwrap());
                         } else {
+                            log_error("failed to open: %s -- %s",
+                                      filename.c_str(),
+                                      open_res.unwrapErr().c_str());
                             retval.fc_name_to_errors.emplace(
                                 filename,
                                 file_error_info{
@@ -384,7 +398,7 @@ file_collection::watch_logfile(const std::string& filename,
                         std::list<archive_manager::extract_progress>::iterator>
                         prog_iter_opt;
 
-                    if (loo2.loo_source == logfile_name_source::ARCHIVE) {
+                    if (loo.loo_source == logfile_name_source::ARCHIVE) {
                         // Don't try to open nested archives
                         return retval;
                     }
@@ -451,7 +465,7 @@ file_collection::watch_logfile(const std::string& filename,
                 default:
                     log_info("loading new file: filename=%s", filename.c_str());
 
-                    auto open_res = logfile::open(filename, loo2);
+                    auto open_res = logfile::open(filename, loo);
                     if (open_res.isOk()) {
                         retval.fc_files.push_back(open_res.unwrap());
                     } else {
@@ -510,6 +524,7 @@ file_collection::expand_filename(
         return;
     }
 
+    auto filename_key = loo.loo_filename.empty() ? path : loo.loo_filename;
     if (glob(path.c_str(), GLOB_NOCHECK, nullptr, gl.inout()) == 0) {
         int lpc;
 
@@ -579,12 +594,18 @@ file_collection::expand_filename(
                         file_collection retval;
 
                         if (gl->gl_pathc == 1) {
-                            retval.fc_name_to_errors.emplace(path,
+                            log_error("failed to find path: %s -- %s",
+                                      path.c_str(),
+                                      errmsg);
+                            retval.fc_name_to_errors.emplace(filename_key,
                                                              file_error_info{
                                                                  time(nullptr),
                                                                  errmsg,
                                                              });
                         } else {
+                            log_error("failed to find path: %s -- %s",
+                                      path_str.c_str(),
+                                      errmsg);
                             retval.fc_name_to_errors.emplace(path_str,
                                                              file_error_info{
                                                                  time(nullptr),
@@ -617,15 +638,19 @@ file_collection::rescan_files(bool required)
         [&retval](auto& fc) { retval.merge(fc); });
 
     for (auto& pair : this->fc_file_names) {
-        if (!pair.second.loo_temp_file) {
+        if (pair.second.loo_piper) {
+            this->expand_filename(
+                fq,
+                pair.second.loo_piper->get_out_pattern().string(),
+                pair.second,
+                required);
+        } else if (!pair.second.loo_temp_file) {
             this->expand_filename(fq, pair.first, pair.second, required);
             if (this->fc_rotated) {
                 std::string path = pair.first + ".*";
 
                 this->expand_filename(fq, path, pair.second, false);
             }
-        } else if (pair.second.loo_fd.get() != -1) {
-            fq.push_back(watch_logfile(pair.first, pair.second, required));
         }
 
         if (retval.fc_files.size() >= 100) {
@@ -647,3 +672,16 @@ file_collection::request_close(const std::shared_ptr<logfile>& lf)
     lf->close();
     this->fc_files_generation += 1;
 }
+
+size_t
+file_collection::active_pipers() const
+{
+    size_t retval = 0;
+    for (const auto& pair : this->fc_file_names) {
+        if (pair.second.loo_piper && !pair.second.loo_piper->is_finished()) {
+            retval += 1;
+        }
+    }
+
+    return retval;
+}

src/file_collection.hh

@@ -175,6 +175,8 @@ struct file_collection {
     void close_files(const std::vector<std::shared_ptr<logfile>>& files);
 
     void regenerate_unique_file_names();
+
+    size_t active_pipers() const;
 };
 
 #endif

src/file_vtab.cc

@@ -203,18 +203,20 @@ CREATE TABLE lnav_file (
                 = this->lf_collection.fc_file_names.find(lf->get_filename());
 
             if (iter != this->lf_collection.fc_file_names.end()) {
-                auto loo = std::move(iter->second);
+                auto loo = iter->second;
 
                 this->lf_collection.fc_file_names.erase(iter);
 
                 loo.loo_include_in_session = true;
-                this->lf_collection.fc_file_names[path] = std::move(loo);
+                this->lf_collection.fc_file_names[path] = loo;
-                lf->set_filename(path);
-                this->lf_collection.regenerate_unique_file_names();
-
-                init_session();
-                load_session();
             }
+
+            lf->set_filename(path);
+            lf->set_include_in_session(true);
+            this->lf_collection.regenerate_unique_file_names();
+
+            init_session();
+            load_session();
         }
 
         return SQLITE_OK;

src/hotkeys.cc

@@ -206,7 +206,6 @@ handle_paging_key(int ch)
 
     textview_curses* tc = *lnav_data.ld_view_stack.top();
     exec_context& ec = lnav_data.ld_exec_context;
-    logfile_sub_source* lss = nullptr;
     text_sub_source* tc_tss = tc->get_sub_source();
     bookmarks<vis_line_t>::type& bm = tc->get_bookmarks();
 
@@ -219,7 +218,8 @@ handle_paging_key(int ch)
         return true;
     }
 
-    lss = dynamic_cast<logfile_sub_source*>(tc->get_sub_source());
+    auto lss = dynamic_cast<logfile_sub_source*>(tc->get_sub_source());
+    auto text_accel_p = dynamic_cast<text_accel_source*>(tc->get_sub_source());
 
     /* process the command keystroke */
     switch (ch) {
@@ -485,20 +485,28 @@ handle_paging_key(int ch)
 #endif
 
         case 's':
-            if (lss) {
+            if (text_accel_p && text_accel_p->is_time_offset_supported()) {
-                auto next_top = tc->get_selection() + 2_vl;
+                auto next_top = tc->get_selection() + 1_vl;
+
+                if (!tc->is_selectable()) {
+                    next_top += 1_vl;
+                }
 
-                if (!lss->is_time_offset_enabled()) {
+                if (!text_accel_p->is_time_offset_enabled()) {
                     lnav_data.ld_rl_view->set_alt_value(
                         HELP_MSG_1(T, "to disable elapsed-time mode"));
                 }
-                lss->set_time_offset(true);
+                text_accel_p->set_time_offset(true);
                 while (next_top < tc->get_inner_height()) {
-                    if (!lss->find_line(lss->at(next_top))->is_message()) {
+                    if (!text_accel_p->text_accel_get_line(next_top)
-                    } else if (lss->get_line_accel_direction(next_top)
+                             ->is_message())
+                    {
+                    } else if (text_accel_p->get_line_accel_direction(next_top)
                                == log_accel::A_DECEL)
                     {
-                        --next_top;
+                        if (!tc->is_selectable()) {
+                            --next_top;
+                        }
                         tc->set_selection(next_top);
                         break;
                     }
@@ -509,20 +517,27 @@ handle_paging_key(int ch)
             break;
 
         case 'S':
-            if (lss) {
+            if (text_accel_p && text_accel_p->is_time_offset_supported()) {
                 auto next_top = tc->get_selection();
 
-                if (!lss->is_time_offset_enabled()) {
+                if (tc->is_selectable() && next_top > 0_vl) {
+                    next_top -= 1_vl;
+                }
+                if (!text_accel_p->is_time_offset_enabled()) {
                     lnav_data.ld_rl_view->set_alt_value(
                         HELP_MSG_1(T, "to disable elapsed-time mode"));
                 }
-                lss->set_time_offset(true);
+                text_accel_p->set_time_offset(true);
                 while (0 <= next_top && next_top < tc->get_inner_height()) {
-                    if (!lss->find_line(lss->at(next_top))->is_message()) {
+                    if (!text_accel_p->text_accel_get_line(next_top)
-                    } else if (lss->get_line_accel_direction(next_top)
+                             ->is_message())
+                    {
+                    } else if (text_accel_p->get_line_accel_direction(next_top)
                                == log_accel::A_DECEL)
                     {
-                        --next_top;
+                        if (!tc->is_selectable()) {
+                            --next_top;
+                        }
                         tc->set_selection(next_top);
                         break;
                     }
@@ -724,12 +739,16 @@ handle_paging_key(int ch)
             break;
 
         case 'T':
-            lnav_data.ld_log_source.toggle_time_offset();
+            if (text_accel_p != nullptr
-            if (lss && lss->is_time_offset_enabled()) {
+                && text_accel_p->is_time_offset_supported())
-                lnav_data.ld_rl_view->set_alt_value(HELP_MSG_2(
+            {
-                    s, S, "to move forward/backward through slow downs"));
+                text_accel_p->toggle_time_offset();
+                if (text_accel_p->is_time_offset_enabled()) {
+                    lnav_data.ld_rl_view->set_alt_value(HELP_MSG_2(
+                        s, S, "to move forward/backward through slow downs"));
+                }
+                tc->reload_data();
             }
-            tc->reload_data();
             break;
 
         case 'I': {

src/internals/cmd-ref.rst

@@ -44,7 +44,7 @@
       :alt-msg Press t to switch to the text view
 
   **See Also**
-    :ref:`echo`, :ref:`eval`, :ref:`export_session_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_view_to`
+    :ref:`cd`, :ref:`echo`, :ref:`eval`, :ref:`export_session_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_view_to`
 
 ----
 
@@ -72,6 +72,22 @@
 ----
 
 
+.. _cd:
+
+:cd *dir*
+^^^^^^^^^
+
+  Change the current directory
+
+  **Parameters**
+    * **dir\*** --- The new current directory
+
+  **See Also**
+    :ref:`alt_msg`, :ref:`echo`, :ref:`eval`, :ref:`export_session_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_view_to`
+
+----
+
+
 .. _clear_comment:
 
 :clear-comment
@@ -414,7 +430,7 @@
       :echo Hello, World!
 
   **See Also**
-    :ref:`alt_msg`, :ref:`append_to`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`
+    :ref:`alt_msg`, :ref:`append_to`, :ref:`cd`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`
 
 ----
 
@@ -473,7 +489,7 @@
       :eval ;SELECT * FROM ${table}
 
   **See Also**
-    :ref:`alt_msg`, :ref:`echo`, :ref:`export_session_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_view_to`
+    :ref:`alt_msg`, :ref:`cd`, :ref:`echo`, :ref:`export_session_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_view_to`
 
 ----
 
@@ -489,7 +505,7 @@
     * **path\*** --- The path to the file to write
 
   **See Also**
-    :ref:`alt_msg`, :ref:`append_to`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`
+    :ref:`alt_msg`, :ref:`append_to`, :ref:`cd`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`
 
 ----
 
@@ -1020,7 +1036,7 @@
   Forcefully rebuild file indexes
 
   **See Also**
-    :ref:`alt_msg`, :ref:`echo`, :ref:`eval`, :ref:`export_session_to`, :ref:`redirect_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_view_to`
+    :ref:`alt_msg`, :ref:`cd`, :ref:`echo`, :ref:`eval`, :ref:`export_session_to`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_view_to`
 
 ----
 
@@ -1043,7 +1059,7 @@
       :redirect-to /tmp/script-output.txt
 
   **See Also**
-    :ref:`alt_msg`, :ref:`append_to`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`
+    :ref:`alt_msg`, :ref:`append_to`, :ref:`cd`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`
 
 ----
 
@@ -1175,6 +1191,22 @@
 ----
 
 
+.. _sh:
+
+:sh *cmdline*
+^^^^^^^^^^^^^
+
+  Execute the given command-line and display the captured output
+
+  **Parameters**
+    * **cmdline\*** --- The command-line to execute.
+
+  **See Also**
+    :ref:`alt_msg`, :ref:`cd`, :ref:`echo`, :ref:`eval`, :ref:`export_session_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_view_to`
+
+----
+
+
 .. _show_fields:
 
 :show-fields *field-name*
@@ -1432,7 +1464,7 @@
       :write-table-to /tmp/table.txt
 
   **See Also**
-    :ref:`alt_msg`, :ref:`append_to`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`, :ref:`write_view_to`
+    :ref:`alt_msg`, :ref:`append_to`, :ref:`cd`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`, :ref:`write_view_to`
 
 ----
 
@@ -1456,7 +1488,7 @@
       :write-csv-to /tmp/table.csv
 
   **See Also**
-    :ref:`alt_msg`, :ref:`append_to`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`, :ref:`write_view_to`
+    :ref:`alt_msg`, :ref:`append_to`, :ref:`cd`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`, :ref:`write_view_to`
 
 ----
 
@@ -1480,7 +1512,7 @@
       :write-json-to /tmp/table.json
 
   **See Also**
-    :ref:`alt_msg`, :ref:`append_to`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`, :ref:`write_view_to`
+    :ref:`alt_msg`, :ref:`append_to`, :ref:`cd`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`, :ref:`write_view_to`
 
 ----
 
@@ -1504,7 +1536,7 @@
       :write-jsonlines-to /tmp/table.json
 
   **See Also**
-    :ref:`alt_msg`, :ref:`append_to`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`, :ref:`write_view_to`
+    :ref:`alt_msg`, :ref:`append_to`, :ref:`cd`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`, :ref:`write_view_to`
 
 ----
 
@@ -1529,7 +1561,7 @@
       :write-raw-to /tmp/table.txt
 
   **See Also**
-    :ref:`alt_msg`, :ref:`append_to`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`, :ref:`write_view_to`
+    :ref:`alt_msg`, :ref:`append_to`, :ref:`cd`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`, :ref:`write_view_to`
 
 ----
 
@@ -1553,7 +1585,7 @@
       :write-screen-to /tmp/table.txt
 
   **See Also**
-    :ref:`alt_msg`, :ref:`append_to`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`, :ref:`write_view_to`
+    :ref:`alt_msg`, :ref:`append_to`, :ref:`cd`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`, :ref:`write_view_to`, :ref:`write_view_to`, :ref:`write_view_to`
 
 ----
 
@@ -1577,7 +1609,7 @@
       :write-to /tmp/interesting-lines.txt
 
   **See Also**
-    :ref:`alt_msg`, :ref:`append_to`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_view_to`, :ref:`write_view_to`
+    :ref:`alt_msg`, :ref:`append_to`, :ref:`cd`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_view_to`, :ref:`write_view_to`
 
 ----
 
@@ -1601,7 +1633,7 @@
       :write-view-to /tmp/table.txt
 
   **See Also**
-    :ref:`alt_msg`, :ref:`append_to`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`
+    :ref:`alt_msg`, :ref:`append_to`, :ref:`cd`, :ref:`create_logline_table`, :ref:`create_search_table`, :ref:`echo`, :ref:`echo`, :ref:`echoln`, :ref:`eval`, :ref:`export_session_to`, :ref:`export_session_to`, :ref:`pipe_line_to`, :ref:`pipe_to`, :ref:`rebuild`, :ref:`redirect_to`, :ref:`redirect_to`, :ref:`sh`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_csv_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_json_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_jsonlines_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_raw_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_screen_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_table_to`, :ref:`write_to`, :ref:`write_to`
 
 ----
 

src/line_buffer.cc

@@ -59,6 +59,7 @@
 #include "fmtlib/fmt/format.h"
 #include "line_buffer.hh"
 #include "lnav_util.hh"
+#include "scn/scn.h"
 
 using namespace std::chrono_literals;
 
@@ -408,7 +409,12 @@ line_buffer::set_fd(auto_fd& fd)
             char gz_id[2 + 1 + 1 + 4];
 
             if (pread(fd, gz_id, sizeof(gz_id), 0) == sizeof(gz_id)) {
-                if (gz_id[0] == '\037' && gz_id[1] == '\213') {
+                if (gz_id[0] == 'L' && gz_id[1] == 0 && gz_id[2] == 'N'
+                    && gz_id[3] == 1 && gz_id[4] == 0)
+                {
+                    this->lb_line_metadata = true;
+                    this->lb_file_offset = 8;
+                } else if (gz_id[0] == '\037' && gz_id[1] == '\213') {
                     int gzfd = dup(fd);
 
                     log_perror(fcntl(gzfd, F_SETFD, FD_CLOEXEC));
@@ -1022,11 +1028,16 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length)
 Result<line_info, std::string>
 line_buffer::load_next_line(file_range prev_line)
 {
+    const char* line_start = nullptr;
     bool done = false;
     line_info retval;
 
     require(this->lb_fd != -1);
 
+    if (this->lb_line_metadata && prev_line.fr_offset == 0) {
+        prev_line.fr_offset = 8;
+    }
+
     auto offset = prev_line.next_offset();
     ssize_t request_size = INITIAL_REQUEST_SIZE;
     retval.li_file_range.fr_offset = offset;
@@ -1044,9 +1055,17 @@ line_buffer::load_next_line(file_range prev_line)
             return Ok(retval);
         }
     }
+    if (prev_line.next_offset() == 0) {
+        auto is_utf_res = is_utf8(string_fragment::from_bytes(
+            this->lb_buffer.begin(), this->lb_buffer.size()));
+        this->lb_is_utf8 = is_utf_res.is_valid();
+        if (!this->lb_is_utf8) {
+            log_warning("input is not utf8 -- %s", is_utf_res.usr_message);
+        }
+    }
     while (!done) {
         auto old_retval_size = retval.li_file_range.fr_size;
-        const char *line_start, *lf = nullptr;
+        const char* lf = nullptr;
 
         /* Find the data in the cache and */
         line_start = this->get_range(offset, retval.li_file_range.fr_size);
@@ -1177,11 +1196,29 @@ line_buffer::load_next_line(file_range prev_line)
         = retval.li_utf8_scan_result.usr_has_ansi;
     retval.li_file_range.fr_metadata.m_valid_utf
         = retval.li_utf8_scan_result.is_valid();
+
+    if (this->lb_line_metadata) {
+        auto sv = scn::string_view{
+            line_start,
+            (size_t) retval.li_file_range.fr_size,
+        };
+
+        char level;
+        auto scan_res = scn::scan(sv,
+                                  "{}.{}:{};",
+                                  retval.li_timestamp.tv_sec,
+                                  retval.li_timestamp.tv_usec,
+                                  level);
+        if (scan_res) {
+            retval.li_level = abbrev2level(&level, 1);
+        }
+    }
+
     return Ok(retval);
 }
 
 Result<shared_buffer_ref, std::string>
-line_buffer::read_range(const file_range fr)
+line_buffer::read_range(file_range fr)
 {
     shared_buffer_ref retval;
     const char* line_start;
@@ -1214,6 +1251,15 @@ line_buffer::read_range(const file_range fr)
         return Err(fmt::format(
             FMT_STRING("short-read (need: {}; avail: {})"), fr.fr_size, avail));
     }
+    if (this->lb_line_metadata) {
+        auto new_start
+            = static_cast<const char*>(memchr(line_start, ';', fr.fr_size));
+        if (new_start) {
+            auto offset = new_start - line_start + 1;
+            line_start += offset;
+            fr.fr_size -= offset;
+        }
+    }
     retval.share(this->lb_share_manager, line_start, fr.fr_size);
     retval.get_metadata() = fr.fr_metadata;
 

src/line_buffer.hh

@@ -48,11 +48,16 @@
 #include "base/is_utf8.hh"
 #include "base/lnav_log.hh"
 #include "base/result.h"
+#include "log_level.hh"
 #include "safe/safe.h"
 #include "shared_buffer.hh"
 
 struct line_info {
     file_range li_file_range;
+    struct timeval li_timestamp {
+        0, 0
+    };
+    log_level_t li_level{LEVEL_UNKNOWN};
     bool li_partial{false};
     utf8_scan_result li_utf8_scan_result{};
 };
@@ -175,6 +180,10 @@ public:
 
     bool is_compressed() const { return this->lb_compressed; }
 
+    bool is_header_utf8() const { return this->lb_is_utf8; }
+
+    bool has_line_metadata() const { return this->lb_line_metadata; }
+
     file_off_t get_read_offset(file_off_t off) const
     {
         if (this->is_compressed()) {
@@ -331,6 +340,7 @@ private:
     auto_fd lb_fd; /*< The file to read data from. */
     safe_gz_indexed lb_gz_file; /*< File reader for gzipped files. */
     bool lb_bz_file{false}; /*< Flag set for bzip2 compressed files. */
+    bool lb_line_metadata{false};
 
     auto_buffer lb_buffer{auto_buffer::alloc(DEFAULT_LINE_BUFFER_SIZE)};
     nonstd::optional<auto_buffer> lb_alt_buffer;
@@ -355,6 +365,7 @@ private:
     time_t lb_file_time{0};
     bool lb_seekable{false}; /*< Flag set for seekable file descriptors. */
     bool lb_compressed{false};
+    bool lb_is_utf8{true};
     file_off_t lb_last_line_offset{-1}; /*< */
 
     std::vector<uint32_t> lb_line_starts;

src/lnav.cc

@@ -114,7 +114,7 @@
 #include "log_vtab_impl.hh"
 #include "logfile.hh"
 #include "logfile_sub_source.hh"
-#include "piper_proc.hh"
+#include "piper.looper.hh"
 #include "readline_curses.hh"
 #include "readline_highlighters.hh"
 #include "regexp_vtab.hh"
@@ -218,8 +218,6 @@ static const std::vector<std::string> DEFAULT_DB_KEY_NAMES = {
     "st_gid",
 };
 
-const static file_ssize_t MAX_STDIN_CAPTURE_SIZE = 10 * 1024 * 1024;
-
 static auto bound_pollable_supervisor
     = injector::bind<pollable_supervisor>::to_singleton();
 
@@ -543,11 +541,14 @@ usage()
 
     ex3_term.append(lnav::roles::ok("$"))
         .append(" ")
-        .append(lnav::roles::file("make"))
-        .append(" 2>&1 | ")
         .append(lnav::roles::file("lnav"))
         .append(" ")
-        .append("-t"_symbol)
+        .append("-e"_symbol)
+        .append(" '")
+        .append(lnav::roles::file("make"))
+        .append(" ")
+        .append("-j4"_symbol)
+        .append("' ")
         .pad_to(40)
         .with_attr_for_all(VC_ROLE.value(role_t::VCR_QUOTED_CODE));
 
@@ -623,19 +624,6 @@ make it easier to navigate through files quickly.
         .append("         ")
         .append("Load older rotated log files as well.\n")
         .append("  ")
-        .append("-t"_symbol)
-        .append("         ")
-        .append(R"(Prepend timestamps to the lines of data being read in
-             from the standard input.
-)")
-        .append("  ")
-        .append("-w"_symbol)
-        .append(" ")
-        .append("file"_variable)
-        .append("    ")
-        .append("Write the contents of the standard input to this file.\n")
-        .append("\n")
-        .append("  ")
         .append("-c"_symbol)
         .append(" ")
         .append("cmd"_variable)
@@ -648,6 +636,12 @@ make it easier to navigate through files quickly.
         .append("    ")
         .append("Execute the commands in the given file.\n")
         .append("  ")
+        .append("-e"_symbol)
+        .append(" ")
+        .append("cmd"_variable)
+        .append("     ")
+        .append("Execute a shell command-line.\n")
+        .append("  ")
         .append("-n"_symbol)
         .append("         ")
         .append("Run without the curses UI. (headless mode)\n")
@@ -704,7 +698,7 @@ make it easier to navigate through files quickly.
         .append("\u2022"_list_glyph)
         .append(" To watch the output of ")
         .append(lnav::roles::file("make"))
-        .append(" with timestamps prepended:\n")
+        .append(":\n")
         .append("     ")
         .append(ex3_term)
         .append("\n\n")
@@ -723,8 +717,8 @@ make it easier to navigate through files quickly.
         .append(lnav::roles::file(lnav::paths::dotlnav().string()))
         .append("\n\n ")
         .append("\u2022"_list_glyph)
-        .append(" Local copies of remote files and files extracted from\n")
+        .append(" Local copies of remote files, files extracted from\n")
-        .append("   archives are stored in:\n")
+        .append("   archives, execution output, and so on are stored in:\n")
         .append("    \U0001F4C2 ")
         .append(lnav::roles::file(lnav::paths::workdir().string()))
         .append("\n\n")
@@ -982,18 +976,6 @@ match_escape_seq(const char* keyseq)
 static void
 gather_pipers()
 {
-    for (auto iter = lnav_data.ld_pipers.begin();
-         iter != lnav_data.ld_pipers.end();)
-    {
-        pid_t child_pid = (*iter)->get_child_pid();
-        if ((*iter)->has_exited()) {
-            log_info("child piper has exited -- %d", child_pid);
-            iter = lnav_data.ld_pipers.erase(iter);
-        } else {
-            ++iter;
-        }
-    }
-
     for (auto iter = lnav_data.ld_child_pollers.begin();
          iter != lnav_data.ld_child_pollers.end();)
     {
@@ -1010,18 +992,25 @@ gather_pipers()
 static void
 wait_for_pipers()
 {
+    static const auto MAX_SLEEP_TIME = std::chrono::milliseconds(300);
+    auto sleep_time = std::chrono::milliseconds(10);
+
     for (;;) {
         gather_pipers();
-        if (lnav_data.ld_pipers.empty() && lnav_data.ld_child_pollers.empty()) {
+        auto piper_count = lnav_data.ld_active_files.active_pipers();
+        if (piper_count == 0 && lnav_data.ld_child_pollers.empty()) {
             log_debug("all pipers finished");
             break;
         }
-        usleep(10000);
+        std::this_thread::sleep_for(sleep_time);
         rebuild_indexes();
 
-        log_debug("%d pipers and %d children still active",
+        log_debug("%d pipers and %d children are still active",
-                  lnav_data.ld_pipers.size(),
+                  piper_count,
                   lnav_data.ld_child_pollers.size());
+        if (sleep_time < MAX_SLEEP_TIME) {
+            sleep_time = sleep_time * 2;
+        }
     }
 }
 
@@ -2106,12 +2095,6 @@ enum class verbosity_t : int {
     verbose,
 };
 
-struct stdin_options_t {
-    ghc::filesystem::path so_out;
-    bool so_timestamp{false};
-    auto_fd so_out_fd;
-};
-
 int
 main(int argc, char* argv[])
 {
@@ -2120,12 +2103,9 @@ main(int argc, char* argv[])
     exec_context& ec = lnav_data.ld_exec_context;
     int retval = EXIT_SUCCESS;
 
-    std::shared_ptr<piper_proc> stdin_reader;
+    bool exec_stdin = false, load_stdin = false;
-    stdin_options_t stdin_opts;
-    bool exec_stdin = false, load_stdin = false, stdin_captured = false;
     mode_flags_t mode_flags;
     const char* LANG = getenv("LANG");
-    ghc::filesystem::path stdin_tmp_path;
     verbosity_t verbosity = verbosity_t::standard;
 
     if (LANG == nullptr || strcmp(LANG, "C") == 0) {
@@ -2314,9 +2294,6 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         auto* install_flag
             = app.add_flag("-i", mode_flags.mf_install, "install");
         app.add_flag("-u", mode_flags.mf_update_formats, "update");
-        auto* write_flag = app.add_option("-w", stdin_opts.so_out, "write");
-        auto* ts_flag
-            = app.add_flag("-t", stdin_opts.so_timestamp, "timestamp");
         auto* no_default_flag
             = app.add_flag("-N", mode_flags.mf_no_default, "no def");
         auto* rotated_flag = app.add_flag(
@@ -2391,15 +2368,24 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                                   ->allow_extra_args(false)
                                   ->each(file_appender);
 
+        auto shexec_appender = [&mode_flags](std::string cmd) {
+            mode_flags.mf_no_default = true;
+            lnav_data.ld_commands.emplace_back(
+                fmt::format(FMT_STRING(":sh {}"), cmd));
+        };
+        auto* cmdline_opt = app.add_option("-e")
+                                ->each(shexec_appender)
+                                ->allow_extra_args(false)
+                                ->trigger_on_parse(true);
+
         install_flag->needs(file_opt);
-        install_flag->excludes(write_flag,
+        install_flag->excludes(no_default_flag,
-                               ts_flag,
-                               no_default_flag,
                                rotated_flag,
                                recurse_flag,
                                headless_flag,
                                cmd_opt,
-                               exec_file_opt);
+                               exec_file_opt,
+                               cmdline_opt);
     }
 
     auto is_mmode = argc >= 2 && strcmp(argv[1], "-m") == 0;
@@ -2668,6 +2654,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
     log_fos->fos_contexts.emplace("", false, true);
     lnav_data.ld_views[LNV_LOG]
         .set_sub_source(&lnav_data.ld_log_source)
+#if 0
         .set_delegate(std::make_shared<action_delegate>(
             lnav_data.ld_log_source,
             [](auto child_pid) { lnav_data.ld_children.push_back(child_pid); },
@@ -2677,11 +2664,14 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                     pp->get_fd());
                 lnav_data.ld_files_to_front.template emplace_back(desc, 0_vl);
             }))
+#endif
         .add_input_delegate(lnav_data.ld_log_source)
         .set_tail_space(2_vl)
         .set_overlay_source(log_fos);
     auto sel_reload_delegate = [](textview_curses& tc) {
-        if (lnav_config.lc_ui_movement.mode == config_movement_mode::CURSOR) {
+        if (!(lnav_data.ld_flags & LNF_HEADLESS)
+            && lnav_config.lc_ui_movement.mode == config_movement_mode::CURSOR)
+        {
             tc.set_selectable(true);
         }
     };
@@ -2803,6 +2793,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
     lnav_data.ld_mode = ln_mode_t::PAGING;
 
     if ((isatty(STDIN_FILENO) || is_dev_null(STDIN_FILENO)) && file_args.empty()
+        && lnav_data.ld_active_files.fc_file_names.empty()
         && !mode_flags.mf_no_default)
     {
         char start_dir[FILENAME_MAX];
@@ -2875,8 +2866,8 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         {
             auto ul = std::make_shared<url_loader>(file_path);
 
-            lnav_data.ld_active_files.fc_file_names[file_path].with_fd(
+            lnav_data.ld_active_files.fc_file_names[ul->get_path()]
-                ul->copy_fd());
+                .with_filename(file_path);
             isc::to<curl_looper&, services::curl_streamer_t>().send(
                 [ul](auto& clooper) { clooper.add_request(ul); });
         }
@@ -2918,25 +2909,15 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                         .with_errno_reason());
                 retval = EXIT_FAILURE;
             } else {
-                auto fifo_tmp_fd
-                    = lnav::filesystem::open_temp_file(
-                          ghc::filesystem::temp_directory_path()
-                          / "lnav.fifo.XXXXXX")
-                          .map([](auto&& pair) {
-                              ghc::filesystem::remove(pair.first);
-
-                              return std::move(pair.second);
-                          })
-                          .expect("Cannot create temporary file for FIFO");
-                auto fifo_piper = std::make_shared<piper_proc>(
-                    std::move(fifo_fd), false, std::move(fifo_tmp_fd));
-                auto fifo_out_fd = fifo_piper->get_fd();
                 auto desc = fmt::format(FMT_STRING("FIFO [{}]"),
                                         lnav_data.ld_fifo_counter++);
+                auto create_piper_res = lnav::piper::create_looper(
+                    desc, std::move(fifo_fd), auto_fd{});
 
-                lnav_data.ld_active_files.fc_file_names[desc].with_fd(
+                if (create_piper_res.isOk()) {
-                    std::move(fifo_out_fd));
+                    lnav_data.ld_active_files.fc_file_names[desc].with_piper(
-                lnav_data.ld_pipers.push_back(fifo_piper);
+                        create_piper_res.unwrap());
+                }
             }
         } else if ((abspath = realpath(file_path.c_str(), nullptr)) == nullptr)
         {
@@ -3039,44 +3020,52 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         retval = EXIT_FAILURE;
     }
 
+    nonstd::optional<ghc::filesystem::path> stdin_pattern;
     if (load_stdin && !isatty(STDIN_FILENO) && !is_dev_null(STDIN_FILENO)
         && !exec_stdin)
     {
-        if (stdin_opts.so_out.empty()) {
+        static const std::string STDIN_NAME = "stdin";
-            auto pattern
+        struct stat stdin_st;
-                = lnav::paths::dotlnav() / "stdin-captures/stdin.XXXXXX";
 
-            auto open_result = lnav::filesystem::open_temp_file(pattern);
+        if (fstat(STDIN_FILENO, &stdin_st) == -1) {
-            if (open_result.isErr()) {
+            lnav::console::print(
-                fprintf(stderr,
+                stderr,
-                        "Unable to open temporary file for stdin: %s",
+                lnav::console::user_message::error("unable to stat() stdin")
-                        open_result.unwrapErr().c_str());
+                    .with_errno_reason());
-                return EXIT_FAILURE;
+            retval = EXIT_FAILURE;
+        } else if (S_ISFIFO(stdin_st.st_mode)) {
+            auto stdin_piper_res = lnav::piper::create_looper(
+                STDIN_NAME, auto_fd::dup_of(STDIN_FILENO), auto_fd{});
+            if (stdin_piper_res.isOk()) {
+                auto stdin_piper = stdin_piper_res.unwrap();
+                stdin_pattern = stdin_piper.get_out_pattern();
+                lnav_data.ld_active_files.fc_file_names[stdin_piper.get_name()]
+                    .with_piper(stdin_piper)
+                    .with_include_in_session(false);
             }
+        } else if (S_ISREG(stdin_st.st_mode)) {
+            // The shell connected a file directly, just open it up and add it
+            // in here.
+            auto loo = logfile_open_options{}
+                           .with_filename(STDIN_NAME)
+                           .with_include_in_session(false);
+
+            auto open_res
+                = logfile::open(STDIN_NAME, loo, auto_fd::dup_of(STDIN_FILENO));
 
-            auto temp_pair = open_result.unwrap();
-            stdin_tmp_path = temp_pair.first;
-            stdin_opts.so_out_fd = std::move(temp_pair.second);
-        } else {
-            auto open_res = lnav::filesystem::create_file(
-                stdin_opts.so_out, O_RDWR | O_TRUNC, 0600);
             if (open_res.isErr()) {
-                fmt::print(stderr, "error: {}\n", open_res.unwrapErr());
+                lnav::console::print(
-                return EXIT_FAILURE;
+                    stderr,
-            }
+                    lnav::console::user_message::error("unable to open stdin")
+                        .with_reason(open_res.unwrapErr()));
+                retval = EXIT_FAILURE;
+            } else {
+                file_collection fc;
 
-            stdin_opts.so_out_fd = open_res.unwrap();
+                fc.fc_files.emplace_back(open_res.unwrap());
+                update_active_files(fc);
+            }
         }
-
-        stdin_captured = true;
-        stdin_reader
-            = std::make_shared<piper_proc>(auto_fd(STDIN_FILENO),
-                                           stdin_opts.so_timestamp,
-                                           std::move(stdin_opts.so_out_fd));
-        lnav_data.ld_active_files.fc_file_names["stdin"]
-            .with_fd(stdin_reader->get_fd())
-            .with_include_in_session(false);
-        lnav_data.ld_pipers.push_back(stdin_reader);
     }
 
     if (!isatty(STDIN_FILENO) && isatty(STDOUT_FILENO)) {
@@ -3085,7 +3074,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         }
     }
 
-    if (retval == EXIT_SUCCESS
+    if (retval == EXIT_SUCCESS && lnav_data.ld_active_files.fc_files.empty()
         && lnav_data.ld_active_files.fc_file_names.empty()
         && lnav_data.ld_commands.empty()
         && !(lnav_data.ld_show_help_view || mode_flags.mf_no_default))
@@ -3155,6 +3144,9 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
 
                 view_colors::init(true);
                 rescan_files(true);
+                wait_for_pipers();
+                rescan_files(true);
+                rebuild_indexes_repeatedly();
                 if (!lnav_data.ld_active_files.fc_name_to_errors.empty()) {
                     for (const auto& pair :
                          lnav_data.ld_active_files.fc_name_to_errors)
@@ -3198,6 +3190,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                 tailer::cleanup_cache();
                 line_buffer::cleanup_cache();
                 wait_for_pipers();
+                rescan_files(true);
                 isc::to<curl_looper&, services::curl_streamer_t>()
                     .send_and_wait(
                         [](auto& clooper) { clooper.process_all(); });
@@ -3332,54 +3325,26 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
 
         // When reading from stdin, tell the user where the capture file is
         // stored so they can look at it later.
-        if (stdin_captured && stdin_opts.so_out.empty()
+        if (stdin_pattern && !(lnav_data.ld_flags & LNF_HEADLESS)) {
-            && !(lnav_data.ld_flags & LNF_HEADLESS))
+            file_size_t stdin_size = 0;
-        {
+            for (const auto& ent : ghc::filesystem::directory_iterator(
-            auto stdin_fd = stdin_reader->get_fd();
+                     stdin_pattern.value().parent_path()))
-            struct stat stdin_stat;
-            nonstd::optional<file_ssize_t> stdin_size;
-
-            // NB: the file can be deleted by the time we get here
-            fchmod(stdin_fd.get(), S_IRUSR);
-            if (fstat(stdin_fd.get(), &stdin_stat) != -1) {
-                stdin_size = stdin_stat.st_size;
-            }
-            if (!ghc::filesystem::exists(stdin_tmp_path)
-                || verbosity == verbosity_t::quiet || !stdin_size
-                || stdin_size.value() == 0
-                || stdin_size.value() > MAX_STDIN_CAPTURE_SIZE)
             {
-                std::error_code rm_err_code;
+                stdin_size += ent.file_size();
-
-                log_info("not saving stdin capture -- %s (size=%d)",
-                         stdin_tmp_path.c_str(),
-                         stdin_size.value_or(-1));
-                ghc::filesystem::remove(stdin_tmp_path, rm_err_code);
-            } else {
-                auto home = getenv_opt("HOME");
-                auto path_str = stdin_tmp_path.string();
-
-                if (home && startswith(path_str, home.value())) {
-                    path_str = path_str.substr(strlen(home.value()));
-                    if (path_str[0] != '/') {
-                        path_str.insert(0, 1, '/');
-                    }
-                    path_str.insert(0, 1, '~');
-                }
-
-                lnav::console::print(
-                    stderr,
-                    lnav::console::user_message::info(
-                        attr_line_t()
-                            .append(lnav::roles::number(humanize::file_size(
-                                stdin_size.value(), humanize::alignment::none)))
-                            .append(" of data from stdin was captured and "
-                                    "will be saved for one day.  You can "
-                                    "reopen it by running:\n")
-                            .appendf(FMT_STRING("   {} "),
-                                     lnav_data.ld_program_name)
-                            .append(lnav::roles::file(path_str))));
             }
+
+            lnav::console::print(
+                stderr,
+                lnav::console::user_message::info(
+                    attr_line_t()
+                        .append(lnav::roles::number(humanize::file_size(
+                            stdin_size, humanize::alignment::none)))
+                        .append(" of data from stdin was captured and "
+                                "will be saved for one day.  You can "
+                                "reopen it by running:\n")
+                        .appendf(FMT_STRING("   {} "),
+                                 lnav_data.ld_program_name)
+                        .append(lnav::roles::file(stdin_pattern.value()))));
         }
     }
 

src/lnav.hh

@@ -62,7 +62,6 @@
 #include "log_format_loader.hh"
 #include "log_vtab_impl.hh"
 #include "logfile.hh"
-#include "piper_proc.hh"
 #include "plain_text_source.hh"
 #include "preview_status_source.hh"
 #include "readline_curses.hh"
@@ -243,7 +242,6 @@ struct lnav_data_t {
     std::unordered_map<std::string, std::string> ld_table_ddl;
 
     std::list<pid_t> ld_children;
-    std::list<std::shared_ptr<piper_proc>> ld_pipers;
 
     input_state_tracker ld_input_state;
     input_dispatcher ld_input_dispatcher;

src/lnav_commands.cc

@@ -2528,8 +2528,8 @@ com_open(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
                 if (!ec.ec_dry_run) {
                     auto ul = std::make_shared<url_loader>(fn);
 
-                    lnav_data.ld_active_files.fc_file_names[fn].with_fd(
+                    lnav_data.ld_active_files.fc_file_names[ul->get_path()]
-                        ul->copy_fd());
+                        .with_filename(fn);
                     isc::to<curl_looper&, services::curl_streamer_t>().send(
                         [ul](auto& clooper) { clooper.add_request(ul); });
                     lnav_data.ld_files_to_front.emplace_back(fn, file_loc);
@@ -2571,25 +2571,20 @@ com_open(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
                 } else if (ec.ec_dry_run) {
                     retval = "";
                 } else {
-                    auto fifo_piper = std::make_shared<piper_proc>(
-                        std::move(fifo_fd),
-                        false,
-                        lnav::filesystem::open_temp_file(
-                            ghc::filesystem::temp_directory_path()
-                            / "lnav.fifo.XXXXXX")
-                            .map([](auto pair) {
-                                ghc::filesystem::remove(pair.first);
-
-                                return pair;
-                            })
-                            .expect("Cannot create temporary file for FIFO")
-                            .second);
-                    auto fifo_out_fd = fifo_piper->get_fd();
                     auto desc = fmt::format(FMT_STRING("FIFO [{}]"),
                                             lnav_data.ld_fifo_counter++);
-                    lnav_data.ld_active_files.fc_file_names[desc].with_fd(
+                    auto create_piper_res = lnav::piper::create_looper(
-                        std::move(fifo_out_fd));
+                        desc, std::move(fifo_fd), auto_fd{});
-                    lnav_data.ld_pipers.push_back(fifo_piper);
+                    if (create_piper_res.isErr()) {
+                        auto um = lnav::console::user_message::error(
+                                      attr_line_t("cannot create piper: ")
+                                          .append(lnav::roles::file(fn)))
+                                      .with_reason(create_piper_res.unwrapErr())
+                                      .with_snippets(ec.ec_source);
+                        return Err(um);
+                    }
+                    lnav_data.ld_active_files.fc_file_names[desc].with_piper(
+                        create_piper_res.unwrap());
                 }
             } else if ((abspath = realpath(fn.c_str(), nullptr)) == nullptr) {
                 auto um = lnav::console::user_message::error(
@@ -4084,6 +4079,127 @@ com_rebuild(exec_context& ec,
     return Ok(std::string());
 }
 
+static Result<std::string, lnav::console::user_message>
+com_cd(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
+{
+    if (args.empty()) {
+        args.emplace_back("dirname");
+        return Ok(std::string());
+    }
+
+    if (lnav_data.ld_flags & LNF_SECURE_MODE) {
+        return ec.make_error("{} -- unavailable in secure mode", args[0]);
+    }
+
+    std::vector<std::string> word_exp;
+    std::string pat;
+
+    pat = trim(remaining_args(cmdline, args));
+
+    std::vector<std::string> split_args;
+    shlex lexer(pat);
+    scoped_resolver scopes = {
+        &ec.ec_local_vars.top(),
+        &ec.ec_global_vars,
+    };
+
+    if (!lexer.split(split_args, scopes)) {
+        return ec.make_error("unable to parse arguments");
+    }
+
+    if (split_args.size() != 1) {
+        return ec.make_error("expecting a single argument");
+    }
+
+    struct stat st;
+
+    if (stat(split_args[0].c_str(), &st) != 0) {
+        return Err(ec.make_error_msg("cannot access -- {}", split_args[0])
+                       .with_errno_reason());
+    }
+
+    if (!S_ISDIR(st.st_mode)) {
+        return ec.make_error("{} is not a directory", split_args[0]);
+    }
+
+    if (!ec.ec_dry_run) {
+        chdir(split_args[0].c_str());
+    }
+
+    return Ok(std::string());
+}
+
+static Result<std::string, lnav::console::user_message>
+com_sh(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
+{
+    if (args.empty()) {
+        return Ok(std::string());
+    }
+
+    if (lnav_data.ld_flags & LNF_SECURE_MODE) {
+        return ec.make_error("{} -- unavailable in secure mode", args[0]);
+    }
+
+    if (!ec.ec_dry_run) {
+        auto carg = trim(cmdline.substr(args[0].size()));
+
+        log_info("executing: %s", carg.c_str());
+
+        auto out_pipe_res = auto_pipe::for_child_fd(STDOUT_FILENO);
+        auto err_pipe_res = auto_pipe::for_child_fd(STDERR_FILENO);
+        auto child_res = lnav::pid::from_fork();
+        if (child_res.isErr()) {
+            auto um
+                = lnav::console::user_message::error("unable to fork() child")
+                      .with_reason(child_res.unwrapErr());
+            ec.add_error_context(um);
+            return Err(um);
+        }
+
+        auto out_pipe = out_pipe_res.unwrap();
+        auto err_pipe = err_pipe_res.unwrap();
+
+        auto child = child_res.unwrap();
+        out_pipe.after_fork(child.in());
+        err_pipe.after_fork(child.in());
+        if (child.in_child()) {
+            auto dev_null = open("/dev/null", O_RDONLY);
+
+            dup2(dev_null, STDIN_FILENO);
+            const char* exec_args[] = {
+                getenv_opt("SHELL").value_or("bash"),
+                "-c",
+                carg.c_str(),
+                nullptr,
+            };
+
+            execvp(exec_args[0], (char**) exec_args);
+            _exit(EXIT_FAILURE);
+        }
+
+        auto create_piper_res
+            = lnav::piper::create_looper(carg,
+                                         std::move(out_pipe.read_end()),
+                                         std::move(err_pipe.read_end()));
+
+        if (create_piper_res.isErr()) {
+            auto um
+                = lnav::console::user_message::error("unable to create piper")
+                      .with_reason(child_res.unwrapErr());
+            ec.add_error_context(um);
+            return Err(um);
+        }
+
+        lnav_data.ld_active_files.fc_file_names[carg].with_piper(
+            create_piper_res.unwrap());
+        lnav_data.ld_child_pollers.emplace_back(
+            child_poller{std::move(child), [](auto& fc, auto& child) {}});
+        lnav_data.ld_files_to_front.emplace_back(carg, file_location_t{});
+    }
+
+    return Ok(std::string());
+}
+
 static Result<std::string, lnav::console::user_message>
 com_shexec(exec_context& ec,
            std::string cmdline,
@@ -5691,6 +5807,29 @@ readline_context::command_t STD_COMMANDS[] = {
          .with_tags({"scripting"})
          .with_examples({{"To substitute the table name from a variable",
                           ";SELECT * FROM ${table}"}})},
+
+    {
+        "sh",
+        com_sh,
+
+        help_text(":sh")
+            .with_summary("Execute the given command-line and display the "
+                          "captured output")
+            .with_parameter(
+                help_text("cmdline", "The command-line to execute."))
+            .with_tags({"scripting"}),
+    },
+
+    {
+        "cd",
+        com_cd,
+
+        help_text(":cd")
+            .with_summary("Change the current directory")
+            .with_parameter(help_text("dir", "The new current directory"))
+            .with_tags({"scripting"}),
+    },
+
     {"config",
      com_config,
 

src/lnav_config.cc

@@ -87,6 +87,9 @@ static auto fvc = injector::bind<file_vtab::config>::to_instance(
 static auto lc = injector::bind<lnav::logfile::config>::to_instance(
     +[]() { return &lnav_config.lc_logfile; });
 
+static auto p = injector::bind<lnav::piper::config>::to_instance(
+    +[]() { return &lnav_config.lc_piper; });
+
 static auto tc = injector::bind<tailer::config>::to_instance(
     +[]() { return &lnav_config.lc_tailer; });
 
@@ -1057,6 +1060,19 @@ static const struct json_path_container archive_handlers = {
                    &archive_manager::config::amc_cache_ttl),
 };
 
+static const struct json_path_container piper_handlers = {
+    yajlpp::property_handler("max-size")
+        .with_synopsis("<bytes>")
+        .with_description("The maximum size of a capture file")
+        .with_min_value(128)
+        .for_field(&_lnav_config::lc_piper, &lnav::piper::config::c_max_size),
+    yajlpp::property_handler("rotations")
+        .with_synopsis("<count>")
+        .with_min_value(2)
+        .with_description("The number of rotated files to keep")
+        .for_field(&_lnav_config::lc_piper, &lnav::piper::config::c_rotations),
+};
+
 static const struct json_path_container file_vtab_handlers = {
     yajlpp::property_handler("max-content-size")
         .with_synopsis("<bytes>")
@@ -1245,6 +1261,9 @@ static const struct json_path_container tuning_handlers = {
     yajlpp::property_handler("archive-manager")
         .with_description("Settings related to opening archive files")
         .with_children(archive_handlers),
+    yajlpp::property_handler("piper")
+        .with_description("Settings related to capturing piped data")
+        .with_children(piper_handlers),
     yajlpp::property_handler("file-vtab")
         .with_description("Settings related to the lnav_file virtual-table")
         .with_children(file_vtab_handlers),

src/lnav_config.hh

@@ -49,6 +49,7 @@
 #include "log_level.hh"
 #include "logfile.cfg.hh"
 #include "logfile_sub_source.cfg.hh"
+#include "piper.looper.cfg.hh"
 #include "styling.hh"
 #include "sysclip.cfg.hh"
 #include "tailer/tailer.looper.cfg.hh"
@@ -109,6 +110,7 @@ struct _lnav_config {
     key_map lc_active_keymap;
 
     archive_manager::config lc_archive_manager;
+    lnav::piper::config lc_piper;
     file_vtab::config lc_file_vtab;
     lnav::logfile::config lc_logfile;
     tailer::config lc_tailer;

src/log_actions.cc

@@ -27,13 +27,14 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include "log_actions.hh"
+#if 0
+#    include "log_actions.hh"
 
-#include "base/fs_util.hh"
+#    include "base/fs_util.hh"
-#include "base/injector.hh"
+#    include "base/injector.hh"
-#include "bound_tags.hh"
+#    include "bound_tags.hh"
-#include "config.h"
+#    include "config.h"
-#include "piper_proc.hh"
+#    include "piper_proc.hh"
 
 std::string
 action_delegate::execute_action(const std::string& action_name)
@@ -235,3 +236,4 @@ action_delegate::text_handle_mouse(textview_curses& tc, mouse_event& me)
 
     return retval;
 }
+#endif

src/log_actions.hh

@@ -30,11 +30,12 @@
 #ifndef log_actions_hh
 #define log_actions_hh
 
-#include <functional>
+#if 0
-#include <utility>
+#    include <functional>
+#    include <utility>
 
-#include "log_data_helper.hh"
+#    include "log_data_helper.hh"
-#include "logfile_sub_source.hh"
+#    include "logfile_sub_source.hh"
 
 class piper_proc;
 
@@ -63,5 +64,6 @@ private:
     int ad_press_value{-1};
     size_t ad_line_index{0};
 };
+#endif
 
 #endif

src/logfile.cc

@@ -68,16 +68,16 @@ static const typed_json_path_container<line_buffer::header_data>
 };
 
 Result<std::shared_ptr<logfile>, std::string>
-logfile::open(std::string filename, logfile_open_options& loo)
+logfile::open(std::string filename, const logfile_open_options& loo, auto_fd fd)
 {
     require(!filename.empty());
 
     auto lf = std::shared_ptr<logfile>(new logfile(std::move(filename), loo));
 
     memset(&lf->lf_stat, 0, sizeof(lf->lf_stat));
-    if (lf->lf_options.loo_fd == -1) {
+    char resolved_path[PATH_MAX] = "";
-        char resolved_path[PATH_MAX];
 
+    if (!fd.has_value()) {
         errno = 0;
         if (realpath(lf->lf_filename.c_str(), resolved_path) == nullptr) {
             return Err(fmt::format(FMT_STRING("realpath({}) failed with: {}"),
@@ -96,37 +96,36 @@ logfile::open(std::string filename, logfile_open_options& loo)
                                    lf->lf_filename,
                                    strerror(errno)));
         }
+    }
 
-        if ((lf->lf_options.loo_fd = ::open(resolved_path, O_RDONLY)) == -1) {
+    auto_fd lf_fd;
-            return Err(fmt::format(FMT_STRING("open({}) failed with: {}"),
+    if (fd.has_value()) {
-                                   lf->lf_filename,
+        lf_fd = std::move(fd);
-                                   strerror(errno)));
+    } else if ((lf_fd = ::open(resolved_path, O_RDONLY)) == -1) {
-        }
+        return Err(fmt::format(FMT_STRING("open({}) failed with: {}"),
-
+                               lf->lf_filename,
-        lf->lf_options.loo_fd.close_on_exec();
+                               strerror(errno)));
-
+    } else {
-        log_info("Creating logfile: fd=%d; size=%" PRId64 "; mtime=%" PRId64
-                 "; filename=%s",
-                 (int) lf->lf_options.loo_fd,
-                 (long long) lf->lf_stat.st_size,
-                 (long long) lf->lf_stat.st_mtime,
-                 lf->lf_filename.c_str());
-
         lf->lf_actual_path = lf->lf_filename;
         lf->lf_valid_filename = true;
-    } else {
-        log_perror(fstat(lf->lf_options.loo_fd, &lf->lf_stat));
-        lf->lf_named_file = false;
-        lf->lf_valid_filename = false;
     }
 
+    lf_fd.close_on_exec();
+
+    log_info("Creating logfile: fd=%d; size=%" PRId64 "; mtime=%" PRId64
+             "; filename=%s",
+             (int) lf_fd,
+             (long long) lf->lf_stat.st_size,
+             (long long) lf->lf_stat.st_mtime,
+             lf->lf_filename.c_str());
+
     if (!lf->lf_options.loo_filename.empty()) {
         lf->set_filename(lf->lf_options.loo_filename);
         lf->lf_valid_filename = false;
     }
 
     lf->lf_content_id = hasher().update(lf->lf_filename).to_string();
-    lf->lf_line_buffer.set_fd(lf->lf_options.loo_fd);
+    lf->lf_line_buffer.set_fd(lf_fd);
     lf->lf_index.reserve(INDEX_RESERVE_INCREMENT);
 
     lf->lf_indexing = lf->lf_options.loo_is_visible;
@@ -142,8 +141,8 @@ logfile::open(std::string filename, logfile_open_options& loo)
     return Ok(lf);
 }
 
-logfile::logfile(std::string filename, logfile_open_options& loo)
+logfile::logfile(std::string filename, const logfile_open_options& loo)
-    : lf_filename(std::move(filename)), lf_options(std::move(loo))
+    : lf_filename(std::move(filename)), lf_options(loo)
 {
     this->lf_opids.writeAccess()->reserve(64);
 }
@@ -416,8 +415,15 @@ logfile::process_prefix(shared_buffer_ref& sbr,
         short last_millis = 0;
         uint8_t last_mod = 0, last_opid = 0;
 
-        if (!this->lf_index.empty()) {
+        if (this->lf_format == nullptr && li.li_timestamp.tv_sec != 0) {
-            logline& ll = this->lf_index.back();
+            last_time = li.li_timestamp.tv_sec;
+            last_millis
+                = std::chrono::duration_cast<std::chrono::milliseconds>(
+                      std::chrono::microseconds(li.li_timestamp.tv_usec))
+                      .count();
+            last_level = li.li_level;
+        } else if (!this->lf_index.empty()) {
+            const auto& ll = this->lf_index.back();
 
             /*
              * Assume this line is part of the previous one(s) and copy the

src/logfile.hh

@@ -114,7 +114,9 @@ public:
      * descriptor needs to be seekable.
      */
     static Result<std::shared_ptr<logfile>, std::string> open(
-        std::string filename, logfile_open_options& loo);
+        std::string filename,
+        const logfile_open_options& loo,
+        auto_fd fd = auto_fd{});
 
     ~logfile() override;
 
@@ -146,6 +148,11 @@ public:
 
     bool is_compressed() const { return this->lf_line_buffer.is_compressed(); }
 
+    bool has_line_metadata() const
+    {
+        return this->lf_line_buffer.has_line_metadata();
+    }
+
     bool is_valid_filename() const { return this->lf_valid_filename; }
 
     file_off_t get_index_size() const { return this->lf_index_size; }
@@ -195,6 +202,11 @@ public:
         return this->lf_options;
     }
 
+    void set_include_in_session(bool enabled)
+    {
+        this->lf_options.with_include_in_session(enabled);
+    }
+
     void reset_state();
 
     bool is_time_adjusted() const
@@ -399,7 +411,7 @@ protected:
     void set_format_base_time(log_format* lf);
 
 private:
-    logfile(std::string filename, logfile_open_options& loo);
+    logfile(std::string filename, const logfile_open_options& loo);
 
     std::string lf_filename;
     logfile_open_options lf_options;

src/logfile_fwd.hh

@@ -37,6 +37,7 @@
 
 #include "base/auto_fd.hh"
 #include "file_format.hh"
+#include "piper.looper.hh"
 
 using ui_clock = std::chrono::steady_clock;
 
@@ -65,6 +66,7 @@ struct logfile_open_options_base {
     ssize_t loo_visible_size_limit{-1};
     bool loo_tail{true};
     file_format_t loo_file_format{file_format_t::UNKNOWN};
+    nonstd::optional<lnav::piper::running_handle> loo_piper;
 };
 
 struct logfile_open_options : public logfile_open_options_base {
@@ -82,14 +84,6 @@ struct logfile_open_options : public logfile_open_options_base {
         return *this;
     }
 
-    logfile_open_options& with_fd(auto_fd fd)
-    {
-        this->loo_fd = std::move(fd);
-        this->loo_temp_file = true;
-
-        return *this;
-    }
-
     logfile_open_options& with_stat_for_temp(const struct stat& st)
     {
         this->loo_temp_dev = st.st_dev;
@@ -131,7 +125,7 @@ struct logfile_open_options : public logfile_open_options_base {
         this->loo_non_utf_is_visible = val;
 
         return *this;
-    };
+    }
 
     logfile_open_options& with_visible_size_limit(ssize_t val)
     {
@@ -154,7 +148,13 @@ struct logfile_open_options : public logfile_open_options_base {
         return *this;
     }
 
-    auto_fd loo_fd;
+    logfile_open_options& with_piper(lnav::piper::running_handle handle)
+    {
+        this->loo_piper = handle;
+        this->loo_filename = handle.get_name();
+
+        return *this;
+    }
 };
 
 #endif

src/logfile_sub_source.cc

@@ -324,33 +324,9 @@ logfile_sub_source::text_value_for_line(textview_curses& tc,
         value_out.insert(0, 1, ' ');
     }
 
-    if (this->lss_flags & F_TIME_OFFSET) {
+    if (this->tas_display_time_offset) {
-        auto curr_tv = this->lss_token_line->get_timeval();
-        struct timeval diff_tv;
         auto row_vl = vis_line_t(row);
-
+        auto relstr = this->get_time_offset_for_line(tc, row_vl);
-        auto prev_umark
-            = tc.get_bookmarks()[&textview_curses::BM_USER].prev(row_vl);
-        auto next_umark
-            = tc.get_bookmarks()[&textview_curses::BM_USER].next(row_vl);
-        auto prev_emark
-            = tc.get_bookmarks()[&textview_curses::BM_USER_EXPR].prev(row_vl);
-        auto next_emark
-            = tc.get_bookmarks()[&textview_curses::BM_USER_EXPR].next(row_vl);
-        if (!prev_umark && !prev_emark && (next_umark || next_emark)) {
-            auto next_line = this->find_line(this->at(
-                std::max(next_umark.value_or(0), next_emark.value_or(0))));
-
-            diff_tv = curr_tv - next_line->get_timeval();
-        } else {
-            auto prev_row
-                = std::max(prev_umark.value_or(0), prev_emark.value_or(0));
-            auto first_line = this->find_line(this->at(prev_row));
-            auto start_tv = first_line->get_timeval();
-            diff_tv = curr_tv - start_tv;
-        }
-
-        auto relstr = humanize::time::duration::from_tv(diff_tv).to_string();
         value_out = fmt::format(FMT_STRING("{: >12}|{}"), relstr, value_out);
     }
     this->lss_in_value_for_line = false;
@@ -492,7 +468,7 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv,
                                    this->lss_token_file->get_filename())));
     }
 
-    if (this->lss_flags & F_TIME_OFFSET) {
+    if (this->tas_display_time_offset) {
         time_offset_end = 13;
         lr.lr_start = 0;
         lr.lr_end = time_offset_end;
@@ -1117,29 +1093,6 @@ logfile_sub_source::text_update_marks(vis_bookmarks& bm)
     }
 }
 
-log_accel::direction_t
-logfile_sub_source::get_line_accel_direction(vis_line_t vl)
-{
-    log_accel la;
-
-    while (vl >= 0) {
-        logline* curr_line = this->find_line(this->at(vl));
-
-        if (!curr_line->is_message()) {
-            --vl;
-            continue;
-        }
-
-        if (!la.add_point(curr_line->get_time_in_millis())) {
-            break;
-        }
-
-        --vl;
-    }
-
-    return la.get_direction();
-}
-
 void
 logfile_sub_source::text_filters_changed()
 {

src/logfile_sub_source.hh

@@ -240,6 +240,7 @@ private:
 class logfile_sub_source
     : public text_sub_source
     , public text_time_translator
+    , public text_accel_source
     , public list_input_delegate {
 public:
     const static bookmark_type_t BM_ERRORS;
@@ -252,12 +253,6 @@ public:
 
     ~logfile_sub_source() = default;
 
-    void toggle_time_offset()
-    {
-        this->lss_flags ^= F_TIME_OFFSET;
-        this->clear_line_size_cache();
-    }
-
     void increase_line_context()
     {
         auto old_flags = this->lss_flags;
@@ -305,20 +300,6 @@ public:
         return 0;
     }
 
-    void set_time_offset(bool enabled)
-    {
-        if (enabled)
-            this->lss_flags |= F_TIME_OFFSET;
-        else
-            this->lss_flags &= ~F_TIME_OFFSET;
-        this->clear_line_size_cache();
-    }
-
-    bool is_time_offset_enabled() const
-    {
-        return (bool) (this->lss_flags & F_TIME_OFFSET);
-    }
-
     bool is_filename_enabled() const
     {
         return (bool) (this->lss_flags & F_FILENAME);
@@ -650,8 +631,6 @@ public:
         return logline_window(*this, start_vl, end_vl);
     }
 
-    log_accel::direction_t get_line_accel_direction(vis_line_t vl);
-
     /**
      * Container for logfile references that keeps of how many lines in the
      * logfile have been indexed.
@@ -834,19 +813,25 @@ public:
 
     void quiesce();
 
+protected:
+    void text_accel_display_changed() { this->clear_line_size_cache(); }
+
+    logline* text_accel_get_line(vis_line_t vl)
+    {
+        return this->find_line(this->at(vl));
+    }
+
 private:
     static const size_t LINE_SIZE_CACHE_SIZE = 512;
 
     enum {
         B_SCRUB,
-        B_TIME_OFFSET,
         B_FILENAME,
         B_BASENAME,
     };
 
     enum {
         F_SCRUB = (1UL << B_SCRUB),
-        F_TIME_OFFSET = (1UL << B_TIME_OFFSET),
         F_FILENAME = (1UL << B_FILENAME),
         F_BASENAME = (1UL << B_BASENAME),
 

src/pcap_manager.cc

@@ -38,6 +38,7 @@
 #include <unistd.h>
 
 #include "base/fs_util.hh"
+#include "base/paths.hh"
 #include "config.h"
 #include "line_buffer.hh"
 
@@ -48,9 +49,9 @@ convert(const std::string& filename)
 {
     log_info("attempting to convert pcap file -- %s", filename.c_str());
 
-    auto outfile = TRY(lnav::filesystem::open_temp_file(
+    ghc::filesystem::create_directories(lnav::paths::workdir());
-        ghc::filesystem::temp_directory_path() / "lnav.pcap.XXXXXX"));
+    auto outfile = TRY(lnav::filesystem::open_temp_file(lnav::paths::workdir()
-    ghc::filesystem::remove(outfile.first);
+                                                        / "pcap.XXXXXX"));
     auto err_pipe = TRY(auto_pipe::for_child_fd(STDERR_FILENO));
     auto child = TRY(lnav::pid::from_fork());
 
@@ -59,7 +60,8 @@ convert(const std::string& filename)
         auto dev_null = open("/dev/null", O_RDONLY);
 
         dup2(dev_null, STDIN_FILENO);
-        dup2(outfile.second.release(), STDOUT_FILENO);
+        dup2(outfile.second.get(), STDOUT_FILENO);
+        outfile.second.reset();
         setenv("TZ", "UTC", 1);
 
         const char* args[] = {
@@ -131,7 +133,7 @@ convert(const std::string& filename)
 
     return Ok(convert_result{
         std::move(child),
-        std::move(outfile.second),
+        outfile.first,
         error_queue,
     });
 }

src/pcap_manager.hh

@@ -38,12 +38,13 @@
 #include "base/auto_fd.hh"
 #include "base/auto_pid.hh"
 #include "base/result.h"
+#include "ghc/filesystem.hpp"
 
 namespace pcap_manager {
 
 struct convert_result {
     auto_pid<process_state::running> cr_child;
-    auto_fd cr_destination;
+    ghc::filesystem::path cr_destination;
     std::shared_ptr<std::vector<std::string>> cr_error_queue;
 };
 

src/piper.looper.cc

@@ -0,0 +1,342 @@
+/**
+ * Copyright (c) 2023, Timothy Stack
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of Timothy Stack nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <chrono>
+#include <thread>
+
+#include "piper.looper.hh"
+
+#include <poll.h>
+
+#include "base/fs_util.hh"
+#include "base/injector.hh"
+#include "base/paths.hh"
+#include "base/time_util.hh"
+#include "config.h"
+#include "line_buffer.hh"
+#include "lnav_util.hh"
+#include "piper.looper.cfg.hh"
+
+using namespace std::chrono_literals;
+
+static ssize_t
+write_timestamp(int fd, log_level_t level, off_t woff)
+{
+    char time_str[64];
+    struct timeval tv;
+
+    gettimeofday(&tv, nullptr);
+    snprintf(time_str,
+             sizeof(time_str),
+             "%ld.%d:%c;",
+             tv.tv_sec,
+             tv.tv_usec,
+             level_names[level][0]);
+
+    return pwrite(fd, time_str, strlen(time_str), woff);
+}
+
+namespace lnav {
+namespace piper {
+
+looper::looper(std::string name, auto_fd stdout_fd, auto_fd stderr_fd)
+    : l_name(std::move(name)), l_stdout(std::move(stdout_fd)),
+      l_stderr(std::move(stderr_fd))
+{
+    size_t count = 0;
+    do {
+        this->l_out_dir
+            = lnav::paths::workdir()
+            / fmt::format(
+                  FMT_STRING("piper-{}-{}"),
+                  hasher().update(getmstime()).update(l_name).to_string(),
+                  count);
+        count += 1;
+    } while (ghc::filesystem::exists(this->l_out_dir));
+    ghc::filesystem::create_directories(this->l_out_dir);
+    this->l_future = std::async(std::launch::async, [this]() { this->loop(); });
+}
+
+looper::~looper()
+{
+    log_info("piper destructed, shutting down: %s", this->l_name.c_str());
+    this->l_looping = false;
+    this->l_future.wait();
+}
+
+enum class read_mode_t {
+    binary,
+    line,
+};
+
+void
+looper::loop()
+{
+    const auto& cfg = injector::get<const config&>();
+    struct pollfd pfd[2];
+    struct {
+        line_buffer lb;
+        file_range last_range;
+        pollfd* pfd{nullptr};
+        log_level_t cf_level{LEVEL_INFO};
+        read_mode_t cf_read_mode{read_mode_t::line};
+
+        void reset_pfd()
+        {
+            this->pfd->fd = this->lb.get_fd();
+            this->pfd->events = POLLIN;
+            this->pfd->revents = 0;
+        }
+    } captured_fds[2];
+    off_t woff = 0, last_woff = 0;
+    auto_fd outfd;
+    size_t rotate_count = 0;
+
+    log_info("starting loop to capture: %s (%d %d)",
+             this->l_name.c_str(),
+             this->l_stdout.get(),
+             this->l_stderr.get());
+    captured_fds[0].lb.set_fd(this->l_stdout);
+    if (this->l_stderr.has_value()) {
+        captured_fds[1].lb.set_fd(this->l_stderr);
+    }
+    captured_fds[1].cf_level = LEVEL_ERROR;
+    do {
+        static const auto TIMEOUT
+            = std::chrono::duration_cast<std::chrono::milliseconds>(1s).count();
+
+        size_t used_pfds = 0;
+        for (auto& cap : captured_fds) {
+            if (cap.lb.get_fd() != -1 && cap.lb.is_pipe()
+                && !cap.lb.is_pipe_closed())
+            {
+                cap.pfd = &pfd[used_pfds];
+                used_pfds += 1;
+                cap.reset_pfd();
+            } else {
+                cap.pfd = nullptr;
+            }
+        }
+
+        if (used_pfds == 0) {
+            log_info("inputs consumed, breaking loop: %s",
+                     this->l_name.c_str());
+            this->l_looping = false;
+            break;
+        }
+
+        auto poll_rc = poll(pfd, used_pfds, TIMEOUT);
+        if (poll_rc == 0) {
+            // update the timestamp to keep the file alive from any
+            // cleanup processes
+            if (outfd.has_value()) {
+                log_perror(futimes(outfd.get(), nullptr));
+            }
+            continue;
+        }
+        for (auto& cap : captured_fds) {
+            while (this->l_looping) {
+                if (cap.pfd == nullptr || !(cap.pfd->revents & POLLIN)) {
+                    break;
+                }
+
+                if (cap.cf_read_mode == read_mode_t::binary) {
+                    char buffer[8192];
+                    auto read_rc
+                        = read(cap.lb.get_fd(), buffer, sizeof(buffer));
+
+                    if (read_rc < 0) {
+                        if (errno == EAGAIN) {
+                            break;
+                        }
+                        log_error("failed to read next chunk: %s -- %s",
+                                  this->l_name.c_str(),
+                                  strerror(errno));
+                        this->l_looping = false;
+                    } else if (read_rc == 0) {
+                        this->l_looping = false;
+                    } else {
+                        auto rc = write(outfd.get(), buffer, read_rc);
+                        if (rc != read_rc) {
+                            log_error(
+                                "failed to write to capture file: %s -- %s",
+                                this->l_name.c_str(),
+                                strerror(errno));
+                        }
+                    }
+                    continue;
+                }
+
+                auto load_result = cap.lb.load_next_line(cap.last_range);
+
+                if (load_result.isErr()) {
+                    log_error("failed to load next line: %s -- %s",
+                              this->l_name.c_str(),
+                              load_result.unwrapErr().c_str());
+                    this->l_looping = false;
+                    break;
+                }
+
+                auto li = load_result.unwrap();
+
+                if (cap.last_range.fr_offset == 0 && !cap.lb.is_header_utf8()) {
+                    log_info("switching capture to binary mode: %s",
+                             this->l_name.c_str());
+                    cap.cf_read_mode = read_mode_t::binary;
+
+                    auto out_path = this->l_out_dir / "out.0";
+                    log_info("creating binary capture file: %s -- %s",
+                             this->l_name.c_str(),
+                             out_path.c_str());
+                    auto create_res = lnav::filesystem::create_file(
+                        out_path, O_WRONLY | O_CLOEXEC | O_TRUNC, 0600);
+                    if (create_res.isErr()) {
+                        log_error("unable to open capture file: %s -- %s",
+                                  this->l_name.c_str(),
+                                  create_res.unwrapErr().c_str());
+                        break;
+                    }
+
+                    outfd = create_res.unwrap();
+                    auto header_avail = cap.lb.get_available();
+                    auto read_res = cap.lb.read_range(header_avail);
+                    if (read_res.isOk()) {
+                        auto sbr = read_res.unwrap();
+                        write(outfd.get(), sbr.get_data(), sbr.length());
+                    } else {
+                        log_error("failed to get header data: %s -- %s",
+                                  this->l_name.c_str(),
+                                  read_res.unwrapErr().c_str());
+                    }
+                    continue;
+                }
+
+                if (li.li_partial && !cap.lb.is_pipe_closed()) {
+                    break;
+                }
+
+                if (li.li_file_range.empty()) {
+                    break;
+                }
+
+                auto read_result = cap.lb.read_range(li.li_file_range);
+
+                if (read_result.isErr()) {
+                    log_error("failed to read next line: %s -- %s",
+                              this->l_name.c_str(),
+                              read_result.unwrapErr().c_str());
+                    this->l_looping = false;
+                    break;
+                }
+
+                auto sbr = read_result.unwrap();
+
+                if (woff > last_woff && woff >= cfg.c_max_size) {
+                    log_info(
+                        "capture file has reached max size, rotating: %s -- "
+                        "%lld",
+                        this->l_name.c_str(),
+                        woff);
+                    outfd.reset();
+                }
+
+                if (!outfd.has_value()) {
+                    auto out_path = this->l_out_dir
+                        / fmt::format(FMT_STRING("out.{}"),
+                                      rotate_count % cfg.c_rotations);
+                    log_info("creating capturing file: %s -- %s",
+                             this->l_name.c_str(),
+                             out_path.c_str());
+                    auto create_res = lnav::filesystem::create_file(
+                        out_path, O_WRONLY | O_CLOEXEC | O_TRUNC, 0600);
+                    if (create_res.isErr()) {
+                        log_error("unable to open capture file: %s -- %s",
+                                  this->l_name.c_str(),
+                                  create_res.unwrapErr().c_str());
+                        break;
+                    }
+
+                    outfd = create_res.unwrap();
+                    rotate_count += 1;
+
+                    static const char lnav_header[]
+                        = {'L', 0, 'N', 1, 0, 0, 0, 0};
+                    auto prc
+                        = write(outfd.get(), lnav_header, sizeof(lnav_header));
+                    woff = prc;
+                }
+
+                ssize_t wrc;
+
+                last_woff = woff;
+                wrc = write_timestamp(outfd.get(), cap.cf_level, woff);
+                if (wrc == -1) {
+                    log_error("unable to write timestamp: %s -- %s",
+                              this->l_name.c_str(),
+                              strerror(errno));
+                    this->l_looping = false;
+                    break;
+                }
+                woff += wrc;
+
+                /* Need to do pwrite here since the fd is used by the main
+                 * lnav process as well.
+                 */
+                wrc = pwrite(outfd.get(), sbr.get_data(), sbr.length(), woff);
+                if (wrc == -1) {
+                    log_error("unable to write captured data: %s -- %s",
+                              this->l_name.c_str(),
+                              strerror(errno));
+                    this->l_looping = false;
+                    break;
+                }
+                woff += wrc;
+
+                cap.last_range = li.li_file_range;
+                if (li.li_partial && sbr.get_data()[sbr.length() - 1] != '\n'
+                    && (cap.last_range.next_offset() != cap.lb.get_file_size()))
+                {
+                    woff = last_woff;
+                }
+            }
+        }
+    } while (this->l_looping);
+
+    log_info("exiting loop to capture: %s", this->l_name.c_str());
+}
+
+Result<handle<state::running>, std::string>
+create_looper(std::string name, auto_fd stdout_fd, auto_fd stderr_fd)
+{
+    return Ok(handle<state::running>(std::make_shared<looper>(
+        name, std::move(stdout_fd), std::move(stderr_fd))));
+}
+
+}  // namespace piper
+}  // namespace lnav

src/piper.looper.cfg.hh

@@ -1,5 +1,5 @@
 /**
- * Copyright (c) 2007-2012, Timothy Stack
+ * Copyright (c) 2023, Timothy Stack
  *
  * All rights reserved.
  *
@@ -25,62 +25,22 @@
  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * @file piper_proc.hh
  */
 
-#ifndef piper_proc_hh
+#ifndef piper_looper_cfg_hh
-#define piper_proc_hh
+#define piper_looper_cfg_hh
-
-#include <string>
-
-#include <sys/types.h>
-
-#include "base/auto_fd.hh"
 
-/**
+#include <stdint.h>
- * Creates a subprocess that reads data from a pipe and writes it to a file so
- * lnav can treat it like any other file and do preads.
- *
- * TODO: Add support for gzipped files.
- */
-class piper_proc {
-public:
-    class error : public std::exception {
-    public:
-        error(int err) : e_err(err) {}
 
-        int e_err;
+namespace lnav {
-    };
+namespace piper {
 
-    /**
+struct config {
-     * Forks a subprocess that will read data from the given file descriptor
+    uint64_t c_max_size{10ULL * 1024ULL * 1024ULL};
-     * and write it to a temporary file.
+    uint32_t c_rotations{4};
-     *
+};
-     * @param pipefd The file descriptor to read the file contents from.
-     * @param timestamp True if an ISO 8601 timestamp should be prepended onto
-     *   the lines read from pipefd.
-     * @param filefd The descriptor for the backing file.
-     */
-    piper_proc(auto_fd pipefd, bool timestamp, auto_fd filefd);
-
-    bool has_exited();
-
-    /**
-     * Terminates the child process.
-     */
-    virtual ~piper_proc();
-
-    /** @return The file descriptor for the temporary file. */
-    auto_fd get_fd() { return this->pp_fd.dup(); }
-
-    pid_t get_child_pid() const { return this->pp_child; }
 
-private:
+}  // namespace piper
-    /** A file descriptor that refers to the temporary file. */
+}  // namespace lnav
-    auto_fd pp_fd;
 
-    /** The child process' pid. */
-    pid_t pp_child;
-};
 #endif

src/piper.looper.hh

@@ -0,0 +1,131 @@
+/**
+ * Copyright (c) 2023, Timothy Stack
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of Timothy Stack nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef piper_looper_hh
+#define piper_looper_hh
+
+#include <future>
+#include <memory>
+#include <string>
+
+#include "base/auto_fd.hh"
+#include "base/result.h"
+#include "ghc/filesystem.hpp"
+
+namespace lnav {
+namespace piper {
+
+enum class state {
+    running,
+    finished,
+};
+
+class looper {
+public:
+    looper(std::string name, auto_fd stdout_fd, auto_fd stderr_fd);
+
+    ~looper();
+
+    std::string get_name() const { return this->l_name; }
+
+    ghc::filesystem::path get_out_dir() const { return this->l_out_dir; }
+
+    ghc::filesystem::path get_out_pattern() const
+    {
+        return this->l_out_dir / "out.*";
+    }
+
+    bool is_finished() const
+    {
+        return this->l_future.wait_for(std::chrono::seconds(0))
+            == std::future_status::ready;
+    }
+
+private:
+    void loop();
+
+    std::atomic<bool> l_looping{true};
+    const std::string l_name;
+    ghc::filesystem::path l_out_dir;
+    auto_fd l_stdout;
+    auto_fd l_stderr;
+    std::future<void> l_future;
+};
+
+template<state LooperState>
+class handle {
+public:
+    explicit handle(std::shared_ptr<looper> looper)
+        : h_looper(std::move(looper))
+    {
+    }
+
+    std::string get_name() const { return this->h_looper->get_name(); }
+
+    ghc::filesystem::path get_out_dir() const
+    {
+        return this->h_looper->get_out_dir();
+    }
+
+    ghc::filesystem::path get_out_pattern() const
+    {
+        return this->h_looper->get_out_pattern();
+    }
+
+    bool is_finished() const { return this->h_looper->is_finished(); }
+
+    handle<state::finished> close() &&
+    {
+        static_assert(LooperState == state::running,
+                      "this method is only available in the running state");
+
+        this->h_looper->close();
+
+        return handle<state::finished>{nullptr};
+    }
+
+    bool operator==(const handle& other) const
+    {
+        return this->h_looper.get() == other.h_looper.get();
+    }
+
+private:
+    std::shared_ptr<looper> h_looper;
+};
+
+using running_handle = handle<state::running>;
+
+Result<handle<state::running>, std::string> create_looper(std::string name,
+                                                          auto_fd stdout_fd,
+                                                          auto_fd stderr_fd);
+
+}  // namespace piper
+}  // namespace lnav
+
+#endif

src/piper_proc.cc

@@ -1,237 +0,0 @@
-/**
- * Copyright (c) 2007-2012, Timothy Stack
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- * * Neither the name of Timothy Stack nor the names of its contributors
- * may be used to endorse or promote products derived from this software
- * without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * @file piper_proc.cc
- */
-
-#include "piper_proc.hh"
-
-#include <errno.h>
-#include <fcntl.h>
-#include <poll.h>
-#include <signal.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/wait.h>
-#include <unistd.h>
-
-#include "base/fs_util.hh"
-#include "base/lnav_log.hh"
-#include "config.h"
-#include "line_buffer.hh"
-
-using namespace std::chrono_literals;
-
-static const char* STDIN_EOF_MSG = "---- END-OF-STDIN ----";
-
-static ssize_t
-write_timestamp(int fd, off_t woff)
-{
-    char time_str[64];
-    struct timeval tv;
-    char ms_str[10];
-
-    gettimeofday(&tv, nullptr);
-    strftime(time_str, sizeof(time_str), "%FT%T", localtime(&tv.tv_sec));
-    snprintf(ms_str, sizeof(ms_str), ".%03d", (int) (tv.tv_usec / 1000));
-    strcat(time_str, ms_str);
-    strcat(time_str, "  ");
-    return pwrite(fd, time_str, strlen(time_str), woff);
-}
-
-piper_proc::piper_proc(auto_fd pipefd, bool timestamp, auto_fd filefd)
-    : pp_fd(std::move(filefd)), pp_child(-1)
-{
-    require(pipefd.get() >= 0);
-    require(this->pp_fd.get() >= 0);
-
-    log_perror(fcntl(this->pp_fd.get(), F_SETFD, FD_CLOEXEC));
-
-    this->pp_child = fork();
-    switch (this->pp_child) {
-        case -1:
-            throw error(errno);
-
-        case 0: {
-            line_buffer lb;
-            off_t woff = 0, last_woff = 0;
-            file_range last_range;
-
-            auto open_res = lnav::filesystem::open_file("/dev/null", O_RDWR);
-            if (open_res.isErr()) {
-                fprintf(stderr,
-                        "unable to open /dev/null: %s\n",
-                        open_res.unwrapErr().c_str());
-                exit(EXIT_FAILURE);
-            }
-            auto nullfd = open_res.unwrap();
-            if (pipefd != STDIN_FILENO) {
-                dup2(nullfd, STDIN_FILENO);
-            }
-            dup2(nullfd, STDOUT_FILENO);
-            for (int fd_to_close = 0; fd_to_close < 1024; fd_to_close++) {
-                int flags;
-
-                if (fd_to_close == this->pp_fd.get()) {
-                    continue;
-                }
-                if ((flags = fcntl(fd_to_close, F_GETFD)) == -1) {
-                    continue;
-                }
-                if (flags & FD_CLOEXEC) {
-                    close(fd_to_close);
-                }
-            }
-            log_perror(fcntl(pipefd.get(), F_SETFL, O_NONBLOCK));
-            lb.set_fd(pipefd);
-            do {
-                static const auto TIMEOUT
-                    = std::chrono::duration_cast<std::chrono::milliseconds>(1s)
-                          .count();
-                struct pollfd pfd = {lb.get_fd(), POLLIN, 0};
-
-                auto poll_rc = poll(&pfd, 1, TIMEOUT);
-                if (poll_rc == 0) {
-                    // update the timestamp to keep the file alive from any
-                    // cleanup processes
-                    log_perror(futimes(this->pp_fd.get(), nullptr));
-                    continue;
-                }
-                while (true) {
-                    auto load_result = lb.load_next_line(last_range);
-
-                    if (load_result.isErr()) {
-                        break;
-                    }
-
-                    auto li = load_result.unwrap();
-
-                    if (li.li_partial && !lb.is_pipe_closed()) {
-                        break;
-                    }
-
-                    if (li.li_file_range.empty()) {
-                        break;
-                    }
-
-                    auto read_result = lb.read_range(li.li_file_range);
-
-                    if (read_result.isErr()) {
-                        break;
-                    }
-
-                    auto sbr = read_result.unwrap();
-
-                    ssize_t wrc;
-
-                    last_woff = woff;
-                    if (timestamp) {
-                        wrc = write_timestamp(this->pp_fd, woff);
-                        if (wrc == -1) {
-                            perror("Unable to write to output file for stdin");
-                            break;
-                        }
-                        woff += wrc;
-                    }
-
-                    /* Need to do pwrite here since the fd is used by the main
-                     * lnav process as well.
-                     */
-                    wrc = pwrite(
-                        this->pp_fd, sbr.get_data(), sbr.length(), woff);
-                    if (wrc == -1) {
-                        perror("Unable to write to output file for stdin");
-                        break;
-                    }
-                    woff += wrc;
-
-                    last_range = li.li_file_range;
-                    if (li.li_partial
-                        && sbr.get_data()[sbr.length() - 1] != '\n'
-                        && (last_range.next_offset() != lb.get_file_size()))
-                    {
-                        woff = last_woff;
-                    }
-                }
-            } while (lb.is_pipe() && !lb.is_pipe_closed());
-
-            if (timestamp) {
-                ssize_t wrc;
-
-                wrc = write_timestamp(this->pp_fd, woff);
-                if (wrc == -1) {
-                    perror("Unable to write to output file for stdin");
-                    break;
-                }
-                woff += wrc;
-                wrc = pwrite(
-                    this->pp_fd, STDIN_EOF_MSG, strlen(STDIN_EOF_MSG), woff);
-                if (wrc == -1) {
-                    perror("Unable to write to output file for stdin");
-                    break;
-                }
-            }
-        }
-            _exit(0);
-            break;
-
-        default:
-            break;
-    }
-}
-
-bool
-piper_proc::has_exited()
-{
-    if (this->pp_child > 0) {
-        int rc, status;
-
-        rc = waitpid(this->pp_child, &status, WNOHANG);
-        if (rc == -1 || rc == 0) {
-            return false;
-        }
-        this->pp_child = -1;
-    }
-
-    return true;
-}
-
-piper_proc::~piper_proc()
-{
-    if (this->pp_child > 0) {
-        int status;
-
-        kill(this->pp_child, SIGTERM);
-        while (waitpid(this->pp_child, &status, 0) < 0 && (errno == EINTR)) {
-            ;
-        }
-
-        this->pp_child = -1;
-    }
-}

src/readline_callbacks.cc

@@ -27,8 +27,10 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "base/fs_util.hh"
 #include "base/humanize.network.hh"
 #include "base/injector.hh"
+#include "base/paths.hh"
 #include "command_executor.hh"
 #include "config.h"
 #include "field_overlay_source.hh"
@@ -736,23 +738,26 @@ rl_callback_int(readline_curses* rc, bool is_alt)
         }
 
         case ln_mode_t::EXEC: {
-            auto_mem<FILE> tmpout(fclose);
+            auto open_temp_res = lnav::filesystem::open_temp_file(
+                lnav::paths::workdir() / "exec.XXXXXX");
 
-            tmpout = std::tmpfile();
+            if (open_temp_res.isErr()) {
-
-            if (!tmpout) {
                 rc->set_value(fmt::format(
                     FMT_STRING("Unable to open temporary output file: {}"),
-                    strerror(errno)));
+                    open_temp_res.unwrapErr()));
             } else {
-                auto fd_copy = auto_fd::dup_of(fileno(tmpout));
                 char desc[256], timestamp[32];
                 time_t current_time = time(nullptr);
                 const auto path_and_args = rc->get_value();
+                auto tmp_pair = open_temp_res.unwrap();
+                auto fd_copy = tmp_pair.second.dup();
 
                 {
                     exec_context::output_guard og(
-                        ec, "tmp", std::make_pair(tmpout.release(), fclose));
+                        ec,
+                        "tmp",
+                        std::make_pair(fdopen(tmp_pair.second.release(), "w"),
+                                       fclose));
 
                     auto exec_res
                         = execute_file(ec, path_and_args.get_string());
@@ -782,8 +787,8 @@ rl_callback_int(readline_curses* rc, bool is_alt)
                              "Output of %s (%s)",
                              path_and_args.get_string().c_str(),
                              timestamp);
-                    lnav_data.ld_active_files.fc_file_names[desc]
+                    lnav_data.ld_active_files.fc_file_names[tmp_pair.first]
-                        .with_fd(std::move(fd_copy))
+                        .with_filename(desc)
                         .with_include_in_session(false)
                         .with_detect_format(false);
                     lnav_data.ld_files_to_front.emplace_back(desc, 0_vl);

src/readline_curses.cc

@@ -421,6 +421,30 @@ readline_context::attempted_completion(const char* text, int start, int end)
 
                 if (proto.empty()) {
                     arg_possibilities = nullptr;
+                } else if (proto[0] == "dirname") {
+                    shlex fn_lexer(rl_line_buffer, rl_point);
+                    std::vector<std::string> fn_list;
+
+                    fn_lexer.split(fn_list, scope);
+
+                    const auto& last_fn = fn_list.size() <= 1 ? ""
+                                                              : fn_list.back();
+
+                    static std::set<std::string> dir_name_set;
+
+                    dir_name_set.clear();
+                    auto_mem<char> completed_fn;
+                    int fn_state = 0;
+
+                    while ((completed_fn = rl_filename_completion_function(
+                                last_fn.c_str(), fn_state))
+                           != nullptr)
+                    {
+                        dir_name_set.insert(completed_fn.in());
+                        fn_state += 1;
+                    }
+                    arg_possibilities = &dir_name_set;
+                    arg_needs_shlex = true;
                 } else if (proto[0] == "filename") {
                     shlex fn_lexer(rl_line_buffer, rl_point);
                     std::vector<std::string> fn_list;
@@ -862,7 +886,7 @@ readline_curses::start()
                 }
             }
             if (FD_ISSET(this->rc_command_pipe[RCF_SLAVE], &ready_rfds)) {
-                char msg[1024 + 1];
+                char msg[8 + MAXPATHLEN + 1024];
 
                 if ((rc = recvstring(this->rc_command_pipe[RCF_SLAVE],
                                      msg,
@@ -875,7 +899,13 @@ readline_curses::start()
                     char type[1024];
 
                     msg[rc] = '\0';
-                    if (sscanf(msg, "i:%d:%n", &rl_point, &prompt_start) == 1) {
+                    if (startswith(msg, "cd:")) {
+                        const char* cwd = &msg[3];
+
+                        log_perror(chdir(cwd));
+                    } else if (sscanf(msg, "i:%d:%n", &rl_point, &prompt_start)
+                               == 1)
+                    {
                         const char* initial = &msg[prompt_start];
 
                         rl_extend_line_buffer(strlen(initial) + 1);
@@ -1247,12 +1277,21 @@ readline_curses::focus(int context,
                        const std::string& prompt,
                        const std::string& initial)
 {
-    char buffer[1024];
+    char cwd[MAXPATHLEN + 1024];
+    char buffer[8 + sizeof(cwd)];
 
     curs_set(1);
 
     this->rc_active_context = context;
 
+    getcwd(cwd, sizeof(cwd));
+    snprintf(buffer, sizeof(buffer), "cd:%s", cwd);
+    if (sendstring(
+            this->rc_command_pipe[RCF_MASTER], buffer, strlen(buffer) + 1)
+        == -1)
+    {
+        perror("focus: write failed");
+    }
     snprintf(buffer, sizeof(buffer), "f:%d:%s", context, prompt.c_str());
     if (sendstring(
             this->rc_command_pipe[RCF_MASTER], buffer, strlen(buffer) + 1)

src/root-config.json

@@ -26,6 +26,10 @@
                 "transfer-command": "cat > {0:} && chmod ugo+rx ./{0:}"
             }
         },
+        "piper": {
+            "max-size": 10485760,
+            "rotations": 4
+        },
         "clipboard": {
             "impls": {
                 "MacOS": {

src/textfile_sub_source.cc

@@ -75,10 +75,18 @@ textfile_sub_source::text_value_for_line(textview_curses& tc,
             if (line < 0 || line >= lfo->lfo_filter_state.tfs_index.size()) {
                 value_out.clear();
             } else {
-                auto read_result = lf->read_line(
+                auto ll = lf->begin() + lfo->lfo_filter_state.tfs_index[line];
-                    lf->begin() + lfo->lfo_filter_state.tfs_index[line]);
+                auto read_result = lf->read_line(ll);
                 if (read_result.isOk()) {
                     value_out = to_string(read_result.unwrap());
+                    if (lf->has_line_metadata()
+                        && this->tas_display_time_offset)
+                    {
+                        auto relstr = this->get_time_offset_for_line(
+                            tc, vis_line_t(line));
+                        value_out = fmt::format(
+                            FMT_STRING("{: >12}|{}"), relstr, value_out);
+                    }
                 }
             }
         } else {
@@ -100,16 +108,53 @@ textfile_sub_source::text_attrs_for_line(textview_curses& tc,
         return;
     }
 
+    struct line_range lr;
+
+    lr.lr_start = 0;
+    lr.lr_end = -1;
     auto rend_iter = this->tss_rendered_files.find(lf->get_filename());
     if (rend_iter != this->tss_rendered_files.end()) {
         rend_iter->second.rf_text_source->text_attrs_for_line(
             tc, row, value_out);
+    } else {
+        auto* lfo
+            = dynamic_cast<line_filter_observer*>(lf->get_logline_observer());
+        if (row >= 0 && row < lfo->lfo_filter_state.tfs_index.size()) {
+            auto ll = lf->begin() + lfo->lfo_filter_state.tfs_index[row];
+
+            value_out.emplace_back(lr, SA_LEVEL.value(ll->get_msg_level()));
+            if (lf->has_line_metadata() && this->tas_display_time_offset) {
+                auto time_offset_end = 13;
+                lr.lr_start = 0;
+                lr.lr_end = time_offset_end;
+
+                shift_string_attrs(value_out, 0, time_offset_end);
+
+                value_out.emplace_back(lr,
+                                       VC_ROLE.value(role_t::VCR_OFFSET_TIME));
+                value_out.emplace_back(line_range(12, 13),
+                                       VC_GRAPHIC.value(ACS_VLINE));
+
+                role_t bar_role = role_t::VCR_NONE;
+
+                switch (this->get_line_accel_direction(vis_line_t(row))) {
+                    case log_accel::A_STEADY:
+                        break;
+                    case log_accel::A_DECEL:
+                        bar_role = role_t::VCR_DIFF_DELETE;
+                        break;
+                    case log_accel::A_ACCEL:
+                        bar_role = role_t::VCR_DIFF_ADD;
+                        break;
+                }
+                if (bar_role != role_t::VCR_NONE) {
+                    value_out.emplace_back(line_range(12, 13),
+                                           VC_ROLE.value(bar_role));
+                }
+            }
+        }
     }
 
-    struct line_range lr;
-
-    lr.lr_start = 0;
-    lr.lr_end = -1;
     value_out.emplace_back(lr, logline::L_FILE.value(this->current_file()));
 }
 
@@ -157,6 +202,7 @@ textfile_sub_source::to_front(const std::shared_ptr<logfile>& lf)
         }
     }
     this->tss_files.push_front(lf);
+    this->set_time_offset(false);
     this->tss_view->reload_data();
 }
 
@@ -166,6 +212,7 @@ textfile_sub_source::rotate_left()
     if (this->tss_files.size() > 1) {
         this->tss_files.push_back(this->tss_files.front());
         this->tss_files.pop_front();
+        this->set_time_offset(false);
         this->tss_view->reload_data();
         this->tss_view->redo_search();
     }
@@ -177,6 +224,7 @@ textfile_sub_source::rotate_right()
     if (this->tss_files.size() > 1) {
         this->tss_files.push_front(this->tss_files.back());
         this->tss_files.pop_back();
+        this->set_time_offset(false);
         this->tss_view->reload_data();
         this->tss_view->redo_search();
     }
@@ -197,6 +245,7 @@ textfile_sub_source::remove(const std::shared_ptr<logfile>& lf)
             detach_observer(lf);
         }
     }
+    this->set_time_offset(false);
 }
 
 void
@@ -873,3 +922,11 @@ textfile_sub_source::to_front(const std::string& filename)
 
     return true;
 }
+
+logline*
+textfile_sub_source::text_accel_get_line(vis_line_t vl)
+{
+    auto lf = this->current_file();
+    auto* lfo = dynamic_cast<line_filter_observer*>(lf->get_logline_observer());
+    return (lf->begin() + lfo->lfo_filter_state.tfs_index[vl]).base();
+}

src/textfile_sub_source.hh

@@ -41,14 +41,13 @@
 class textfile_sub_source
     : public text_sub_source
     , public vis_location_history
+    , public text_accel_source
     , public text_anchors {
 public:
     using file_iterator = std::deque<std::shared_ptr<logfile>>::iterator;
 
     textfile_sub_source() { this->tss_supports_filtering = true; }
 
-    ~textfile_sub_source() override = default;
-
     bool empty() const { return this->tss_files.empty(); }
 
     size_t size() const { return this->tss_files.size(); }
@@ -109,6 +108,8 @@ public:
 
     class scan_callback {
     public:
+        virtual ~scan_callback() = default;
+
         virtual void closed_files(
             const std::vector<std::shared_ptr<logfile>>& files)
             = 0;
@@ -144,6 +145,18 @@ public:
 
     void quiesce() override;
 
+    bool is_time_offset_supported() const override
+    {
+        const auto lf = this->current_file();
+        if (lf != nullptr && lf->has_line_metadata()) {
+            return true;
+        }
+
+        return false;
+    }
+
+    logline* text_accel_get_line(vis_line_t vl) override;
+
 private:
     void detach_observer(std::shared_ptr<logfile> lf)
     {

src/textview_curses.cc

@@ -33,6 +33,7 @@
 #include "textview_curses.hh"
 
 #include "base/ansi_scrubber.hh"
+#include "base/humanize.time.hh"
 #include "base/injector.hh"
 #include "base/time_util.hh"
 #include "config.h"
@@ -124,6 +125,58 @@ text_filter::end_of_message(logfile_filter_state& lfs)
     lfs.tfs_lines_for_message[this->lf_index] = 0;
 }
 
+log_accel::direction_t
+text_accel_source::get_line_accel_direction(vis_line_t vl)
+{
+    log_accel la;
+
+    while (vl >= 0) {
+        const auto* curr_line = this->text_accel_get_line(vl);
+
+        if (!curr_line->is_message()) {
+            --vl;
+            continue;
+        }
+
+        if (!la.add_point(curr_line->get_time_in_millis())) {
+            break;
+        }
+
+        --vl;
+    }
+
+    return la.get_direction();
+}
+
+std::string
+text_accel_source::get_time_offset_for_line(textview_curses& tc, vis_line_t vl)
+{
+    auto ll = this->text_accel_get_line(vl);
+    auto curr_tv = ll->get_timeval();
+    struct timeval diff_tv;
+
+    auto prev_umark = tc.get_bookmarks()[&textview_curses::BM_USER].prev(vl);
+    auto next_umark = tc.get_bookmarks()[&textview_curses::BM_USER].next(vl);
+    auto prev_emark
+        = tc.get_bookmarks()[&textview_curses::BM_USER_EXPR].prev(vl);
+    auto next_emark
+        = tc.get_bookmarks()[&textview_curses::BM_USER_EXPR].next(vl);
+    if (!prev_umark && !prev_emark && (next_umark || next_emark)) {
+        auto next_line = this->text_accel_get_line(
+            std::max(next_umark.value_or(0), next_emark.value_or(0)));
+
+        diff_tv = curr_tv - next_line->get_timeval();
+    } else {
+        auto prev_row
+            = std::max(prev_umark.value_or(0), prev_emark.value_or(0));
+        auto first_line = this->text_accel_get_line(prev_row);
+        auto start_tv = first_line->get_timeval();
+        diff_tv = curr_tv - start_tv;
+    }
+
+    return humanize::time::duration::from_tv(diff_tv).to_string();
+}
+
 const bookmark_type_t textview_curses::BM_USER("user");
 const bookmark_type_t textview_curses::BM_USER_EXPR("user-expr");
 const bookmark_type_t textview_curses::BM_SEARCH("search");

src/textview_curses.hh

@@ -43,6 +43,7 @@
 #include "highlighter.hh"
 #include "listview_curses.hh"
 #include "lnav_config_fwd.hh"
+#include "log_accel.hh"
 #include "logfile_fwd.hh"
 #include "ring_span.hh"
 #include "text_format.hh"
@@ -237,6 +238,43 @@ protected:
     };
 };
 
+class text_accel_source {
+public:
+    virtual ~text_accel_source() = default;
+
+    virtual log_accel::direction_t get_line_accel_direction(vis_line_t vl);
+
+    void toggle_time_offset()
+    {
+        this->tas_display_time_offset = !this->tas_display_time_offset;
+        this->text_accel_display_changed();
+    }
+
+    void set_time_offset(bool enabled)
+    {
+        if (this->tas_display_time_offset != enabled) {
+            this->tas_display_time_offset = enabled;
+            this->text_accel_display_changed();
+        }
+    }
+
+    bool is_time_offset_enabled() const
+    {
+        return this->tas_display_time_offset;
+    }
+
+    virtual bool is_time_offset_supported() const { return true; }
+
+    virtual logline* text_accel_get_line(vis_line_t vl) = 0;
+
+    std::string get_time_offset_for_line(textview_curses& tc, vis_line_t vl);
+
+protected:
+    virtual void text_accel_display_changed() {}
+
+    bool tas_display_time_offset{false};
+};
+
 class text_anchors {
 public:
     virtual ~text_anchors() = default;

src/url_loader.hh

@@ -37,20 +37,21 @@
 #    include <paths.h>
 
 #    include "base/fs_util.hh"
+#    include "base/paths.hh"
 #    include "curl_looper.hh"
 
 class url_loader : public curl_request {
 public:
     url_loader(const std::string& url) : curl_request(url)
     {
-        auto tmp_res = lnav::filesystem::open_temp_file(
+        auto tmp_res = lnav::filesystem::open_temp_file(lnav::paths::workdir()
-            ghc::filesystem::temp_directory_path() / "lnav.url.XXXXXX");
+                                                        / "url.XXXXXX");
         if (tmp_res.isErr()) {
             return;
         }
 
         auto tmp_pair = tmp_res.unwrap();
-        ghc::filesystem::remove(tmp_pair.first);
+        this->ul_path = tmp_pair.first;
         this->ul_fd = std::move(tmp_pair.second);
 
         curl_easy_setopt(this->cr_handle, CURLOPT_URL, this->cr_name.c_str());
@@ -60,9 +61,7 @@ public:
         curl_easy_setopt(this->cr_handle, CURLOPT_BUFFERSIZE, 128L * 1024L);
     }
 
-    int get_fd() const { return this->ul_fd.get(); }
+    ghc::filesystem::path get_path() const { return this->ul_path; }
-
-    auto_fd copy_fd() const { return this->ul_fd.dup(); }
 
     long complete(CURLcode result)
     {
@@ -93,7 +92,8 @@ public:
 
             time(&current_time);
             if (file_time == -1
-                || (current_time - file_time) < FOLLOW_IF_MODIFIED_SINCE) {
+                || (current_time - file_time) < FOLLOW_IF_MODIFIED_SINCE)
+            {
                 char range[64];
                 struct stat st;
                 off_t start;
@@ -142,6 +142,7 @@ private:
         return retval;
     }
 
+    ghc::filesystem::path ul_path;
     auto_fd ul_fd;
     off_t ul_resume_offset{0};
 };

test/expected/expected.am

@@ -6,10 +6,14 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_cli.sh_5524542b1a6954ff9741155101497270a2f0c557.out \
     $(srcdir)/%reldir%/test_cli.sh_97e19b9ff3775d84074455a2e8993a0611b1c269.err \
     $(srcdir)/%reldir%/test_cli.sh_97e19b9ff3775d84074455a2e8993a0611b1c269.out \
-    $(srcdir)/%reldir%/test_cli.sh_a1a09f890f4604309d0a81bbbec8e50fb7d5e887.err \
+    $(srcdir)/%reldir%/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.err \
-    $(srcdir)/%reldir%/test_cli.sh_a1a09f890f4604309d0a81bbbec8e50fb7d5e887.out \
+    $(srcdir)/%reldir%/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.out \
     $(srcdir)/%reldir%/test_cli.sh_f2e41555f1a5f40f54ce241207af602ed1503a2b.err \
     $(srcdir)/%reldir%/test_cli.sh_f2e41555f1a5f40f54ce241207af602ed1503a2b.out \
+    $(srcdir)/%reldir%/test_cli.sh_ff7da172f4350a2adb74b8764575823d798ed8b6.err \
+    $(srcdir)/%reldir%/test_cli.sh_ff7da172f4350a2adb74b8764575823d798ed8b6.out \
+    $(srcdir)/%reldir%/test_cmds.sh_015ffe79a08f4c9f0cd1cb84c6afa4398f879fc7.err \
+    $(srcdir)/%reldir%/test_cmds.sh_015ffe79a08f4c9f0cd1cb84c6afa4398f879fc7.out \
     $(srcdir)/%reldir%/test_cmds.sh_017b495b95218b7c083951e2dba331cfec6e90be.err \
     $(srcdir)/%reldir%/test_cmds.sh_017b495b95218b7c083951e2dba331cfec6e90be.out \
     $(srcdir)/%reldir%/test_cmds.sh_0b1e4b1523dfca71927b1fe721c74490c51361d1.err \
@@ -96,6 +100,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_cmds.sh_62d68c0a11757c996f24c8f003e6b4059c3e30b2.out \
     $(srcdir)/%reldir%/test_cmds.sh_661ec61acdd8f6fa6ec1e3c2cf5f896eef431351.err \
     $(srcdir)/%reldir%/test_cmds.sh_661ec61acdd8f6fa6ec1e3c2cf5f896eef431351.out \
+    $(srcdir)/%reldir%/test_cmds.sh_68c774418bac897bd4d4fe9dbbf08454886b2e15.err \
+    $(srcdir)/%reldir%/test_cmds.sh_68c774418bac897bd4d4fe9dbbf08454886b2e15.out \
     $(srcdir)/%reldir%/test_cmds.sh_6a6031113aca32fabc5a3da64b7be46f5ce5a312.err \
     $(srcdir)/%reldir%/test_cmds.sh_6a6031113aca32fabc5a3da64b7be46f5ce5a312.out \
     $(srcdir)/%reldir%/test_cmds.sh_6e016c0ed61fc652be1a79b864875ffede64f281.err \
@@ -130,10 +136,14 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_cmds.sh_8d5b43c693e78804a8fb06989392fa8cccb46b7b.out \
     $(srcdir)/%reldir%/test_cmds.sh_9445861db011dfa2d21a44788047de345ee291e8.err \
     $(srcdir)/%reldir%/test_cmds.sh_9445861db011dfa2d21a44788047de345ee291e8.out \
+    $(srcdir)/%reldir%/test_cmds.sh_9527f941dc84a2ac3a030f222e41c6ccd1961cbe.err \
+    $(srcdir)/%reldir%/test_cmds.sh_9527f941dc84a2ac3a030f222e41c6ccd1961cbe.out \
     $(srcdir)/%reldir%/test_cmds.sh_95beaabe41d72cf4c6810e79c623da759ac1c71b.err \
     $(srcdir)/%reldir%/test_cmds.sh_95beaabe41d72cf4c6810e79c623da759ac1c71b.out \
     $(srcdir)/%reldir%/test_cmds.sh_968dac54dc80d91a5da2322890c6c26dfa0d8462.err \
     $(srcdir)/%reldir%/test_cmds.sh_968dac54dc80d91a5da2322890c6c26dfa0d8462.out \
+    $(srcdir)/%reldir%/test_cmds.sh_9dfc433f14b811afb3ec4daef0f33e4c9b14a6d7.err \
+    $(srcdir)/%reldir%/test_cmds.sh_9dfc433f14b811afb3ec4daef0f33e4c9b14a6d7.out \
     $(srcdir)/%reldir%/test_cmds.sh_a00943ef715598c7554b85de8502454e41bb9e28.err \
     $(srcdir)/%reldir%/test_cmds.sh_a00943ef715598c7554b85de8502454e41bb9e28.out \
     $(srcdir)/%reldir%/test_cmds.sh_a1123427c31c022433d66d05ee5d5e1c8ab415e4.err \
@@ -150,6 +160,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_cmds.sh_ac45fb0f8f9578c3ded0855f694698ec38ce31ad.out \
     $(srcdir)/%reldir%/test_cmds.sh_af0fcbd30b3fd0d13477aa3325ef0302052a4d9f.err \
     $(srcdir)/%reldir%/test_cmds.sh_af0fcbd30b3fd0d13477aa3325ef0302052a4d9f.out \
+    $(srcdir)/%reldir%/test_cmds.sh_b3d0588ad144a841200692b46125bddf66f5d8bb.err \
+    $(srcdir)/%reldir%/test_cmds.sh_b3d0588ad144a841200692b46125bddf66f5d8bb.out \
     $(srcdir)/%reldir%/test_cmds.sh_b5a530d16c982cf769151291f0bfd612ea71183f.err \
     $(srcdir)/%reldir%/test_cmds.sh_b5a530d16c982cf769151291f0bfd612ea71183f.out \
     $(srcdir)/%reldir%/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.err \
@@ -184,6 +196,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_cmds.sh_ca66660c973f76a3c2a147c7f5035bcb4e8a8bbc.out \
     $(srcdir)/%reldir%/test_cmds.sh_ccd326da92d1cacda63501cd1a3077381a18e8f2.err \
     $(srcdir)/%reldir%/test_cmds.sh_ccd326da92d1cacda63501cd1a3077381a18e8f2.out \
+    $(srcdir)/%reldir%/test_cmds.sh_d1afefacbdd387f02562c8633968b0162a588502.err \
+    $(srcdir)/%reldir%/test_cmds.sh_d1afefacbdd387f02562c8633968b0162a588502.out \
     $(srcdir)/%reldir%/test_cmds.sh_d3b69abdfb39e4bfa5828c2f9593e2b2b7ed4d5d.err \
     $(srcdir)/%reldir%/test_cmds.sh_d3b69abdfb39e4bfa5828c2f9593e2b2b7ed4d5d.out \
     $(srcdir)/%reldir%/test_cmds.sh_d76d77ad95b9f120825417a6a8220c13df9541fc.err \

test/expected/test_cli.sh_a1a09f890f4604309d0a81bbbec8e50fb7d5e887.out

@@ -1,3 +0,0 @@
-2013-06-06T19:13:20.123  Hello, World!
-2013-06-06T19:13:20.123  Goodbye, World!
-2013-06-06T19:13:20.123  ---- END-OF-STDIN ----

test/expected/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.out

@@ -0,0 +1,4 @@
+Feb 25 16:20:12 192.168.4.2 haproxy[7]: 95.216.197.33:56224 [25/Feb/2019:16:20:10.111] prod_http_in/sktst2: SSL handshake failure
+Feb 25 16:20:12 192.168.4.2 haproxy[7]: 87.183.41.77:50189 [25/Feb/2019:16:20:12.331] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 2496 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/core/core.css?1550939640 HTTP/1.1"
+Feb 25 16:20:12 192.168.4.2 haproxy[7]: 87.183.41.77:50187 [25/Feb/2019:16:20:12.325] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 1859 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/pi_popup/1.1.0/magnific-popup.css?1550939704 HTTP/1.1"
+Feb 25 16:20:12 192.168.4.2 haproxy[7]: 87.183.41.77:50188 [25/Feb/2019:16:20:12.321] prod_http_in~ bk_ktest_sonst/nginx_sonst 0/0/1/0/1 200 5959 - - ---- 9/9/0/0/0 0/0 {Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0} {} "GET /media/pi_fontawesome/css/font-awesome.css?1550939694 HTTP/1.1"

test/expected/test_cli.sh_ff7da172f4350a2adb74b8764575823d798ed8b6.out

@@ -0,0 +1,2 @@
+Hello, World!
+Goodbye, World!

test/expected/test_cmds.sh_015ffe79a08f4c9f0cd1cb84c6afa4398f879fc7.err

@@ -0,0 +1,7 @@
+✘ error: cannot access -- /bad-dir
+ reason: No such file or directory
+ --> command-option:1
+ | :cd /bad-dir                            
+ = help: :cd dir
+         ══════════════════════════════════════════════════════════════════════
+           Change the current directory

test/expected/test_cmds.sh_68c774418bac897bd4d4fe9dbbf08454886b2e15.out

@@ -0,0 +1,3 @@
+192.168.202.254 - - [20/Jul/2009:22:59:26 +0000] "GET /vmw/cgi/tramp HTTP/1.0" 200 134 "-" "gPXE/0.9.7"
+192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkboot.gz HTTP/1.0" 404 46210 "-" "gPXE/0.9.7"
+192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkernel.gz HTTP/1.0" 200 78929 "-" "gPXE/0.9.7"

test/expected/test_cmds.sh_9527f941dc84a2ac3a030f222e41c6ccd1961cbe.out

@@ -0,0 +1 @@
+Hello, World!

test/expected/test_cmds.sh_9dfc433f14b811afb3ec4daef0f33e4c9b14a6d7.out

@@ -0,0 +1 @@
+/bin/bash: bad-command: command not found

test/expected/test_cmds.sh_b3d0588ad144a841200692b46125bddf66f5d8bb.out

@@ -0,0 +1 @@
+Hello, World!

test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out

@@ -702,7 +702,7 @@ For support questions, email:
 Parameter
   msg   The message to display
 See Also
-  :echo, :eval, :export-session-to, :rebuild, :redirect-to, 
+  :cd, :echo, :eval, :export-session-to, :rebuild, :redirect-to, :sh, 
   :write-csv-to, :write-json-to, :write-jsonlines-to, :write-raw-to, 
   :write-screen-to, :write-table-to, :write-to, :write-view-to
 Example
@@ -726,6 +726,16 @@ For support questions, email:
    
 
 
+:cd dir
+══════════════════════════════════════════════════════════════════════
+  Change the current directory
+Parameter
+  dir   The new current directory
+See Also
+  :alt-msg, :echo, :eval, :export-session-to, :rebuild, :redirect-to, 
+  :sh, :write-csv-to, :write-json-to, :write-jsonlines-to, :write-raw-to, 
+  :write-screen-to, :write-table-to, :write-to, :write-view-to
+
 :clear-comment
 ══════════════════════════════════════════════════════════════════════
   Clear the comment attached to the top log line
@@ -934,12 +944,13 @@ For support questions, email:
   -n    Do not print a line-feed at the end of the output
   msg   The message to display
 See Also
-  :alt-msg, :append-to, :eval, :export-session-to, :export-session-to, 
+  :alt-msg, :append-to, :cd, :eval, :export-session-to, 
-  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, 
+  :export-session-to, :pipe-line-to, :pipe-to, :rebuild, :redirect-to, 
-  :write-csv-to, :write-csv-to, :write-json-to, :write-json-to, 
+  :redirect-to, :sh, :write-csv-to, :write-csv-to, :write-json-to, 
-  :write-jsonlines-to, :write-jsonlines-to, :write-raw-to, :write-raw-to, 
+  :write-json-to, :write-jsonlines-to, :write-jsonlines-to, 
-  :write-screen-to, :write-screen-to, :write-table-to, :write-table-to, 
+  :write-raw-to, :write-raw-to, :write-screen-to, :write-screen-to, 
-  :write-to, :write-to, :write-view-to, :write-view-to, echoln()
+  :write-table-to, :write-table-to, :write-to, :write-to, :write-view-to, 
+  :write-view-to, echoln()
 Example
 #1 To output 'Hello, World!':
    :echo Hello, World!                               
@@ -974,7 +985,7 @@ For support questions, email:
 Parameter
   command   The command or query to perform substitution on.
 See Also
-  :alt-msg, :echo, :export-session-to, :rebuild, :redirect-to, 
+  :alt-msg, :cd, :echo, :export-session-to, :rebuild, :redirect-to, :sh, 
   :write-csv-to, :write-json-to, :write-jsonlines-to, :write-raw-to, 
   :write-screen-to, :write-table-to, :write-to, :write-view-to
 Example
@@ -990,9 +1001,9 @@ For support questions, email:
 Parameter
   path   The path to the file to write
 See Also
-  :alt-msg, :append-to, :echo, :echo, :eval, :pipe-line-to, :pipe-to, 
+  :alt-msg, :append-to, :cd, :echo, :echo, :eval, :pipe-line-to, 
-  :rebuild, :redirect-to, :redirect-to, :write-csv-to, :write-csv-to, 
+  :pipe-to, :rebuild, :redirect-to, :redirect-to, :sh, :write-csv-to, 
-  :write-json-to, :write-json-to, :write-jsonlines-to, 
+  :write-csv-to, :write-json-to, :write-json-to, :write-jsonlines-to, 
   :write-jsonlines-to, :write-raw-to, :write-raw-to, :write-screen-to, 
   :write-screen-to, :write-table-to, :write-table-to, :write-to, 
   :write-to, :write-view-to, :write-view-to, echoln()
@@ -1336,7 +1347,7 @@ For support questions, email:
 ══════════════════════════════════════════════════════════════════════
   Forcefully rebuild file indexes
 See Also
-  :alt-msg, :echo, :eval, :export-session-to, :redirect-to, 
+  :alt-msg, :cd, :echo, :eval, :export-session-to, :redirect-to, :sh, 
   :write-csv-to, :write-json-to, :write-jsonlines-to, :write-raw-to, 
   :write-screen-to, :write-table-to, :write-to, :write-view-to
 
@@ -1348,12 +1359,12 @@ For support questions, email:
   path   The path to the file to write.  If not specified, the
          current redirect will be cleared
 See Also
-  :alt-msg, :append-to, :echo, :echo, :eval, :export-session-to, 
+  :alt-msg, :append-to, :cd, :echo, :echo, :eval, :export-session-to, 
-  :export-session-to, :pipe-line-to, :pipe-to, :rebuild, :write-csv-to, 
+  :export-session-to, :pipe-line-to, :pipe-to, :rebuild, :sh, 
-  :write-csv-to, :write-json-to, :write-json-to, :write-jsonlines-to, 
+  :write-csv-to, :write-csv-to, :write-json-to, :write-json-to, 
-  :write-jsonlines-to, :write-raw-to, :write-raw-to, :write-screen-to, 
+  :write-jsonlines-to, :write-jsonlines-to, :write-raw-to, :write-raw-to, 
-  :write-screen-to, :write-table-to, :write-table-to, :write-to, 
+  :write-screen-to, :write-screen-to, :write-table-to, :write-table-to, 
-  :write-to, :write-view-to, :write-view-to, echoln()
+  :write-to, :write-to, :write-view-to, :write-view-to, echoln()
 Example
 #1 To write the output of lnav commands to the file /tmp/script-output.txt:
    :redirect-to /tmp/script-output.txt               
@@ -1430,6 +1441,17 @@ For support questions, email:
    
 
 
+:sh cmdline
+══════════════════════════════════════════════════════════════════════
+  Execute the given command-line and display the captured output
+Parameter
+  cmdline   The command-line to execute.
+See Also
+  :alt-msg, :cd, :echo, :eval, :export-session-to, :rebuild, 
+  :redirect-to, :write-csv-to, :write-json-to, :write-jsonlines-to, 
+  :write-raw-to, :write-screen-to, :write-table-to, :write-to, 
+  :write-view-to
+
 :show-fields field-name1 [... field-nameN]
 ══════════════════════════════════════════════════════════════════════
   Show log message fields that were previously hidden
@@ -1577,9 +1599,9 @@ For support questions, email:
   --anonymize   Anonymize the table contents
   path          The path to the file to write
 See Also
-  :alt-msg, :append-to, :create-logline-table, :create-search-table, 
+  :alt-msg, :append-to, :cd, :create-logline-table, :create-search-table, 
   :echo, :echo, :eval, :export-session-to, :export-session-to, 
-  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, 
+  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, :sh, 
   :write-csv-to, :write-csv-to, :write-csv-to, :write-json-to, 
   :write-json-to, :write-json-to, :write-jsonlines-to, 
   :write-jsonlines-to, :write-jsonlines-to, :write-raw-to, :write-raw-to, 
@@ -1599,9 +1621,9 @@ For support questions, email:
   --anonymize   Anonymize the row contents
   path          The path to the file to write
 See Also
-  :alt-msg, :append-to, :create-logline-table, :create-search-table, 
+  :alt-msg, :append-to, :cd, :create-logline-table, :create-search-table, 
   :echo, :echo, :eval, :export-session-to, :export-session-to, 
-  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, 
+  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, :sh, 
   :write-json-to, :write-json-to, :write-json-to, :write-jsonlines-to, 
   :write-jsonlines-to, :write-jsonlines-to, :write-raw-to, :write-raw-to, 
   :write-raw-to, :write-screen-to, :write-screen-to, :write-screen-to, 
@@ -1620,9 +1642,9 @@ For support questions, email:
   --anonymize   Anonymize the JSON values
   path          The path to the file to write
 See Also
-  :alt-msg, :append-to, :create-logline-table, :create-search-table, 
+  :alt-msg, :append-to, :cd, :create-logline-table, :create-search-table, 
   :echo, :echo, :eval, :export-session-to, :export-session-to, 
-  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, 
+  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, :sh, 
   :write-csv-to, :write-csv-to, :write-csv-to, :write-jsonlines-to, 
   :write-jsonlines-to, :write-jsonlines-to, :write-raw-to, :write-raw-to, 
   :write-raw-to, :write-screen-to, :write-screen-to, :write-screen-to, 
@@ -1641,9 +1663,9 @@ For support questions, email:
   --anonymize   Anonymize the JSON values
   path          The path to the file to write
 See Also
-  :alt-msg, :append-to, :create-logline-table, :create-search-table, 
+  :alt-msg, :append-to, :cd, :create-logline-table, :create-search-table, 
   :echo, :echo, :eval, :export-session-to, :export-session-to, 
-  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, 
+  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, :sh, 
   :write-csv-to, :write-csv-to, :write-csv-to, :write-json-to, 
   :write-json-to, :write-json-to, :write-raw-to, :write-raw-to, 
   :write-raw-to, :write-screen-to, :write-screen-to, :write-screen-to, 
@@ -1666,9 +1688,9 @@ For support questions, email:
   --anonymize       Anonymize the lines
   path              The path to the file to write
 See Also
-  :alt-msg, :append-to, :create-logline-table, :create-search-table, 
+  :alt-msg, :append-to, :cd, :create-logline-table, :create-search-table, 
   :echo, :echo, :eval, :export-session-to, :export-session-to, 
-  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, 
+  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, :sh, 
   :write-csv-to, :write-csv-to, :write-csv-to, :write-json-to, 
   :write-json-to, :write-json-to, :write-jsonlines-to, 
   :write-jsonlines-to, :write-jsonlines-to, :write-screen-to, 
@@ -1689,9 +1711,9 @@ For support questions, email:
   --anonymize   Anonymize the lines
   path          The path to the file to write
 See Also
-  :alt-msg, :append-to, :create-logline-table, :create-search-table, 
+  :alt-msg, :append-to, :cd, :create-logline-table, :create-search-table, 
   :echo, :echo, :eval, :export-session-to, :export-session-to, 
-  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, 
+  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, :sh, 
   :write-csv-to, :write-csv-to, :write-csv-to, :write-json-to, 
   :write-json-to, :write-json-to, :write-jsonlines-to, 
   :write-jsonlines-to, :write-jsonlines-to, :write-raw-to, :write-raw-to, 
@@ -1711,9 +1733,9 @@ For support questions, email:
   --anonymize   Anonymize the table contents
   path          The path to the file to write
 See Also
-  :alt-msg, :append-to, :create-logline-table, :create-search-table, 
+  :alt-msg, :append-to, :cd, :create-logline-table, :create-search-table, 
   :echo, :echo, :eval, :export-session-to, :export-session-to, 
-  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, 
+  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, :sh, 
   :write-csv-to, :write-csv-to, :write-csv-to, :write-json-to, 
   :write-json-to, :write-json-to, :write-jsonlines-to, 
   :write-jsonlines-to, :write-jsonlines-to, :write-raw-to, :write-raw-to, 
@@ -1733,9 +1755,9 @@ For support questions, email:
   --anonymize   Anonymize the lines
   path          The path to the file to write
 See Also
-  :alt-msg, :append-to, :echo, :echo, :eval, :export-session-to, 
+  :alt-msg, :append-to, :cd, :echo, :echo, :eval, :export-session-to, 
   :export-session-to, :pipe-line-to, :pipe-to, :rebuild, :redirect-to, 
-  :redirect-to, :write-csv-to, :write-csv-to, :write-json-to, 
+  :redirect-to, :sh, :write-csv-to, :write-csv-to, :write-json-to, 
   :write-json-to, :write-jsonlines-to, :write-jsonlines-to, 
   :write-raw-to, :write-raw-to, :write-screen-to, :write-screen-to, 
   :write-table-to, :write-table-to, :write-view-to, :write-view-to, 
@@ -1754,9 +1776,9 @@ For support questions, email:
   --anonymize   Anonymize the lines
   path          The path to the file to write
 See Also
-  :alt-msg, :append-to, :create-logline-table, :create-search-table, 
+  :alt-msg, :append-to, :cd, :create-logline-table, :create-search-table, 
   :echo, :echo, :eval, :export-session-to, :export-session-to, 
-  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, 
+  :pipe-line-to, :pipe-to, :rebuild, :redirect-to, :redirect-to, :sh, 
   :write-csv-to, :write-csv-to, :write-csv-to, :write-json-to, 
   :write-json-to, :write-json-to, :write-jsonlines-to, 
   :write-jsonlines-to, :write-jsonlines-to, :write-raw-to, :write-raw-to, 

test/expected/test_cmds.sh_d1afefacbdd387f02562c8633968b0162a588502.err

@@ -0,0 +1,6 @@
+✘ error: {test_dir}/logfile_access_log.0 is not a directory
+ --> command-option:1
+ | :cd {test_dir}/logfile_access_log.0
+ = help: :cd dir
+         ══════════════════════════════════════════════════════════════════════
+           Change the current directory

test/test_cli.sh

@@ -5,7 +5,7 @@ export YES_COLOR=1
 
 run_cap_test ${lnav_test} -n -c 'foo'
 
-run_cap_test ${lnav_test} -d /tmp/lnav.err -t -n <<EOF
+run_cap_test ${lnav_test} -d /tmp/lnav.err -n <<EOF
 Hello, World!
 Goodbye, World!
 EOF
@@ -29,3 +29,12 @@ printf "a\ba _\ba a\b_" | run_cap_test env TEST_COMMENT="overstrike bold" \
 grep abcd textfile_long_lines.0 | run_cap_test \
     ${lnav_test} -n -d /tmp/lnav.err \
     -c ';SELECT filepath, lines FROM lnav_file'
+
+export HOME="./piper-config"
+rm -rf ./piper-config
+mkdir -p $HOME/.config
+
+${lnav_test} -Nn -c ':config /tuning/piper/max-size 128'
+
+cat ${test_dir}/logfile_haproxy.0 | run_cap_test \
+    env TEST_COMMENT="stdin rotation" ${lnav_test} -n

test/test_cmds.sh

@@ -2,6 +2,25 @@
 
 export YES_COLOR=1
 
+run_cap_test ${lnav_test} -nN \
+    -c ":cd /bad-dir"
+
+run_cap_test ${lnav_test} -nN \
+    -c ":cd ${test_dir}/logfile_access_log.0"
+
+run_cap_test ${lnav_test} -nN \
+    -c ":cd ${test_dir}" \
+    -c ":open logfile_access_log.0"
+
+run_cap_test env SHELL=/bin/bash ${lnav_test} -nN \
+    -e "bad-command"
+
+run_cap_test ${lnav_test} -nN \
+    -e "echo Hello, World!"
+
+run_cap_test ${lnav_test} -nN \
+    -e "echo Hello, World! > /dev/stderr"
+
 run_cap_test ${lnav_test} -n \
     -c ":switch-to-view help" \
     ${test_dir}/logfile_access_log.0

test/test_logfile.sh

@@ -37,17 +37,18 @@ run_cap_test ${lnav_test} -n \
     -c ';SELECT * FROM logline' \
     ${test_dir}/logfile_block.1
 
-run_test ${lnav_test} -d /tmp/lnav.err -n -w logfile_stdin.0.log \
+if test x"${TSHARK_CMD}" != x""; then
-    -c ':shexec sleep 1 && touch -t 200711030923 logfile_stdin.0.log' <<EOF
+  run_test env TZ=UTC ${lnav_test} -n ${test_dir}/dhcp.pcapng
-2013-06-06T19:13:20.123  Hi
-EOF
 
-check_output "piping to stdin is not working?" <<EOF
+  check_output "pcap file is not recognized" <<EOF
-2013-06-06T19:13:20.123  Hi
+2004-12-05T19:16:24.317 0.0.0.0 → 255.255.255.255 DHCP 314 DHCP Discover - Transaction ID 0x3d1d
+2004-12-05T19:16:24.317 192.168.0.1 → 192.168.0.10 DHCP 342 DHCP Offer    - Transaction ID 0x3d1d
+2004-12-05T19:16:24.387 0.0.0.0 → 255.255.255.255 DHCP 314 DHCP Request  - Transaction ID 0x3d1e
+2004-12-05T19:16:24.387 192.168.0.1 → 192.168.0.10 DHCP 342 DHCP ACK      - Transaction ID 0x3d1e
 EOF
 
-if test x"${TSHARK_CMD}" != x""; then
+  # make sure piped binary data is left alone
-  run_test env TZ=UTC ${lnav_test} -n ${test_dir}/dhcp.pcapng
+  run_test cat ${test_dir}/dhcp.pcapng | env TZ=UTC ${lnav_test} -n
 
   check_output "pcap file is not recognized" <<EOF
 2004-12-05T19:16:24.317 0.0.0.0 → 255.255.255.255 DHCP 314 DHCP Discover - Transaction ID 0x3d1d
@@ -59,7 +60,8 @@ EOF
   run_test ${lnav_test} -n ${test_dir}/dhcp-trunc.pcapng
 
   check_error_output "truncated pcap file is not recognized" <<EOF
-error: unable to open file: {test_dir}/dhcp-trunc.pcapng -- tshark: The file "{test_dir}/dhcp-trunc.pcapng" appears to have been cut short in the middle of a packet.
+✘ error: unable to open file: {test_dir}/dhcp-trunc.pcapng
+ reason: tshark: The file "{test_dir}/dhcp-trunc.pcapng" appears to have been cut short in the middle of a packet.
 EOF
 fi
 
@@ -588,15 +590,6 @@ info 0x0
 error 0x0
 EOF
 
-run_test ${lnav_test} -d /tmp/lnav.err -nt -w logfile_stdin.log <<EOF
-Hi
-EOF
-
-check_output "piping to stdin is not working?" <<EOF
-2013-06-06T19:13:20.123  Hi
-2013-06-06T19:13:20.123  ---- END-OF-STDIN ----
-EOF
-
 run_test ${lnav_test} -C ${test_dir}/logfile_bad_access_log.0
 
 sed -ibak -e "s|/.*/logfile_bad_access_log.0|logfile_bad_access_log.0|g" `test_err_filename`