[:eval] treat the argument like the contents of a file with multiple commands

NEWS.md

@@ -32,9 +32,12 @@ Features:
   lnav script.  Schemes can be defined under
   `/tuning/url-schemes`.  See the main docs for more details.
 * Added a `docker://` URL scheme that can be used to tail
-  the logs for a container (e.g. `docker://my-container`) or
+  the logs for containers (e.g. `docker://my-container`) or
   files within a container (e.g.
-  `docker://my-serv/var/log/dpkg.log`).
+  `docker://my-serv/var/log/dpkg.log`).  Containers mentioned
+  in a "Compose" configuration file can be tailed by using
+  `compose` as the host name with the path to the configuration
+  file (e.g. `docker://compose/compose.yaml`).
 * Added an `:annotate` command that can trigger a call-out
   to a script to analyze a log message and generate an
   annotation that is attached to the message.  The script
@@ -85,6 +88,9 @@ Features:
   are now recognized and styled as appropriate.
 * Added a `data` column to the `fstat()` table-valued-
   function so the contents of a file can be read.
+* Added a log format for Redis.
+* The `:eval` command will now treat its argument(s) as a
+  script, allowing multiple commands to be executed.
 
 Bug Fixes:
 * Binary data piped into stdin should now be treated the same

docs/schemas/config-v1.schema.json

@@ -214,7 +214,7 @@
                     "title": "/tuning/url-scheme",
                     "type": "object",
                     "patternProperties": {
-                        "(\\w+)": {
+                        "([a-z][\\w\\-\\+\\.]+)": {
                             "description": "Definition of a custom URL scheme",
                             "title": "/tuning/url-scheme/<url_scheme>",
                             "type": "object",

src/CMakeLists.txt

@@ -96,6 +96,7 @@ set(TIME_FORMATS
         "%m/%e/%Y %l:%M:%S%p"
         "%m/%d/%y %H:%M:%S"
         "%m/%d/%Y %H:%M:%S"
+        "%d/%b/%Y %H:%M:%S"
         "%d/%b/%y %H:%M:%S"
         "%m%d %H:%M:%S"
         "%Y%m%d %H:%M:%S"

src/base/intern_string.cc

@@ -225,7 +225,9 @@ string_fragment::split_lines() const
             start = index + 1;
         }
     }
-    retval.emplace_back(this->sf_string, start, this->sf_end);
+    if (retval.empty() || start < this->sf_end) {
+        retval.emplace_back(this->sf_string, start, this->sf_end);
+    }
 
     return retval;
 }

src/base/math_util.hh

@@ -124,6 +124,10 @@ public:
 
     bool try_consume(T rhs)
     {
+        if (rhs == 0) {
+            return false;
+        }
+
         if (this->c_value - rhs > this->c_min) {
             this->c_value -= rhs;
             return true;

src/command_executor.cc

@@ -106,7 +106,7 @@ sql_progress_finished()
 
 static Result<std::string, lnav::console::user_message> execute_from_file(
     exec_context& ec,
-    const ghc::filesystem::path& path,
+    const std::string& src,
     int line_number,
     const std::string& cmdline);
 
@@ -499,6 +499,7 @@ execute_sql(exec_context& ec, const std::string& sql, std::string& alt_msg)
 
             if (lnav_data.ld_flags & LNF_HEADLESS) {
                 if (ec.ec_local_vars.size() == 1) {
+                    lnav_data.ld_views[LNV_DB].reload_data();
                     ensure_view(&lnav_data.ld_views[LNV_DB]);
                 }
             }
@@ -509,13 +510,73 @@ execute_sql(exec_context& ec, const std::string& sql, std::string& alt_msg)
     return Ok(retval);
 }
 
+Result<void, lnav::console::user_message>
+multiline_executor::push_back(string_fragment line)
+{
+    this->me_line_number += 1;
+
+    if (line.trim().empty()) {
+        if (this->me_cmdline) {
+            this->me_cmdline = this->me_cmdline.value() + "\n";
+        }
+        return Ok();
+    }
+    if (line[0] == '#') {
+        return Ok();
+    }
+
+    switch (line[0]) {
+        case ':':
+        case '/':
+        case ';':
+        case '|':
+            if (this->me_cmdline) {
+                this->me_last_result
+                    = TRY(execute_from_file(this->me_exec_context,
+                                            this->me_source,
+                                            this->me_starting_line_number,
+                                            trim(this->me_cmdline.value())));
+            }
+
+            this->me_starting_line_number = this->me_line_number;
+            this->me_cmdline = line.to_string();
+            break;
+        default:
+            if (this->me_cmdline) {
+                this->me_cmdline = fmt::format(
+                    FMT_STRING("{}{}"), this->me_cmdline.value(), line);
+            } else {
+                this->me_last_result = TRY(
+                    execute_from_file(this->me_exec_context,
+                                      this->me_source,
+                                      this->me_line_number,
+                                      fmt::format(FMT_STRING(":{}"), line)));
+            }
+            break;
+    }
+
+    return Ok();
+}
+
+Result<std::string, lnav::console::user_message>
+multiline_executor::final()
+{
+    if (this->me_cmdline) {
+        this->me_last_result
+            = TRY(execute_from_file(this->me_exec_context,
+                                    this->me_source,
+                                    this->me_starting_line_number,
+                                    trim(this->me_cmdline.value())));
+    }
+
+    return Ok(this->me_last_result);
+}
+
 static Result<std::string, lnav::console::user_message>
-execute_file_contents(exec_context& ec,
-                      const ghc::filesystem::path& path,
-                      bool multiline)
+execute_file_contents(exec_context& ec, const ghc::filesystem::path& path)
 {
-    static ghc::filesystem::path stdin_path("-");
-    static ghc::filesystem::path dev_stdin_path("/dev/stdin");
+    static const ghc::filesystem::path stdin_path("-");
+    static const ghc::filesystem::path dev_stdin_path("/dev/stdin");
 
     std::string retval;
     FILE* file;
@@ -529,59 +590,18 @@ execute_file_contents(exec_context& ec,
         return ec.make_error("unable to open file");
     }
 
-    int line_number = 0, starting_line_number = 0;
     auto_mem<char> line;
     size_t line_max_size;
     ssize_t line_size;
-    nonstd::optional<std::string> cmdline;
+    multiline_executor me(ec, path.string());
 
     ec.ec_path_stack.emplace_back(path.parent_path());
     exec_context::output_guard og(ec);
     while ((line_size = getline(line.out(), &line_max_size, file)) != -1) {
-        line_number += 1;
-
-        if (trim(line.in()).empty()) {
-            if (multiline && cmdline) {
-                cmdline = cmdline.value() + "\n";
-            }
-            continue;
-        }
-        if (line[0] == '#') {
-            continue;
-        }
-
-        switch (line[0]) {
-            case ':':
-            case '/':
-            case ';':
-            case '|':
-                if (cmdline) {
-                    retval = TRY(execute_from_file(
-                        ec, path, starting_line_number, trim(cmdline.value())));
-                }
-
-                starting_line_number = line_number;
-                cmdline = std::string(line);
-                break;
-            default:
-                if (multiline && cmdline) {
-                    cmdline = fmt::format(
-                        FMT_STRING("{}{}"), cmdline.value(), line.in());
-                } else {
-                    retval = TRY(execute_from_file(
-                        ec,
-                        path,
-                        line_number,
-                        fmt::format(FMT_STRING(":{}"), line.in())));
-                }
-                break;
-        }
+        TRY(me.push_back(string_fragment::from_bytes(line.in(), line_size)));
     }
 
-    if (cmdline) {
-        retval = TRY(execute_from_file(
-            ec, path, starting_line_number, trim(cmdline.value())));
-    }
+    retval = TRY(me.final());
 
     if (file == stdin) {
         if (isatty(STDOUT_FILENO)) {
@@ -596,25 +616,35 @@ execute_file_contents(exec_context& ec,
 }
 
 Result<std::string, lnav::console::user_message>
-execute_file(exec_context& ec, const std::string& path_and_args, bool multiline)
+execute_file(exec_context& ec, const std::string& path_and_args)
 {
+    static const intern_string_t SRC = intern_string::lookup("cmdline");
+
     available_scripts scripts;
-    std::vector<std::string> split_args;
     std::string retval, msg;
     shlex lexer(path_and_args);
 
     log_info("Executing file: %s", path_and_args.c_str());
 
-    if (!lexer.split(split_args, scoped_resolver{&ec.ec_local_vars.top()})) {
-        return ec.make_error("unable to parse path");
+    auto split_args_res = lexer.split(scoped_resolver{&ec.ec_local_vars.top()});
+    if (split_args_res.isErr()) {
+        auto split_err = split_args_res.unwrapErr();
+        auto um = lnav::console::user_message::error(
+                      "unable to parse script command-line")
+                      .with_reason(split_err.te_msg)
+                      .with_snippet(lnav::console::snippet::from(
+                          SRC, lexer.to_attr_line(split_err)));
+
+        return Err(um);
     }
+    auto split_args = split_args_res.unwrap();
     if (split_args.empty()) {
         return ec.make_error("no script specified");
     }
 
     ec.ec_local_vars.push({});
 
-    auto script_name = split_args[0];
+    auto script_name = split_args[0].se_value;
     auto& vars = ec.ec_local_vars.top();
     char env_arg_name[32];
     std::string star, open_error = "file not found";
@@ -627,13 +657,13 @@ execute_file(exec_context& ec, const std::string& path_and_args, bool multiline)
     vars["#"] = env_arg_name;
     for (size_t lpc = 0; lpc < split_args.size(); lpc++) {
         snprintf(env_arg_name, sizeof(env_arg_name), "%lu", lpc);
-        vars[env_arg_name] = split_args[lpc];
+        vars[env_arg_name] = split_args[lpc].se_value;
     }
     for (size_t lpc = 1; lpc < split_args.size(); lpc++) {
         if (lpc > 1) {
             star.append(" ");
         }
-        star.append(split_args[lpc]);
+        star.append(split_args[lpc].se_value);
     }
     vars["__all__"] = star;
 
@@ -674,8 +704,7 @@ execute_file(exec_context& ec, const std::string& path_and_args, bool multiline)
 
     if (!paths_to_exec.empty()) {
         for (auto& path_iter : paths_to_exec) {
-            retval
-                = TRY(execute_file_contents(ec, path_iter.sm_path, multiline));
+            retval = TRY(execute_file_contents(ec, path_iter.sm_path));
         }
     }
     ec.ec_local_vars.pop();
@@ -690,13 +719,13 @@ execute_file(exec_context& ec, const std::string& path_and_args, bool multiline)
 
 Result<std::string, lnav::console::user_message>
 execute_from_file(exec_context& ec,
-                  const ghc::filesystem::path& path,
+                  const std::string& src,
                   int line_number,
                   const std::string& cmdline)
 {
     std::string retval, alt_msg;
-    auto _sg = ec.enter_source(
-        intern_string::lookup(path.string()), line_number, cmdline);
+    auto _sg
+        = ec.enter_source(intern_string::lookup(src), line_number, cmdline);
 
     switch (cmdline[0]) {
         case ':':
@@ -717,10 +746,8 @@ execute_from_file(exec_context& ec,
             break;
     }
 
-    log_info("%s:%d:execute result -- %s",
-             path.c_str(),
-             line_number,
-             retval.c_str());
+    log_info(
+        "%s:%d:execute result -- %s", src.c_str(), line_number, retval.c_str());
 
     return Ok(retval);
 }
@@ -842,6 +869,7 @@ execute_init_commands(
             }
             if (dls.dls_rows.size() > 1 && lnav_data.ld_view_stack.size() == 1)
             {
+                lnav_data.ld_views[LNV_DB].reload_data();
                 ensure_view(LNV_DB);
             }
         }

src/command_executor.hh

@@ -284,8 +284,28 @@ Result<std::string, lnav::console::user_message> execute_command(
 
 Result<std::string, lnav::console::user_message> execute_sql(
     exec_context& ec, const std::string& sql, std::string& alt_msg);
+
+class multiline_executor {
+public:
+    exec_context& me_exec_context;
+    std::string me_source;
+    nonstd::optional<std::string> me_cmdline;
+    int me_line_number{0};
+    int me_starting_line_number{0};
+    std::string me_last_result;
+
+    multiline_executor(exec_context& ec, std::string src)
+        : me_exec_context(ec), me_source(src)
+    {
+    }
+
+    Result<void, lnav::console::user_message> push_back(string_fragment line);
+
+    Result<std::string, lnav::console::user_message> final();
+};
+
 Result<std::string, lnav::console::user_message> execute_file(
-    exec_context& ec, const std::string& path_and_args, bool multiline = true);
+    exec_context& ec, const std::string& path_and_args);
 Result<std::string, lnav::console::user_message> execute_any(
     exec_context& ec, const std::string& cmdline);
 void execute_init_commands(

src/formats/access_log.json

@@ -111,6 +111,9 @@
             {
                 "line": "10.112.2.3 - - [16/Sep/2022:00:53:14 +0200] \"POST /api/v4/jobs/request HTTP/1.1\" 204 0 \"\" \"gitlab-runner 15.3.0 (15-3-stable; go1.19; linux/amd64)\" -",
                 "level": "info"
+            },
+            {
+                "line": "172.18.0.1 - - [29/Aug/2023 22:02:58] \"GET / HTTP/1.1\" 200 -"
             }
         ]
     }

src/formats/formats.am

@@ -27,6 +27,7 @@ FORMAT_FILES = \
     $(srcdir)/%reldir%/papertrail_log.json \
     $(srcdir)/%reldir%/pcap_log.json \
     $(srcdir)/%reldir%/procstate_log.json \
+    $(srcdir)/%reldir%/redis_log.json \
     $(srcdir)/%reldir%/snaplogic_log.json \
     $(srcdir)/%reldir%/sssd_log.json \
     $(srcdir)/%reldir%/strace_log.json \

src/formats/pcap_log.json

@@ -4,7 +4,6 @@
         "json": true,
         "title": "Packet Capture",
         "description": "Internal format for pcap files",
-        "multiline": false,
         "convert-to-local-time": true,
         "converter": {
             "header": {

src/formats/redis_log.json

@@ -0,0 +1,51 @@
+{
+    "$schema": "https://lnav.org/schemas/format-v1.schema.json",
+    "redis_log": {
+        "url": [
+            "https://redis.com",
+            "https://build47.com/redis-log-format-levels/"
+        ],
+        "description": "The Redis database",
+        "regex": {
+            "v3.x": {
+                "pattern": "(?<pid>\\d+):(?<role>[XCSM])\\s+(?<timestamp>\\d{1,2} [a-zA-Z]{3} \\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{3})\\s+(?<level>[\\.\\0\\*\\#])\\s+(?<body>.*)"
+            },
+            "sig": {
+                "pattern": "(?<pid>\\d+):signal-handler \\((?<timestamp>\\d+)\\) (?<body>.*)"
+            }
+        },
+        "timestamp-format": [
+            "%s",
+            "%d %b %Y %H:%M:%S.%L"
+        ],
+        "level": {
+            "debug": "^\\.$",
+            "trace": "^-$",
+            "notice": "^\\*$",
+            "warning": "^#$"
+        },
+        "value": {
+            "level": {
+                "kind": "string"
+            },
+            "pid": {
+                "kind": "string",
+                "identifier": true
+            },
+            "role": {
+                "kind": "string"
+            },
+            "timestamp": {
+                "kind": "string"
+            }
+        },
+        "sample": [
+            {
+                "line": "1:M 29 Aug 2023 13:47:38.984 * monotonic clock: POSIX clock_gettime"
+            },
+            {
+                "line": "1:signal-handler (1693279182) Received SIGTERM scheduling shutdown..."
+            }
+        ]
+    }
+}

src/internals/cmd-ref.rst

@@ -1209,12 +1209,13 @@
 
 .. _sh:
 
-:sh *cmdline*
-^^^^^^^^^^^^^
+:sh *--name=<name>* *cmdline*
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
   Execute the given command-line and display the captured output
 
   **Parameters**
+    * **--name=<name>\*** --- The name to give to the captured output
     * **cmdline\*** --- The command-line to execute.
 
   **See Also**

src/listview_curses.cc

@@ -255,6 +255,14 @@ listview_curses::handle_key(int ch)
 
         case 'g':
         case KEY_HOME:
+            if (this->lv_overlay_focused) {
+                this->lv_focused_overlay_top = 0_vl;
+                this->lv_focused_overlay_selection = 0_vl;
+                this->lv_source->listview_selection_changed(*this);
+                this->set_needs_update();
+                break;
+            }
+
             if (this->is_selectable()) {
                 this->set_selection(0_vl);
             } else {
@@ -264,6 +272,23 @@ listview_curses::handle_key(int ch)
 
         case 'G':
         case KEY_END: {
+            if (this->lv_overlay_focused) {
+                std::vector<attr_line_t> overlay_content;
+                this->lv_overlay_source->list_value_for_overlay(
+                    *this, this->get_selection(), overlay_content);
+                auto overlay_height
+                    = this->get_overlay_height(overlay_content.size(), height);
+                auto ov_top_for_last = vis_line_t{
+                    static_cast<int>(overlay_content.size() - overlay_height)};
+
+                this->lv_focused_overlay_top = ov_top_for_last;
+                this->lv_focused_overlay_selection
+                    = vis_line_t(overlay_content.size() - 1);
+                this->lv_source->listview_selection_changed(*this);
+                this->set_needs_update();
+                break;
+            }
+
             vis_line_t last_line(this->get_inner_height() - 1);
             vis_line_t tail_bottom(this->get_top_for_last_row());
 

src/lnav.cc

@@ -2242,6 +2242,8 @@ main(int argc, char* argv[])
 SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
 )";
 
+        lnav_data.ld_child_pollers.clear();
+
         for (auto& lf : lnav_data.ld_active_files.fc_files) {
             lf->close();
         }

src/lnav_commands.cc

@@ -273,7 +273,10 @@ com_unix_time(exec_context& ec,
         char* rest;
 
         u_time = time(nullptr);
-        log_time = *localtime(&u_time);
+        if (localtime_r(&u_time, &log_time) == nullptr) {
+            return ec.make_error(
+                "invalid epoch time: {} -- {}", u_time, strerror(errno));
+        }
 
         log_time.tm_isdst = -1;
 
@@ -294,7 +297,10 @@ com_unix_time(exec_context& ec,
             u_time = mktime(&log_time);
             parsed = true;
         } else if (sscanf(args[1].c_str(), "%ld", &u_time)) {
-            log_time = *localtime(&u_time);
+            if (localtime_r(&u_time, &log_time) == nullptr) {
+                return ec.make_error(
+                    "invalid epoch time: {} -- {}", args[1], strerror(errno));
+            }
 
             parsed = true;
         }
@@ -305,7 +311,7 @@ com_unix_time(exec_context& ec,
             strftime(ftime,
                      sizeof(ftime),
                      "%a %b %d %H:%M:%S %Y  %z %Z",
-                     localtime(&u_time));
+                     localtime_r(&u_time, &log_time));
             len = strlen(ftime);
             snprintf(ftime + len, sizeof(ftime) - len, " -- %ld", u_time);
             retval = std::string(ftime);
@@ -1026,6 +1032,8 @@ com_save_to(exec_context& ec,
             std::string cmdline,
             std::vector<std::string>& args)
 {
+    static const intern_string_t SRC = intern_string::lookup("path");
+
     FILE *outfile = nullptr, *toclose = nullptr;
     const char* mode = "";
     std::string fn, retval;
@@ -1040,13 +1048,22 @@ com_save_to(exec_context& ec,
 
     fn = trim(remaining_args(cmdline, args));
 
-    std::vector<std::string> split_args;
     shlex lexer(fn);
 
-    if (!lexer.split(split_args, ec.create_resolver())) {
-        return ec.make_error("unable to parse arguments");
+    auto split_args_res = lexer.split(ec.create_resolver());
+    if (split_args_res.isErr()) {
+        auto split_err = split_args_res.unwrapErr();
+        auto um
+            = lnav::console::user_message::error("unable to parse file name")
+                  .with_reason(split_err.te_msg)
+                  .with_snippet(lnav::console::snippet::from(
+                      SRC, lexer.to_attr_line(split_err)));
+
+        return Err(um);
     }
 
+    auto split_args = split_args_res.unwrap()
+        | lnav::itertools::map([](const auto& elem) { return elem.se_value; });
     auto anon_iter
         = std::find(split_args.begin(), split_args.end(), "--anonymize");
     if (anon_iter != split_args.end()) {
@@ -1724,6 +1741,8 @@ com_redirect_to(exec_context& ec,
                 std::string cmdline,
                 std::vector<std::string>& args)
 {
+    static const intern_string_t SRC = intern_string::lookup("path");
+
     if (args.empty()) {
         args.emplace_back("filename");
         return Ok(std::string());
@@ -1739,16 +1758,21 @@ com_redirect_to(exec_context& ec,
     }
 
     std::string fn = trim(remaining_args(cmdline, args));
-    std::vector<std::string> split_args;
     shlex lexer(fn);
-    scoped_resolver scopes = {
-        &ec.ec_local_vars.top(),
-        &ec.ec_global_vars,
-    };
 
-    if (!lexer.split(split_args, scopes)) {
-        return ec.make_error("unable to parse arguments");
+    auto split_args_res = lexer.split(ec.create_resolver());
+    if (split_args_res.isErr()) {
+        auto split_err = split_args_res.unwrapErr();
+        auto um
+            = lnav::console::user_message::error("unable to parse file name")
+                  .with_reason(split_err.te_msg)
+                  .with_snippet(lnav::console::snippet::from(
+                      SRC, lexer.to_attr_line(split_err)));
+
+        return Err(um);
     }
+    auto split_args = split_args_res.unwrap()
+        | lnav::itertools::map([](const auto& elem) { return elem.se_value; });
     if (split_args.size() > 1) {
         return ec.make_error("more than one file name was matched");
     }
@@ -2521,6 +2545,7 @@ com_session(exec_context& ec,
 static Result<std::string, lnav::console::user_message>
 com_open(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
 {
+    static const intern_string_t SRC = intern_string::lookup("path");
     std::string retval;
 
     if (args.empty()) {
@@ -2542,17 +2567,22 @@ com_open(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
 
     pat = trim(remaining_args(cmdline, args));
 
-    std::vector<std::string> split_args;
     shlex lexer(pat);
-    scoped_resolver scopes = {
-        &ec.ec_local_vars.top(),
-        &ec.ec_global_vars,
-    };
+    auto split_args_res = lexer.split(ec.create_resolver());
+    if (split_args_res.isErr()) {
+        auto split_err = split_args_res.unwrapErr();
+        auto um
+            = lnav::console::user_message::error("unable to parse file names")
+                  .with_reason(split_err.te_msg)
+                  .with_snippet(lnav::console::snippet::from(
+                      SRC, lexer.to_attr_line(split_err)));
 
-    if (!lexer.split(split_args, scopes)) {
-        return ec.make_error("unable to parse arguments");
+        return Err(um);
     }
 
+    auto split_args = split_args_res.unwrap()
+        | lnav::itertools::map([](const auto& elem) { return elem.se_value; });
+
     std::vector<std::pair<std::string, file_location_t>> files_to_front;
     std::vector<std::string> closed_files;
     logfile_open_options loo;
@@ -2871,6 +2901,7 @@ com_open(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
 static Result<std::string, lnav::console::user_message>
 com_close(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
 {
+    static const intern_string_t SRC = intern_string::lookup("path");
     std::string retval;
 
     if (args.empty()) {
@@ -2885,7 +2916,21 @@ com_close(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
     if (args.size() > 1) {
         auto lexer = shlex(cmdline);
 
-        lexer.split(args, ec.create_resolver());
+        auto split_args_res = lexer.split(ec.create_resolver());
+        if (split_args_res.isErr()) {
+            auto split_err = split_args_res.unwrapErr();
+            auto um = lnav::console::user_message::error(
+                          "unable to parse file name")
+                          .with_reason(split_err.te_msg)
+                          .with_snippet(lnav::console::snippet::from(
+                              SRC, lexer.to_attr_line(split_err)));
+
+            return Err(um);
+        }
+
+        auto args = split_args_res.unwrap()
+            | lnav::itertools::map(
+                        [](const auto& elem) { return elem.se_value; });
         args.erase(args.begin());
 
         for (const auto& lf : lnav_data.ld_active_files.fc_files) {
@@ -2976,6 +3021,7 @@ com_file_visibility(exec_context& ec,
                     std::string cmdline,
                     std::vector<std::string>& args)
 {
+    static const intern_string_t SRC = intern_string::lookup("path");
     bool only_this_file = false;
     bool make_visible;
     std::string retval;
@@ -3031,7 +3077,21 @@ com_file_visibility(exec_context& ec,
         int text_file_count = 0, log_file_count = 0;
         auto lexer = shlex(cmdline);
 
-        lexer.split(args, ec.create_resolver());
+        auto split_args_res = lexer.split(ec.create_resolver());
+        if (split_args_res.isErr()) {
+            auto split_err = split_args_res.unwrapErr();
+            auto um = lnav::console::user_message::error(
+                          "unable to parse file name")
+                          .with_reason(split_err.te_msg)
+                          .with_snippet(lnav::console::snippet::from(
+                              SRC, lexer.to_attr_line(split_err)));
+
+            return Err(um);
+        }
+
+        auto args = split_args_res.unwrap()
+            | lnav::itertools::map(
+                        [](const auto& elem) { return elem.se_value; });
         args.erase(args.begin());
 
         for (const auto& lf : lnav_data.ld_active_files.fc_files) {
@@ -4263,6 +4323,8 @@ com_rebuild(exec_context& ec,
 static Result<std::string, lnav::console::user_message>
 com_cd(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
 {
+    static const intern_string_t SRC = intern_string::lookup("path");
+
     if (args.empty()) {
         args.emplace_back("dirname");
         return Ok(std::string());
@@ -4277,17 +4339,22 @@ com_cd(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
 
     pat = trim(remaining_args(cmdline, args));
 
-    std::vector<std::string> split_args;
     shlex lexer(pat);
-    scoped_resolver scopes = {
-        &ec.ec_local_vars.top(),
-        &ec.ec_global_vars,
-    };
+    auto split_args_res = lexer.split(ec.create_resolver());
+    if (split_args_res.isErr()) {
+        auto split_err = split_args_res.unwrapErr();
+        auto um
+            = lnav::console::user_message::error("unable to parse file name")
+                  .with_reason(split_err.te_msg)
+                  .with_snippet(lnav::console::snippet::from(
+                      SRC, lexer.to_attr_line(split_err)));
 
-    if (!lexer.split(split_args, scopes)) {
-        return ec.make_error("unable to parse arguments");
+        return Err(um);
     }
 
+    auto split_args = split_args_res.unwrap()
+        | lnav::itertools::map([](const auto& elem) { return elem.se_value; });
+
     if (split_args.size() != 1) {
         return ec.make_error("expecting a single argument");
     }
@@ -4325,7 +4392,24 @@ com_sh(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
     static size_t EXEC_COUNT = 0;
 
     if (!ec.ec_dry_run) {
-        auto carg = trim(cmdline.substr(args[0].size()));
+        nonstd::optional<std::string> name_flag;
+
+        shlex lexer(cmdline);
+        auto cmd_start = args[0].size();
+        auto split_res = lexer.split(ec.create_resolver());
+        if (split_res.isOk()) {
+            auto flags = split_res.unwrap();
+            if (flags.size() >= 2) {
+                static const char* NAME_FLAG = "--name=";
+
+                if (startswith(flags[1].se_value, NAME_FLAG)) {
+                    name_flag = flags[1].se_value.substr(strlen(NAME_FLAG));
+                    cmd_start = flags[1].se_origin.sf_end;
+                }
+            }
+        }
+
+        auto carg = trim(cmdline.substr(cmd_start));
 
         log_info("executing: %s", carg.c_str());
 
@@ -4386,10 +4470,36 @@ com_sh(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
             _exit(EXIT_FAILURE);
         }
 
-        auto display_name = ec.get_provenance<exec_context::file_open>()
-                                .value_or(exec_context::file_open{fmt::format(
-                                    FMT_STRING("[{}] {}"), EXEC_COUNT++, carg)})
-                                .fo_name;
+        std::string display_name;
+        auto open_prov = ec.get_provenance<exec_context::file_open>();
+        if (open_prov) {
+            if (name_flag) {
+                display_name = fmt::format(
+                    FMT_STRING("{}/{}"), open_prov->fo_name, name_flag.value());
+            } else {
+                display_name = open_prov->fo_name;
+            }
+        } else if (name_flag) {
+            display_name = name_flag.value();
+        } else {
+            display_name
+                = fmt::format(FMT_STRING("[{}] {}"), EXEC_COUNT++, carg);
+        }
+
+        auto name_base = display_name;
+        size_t name_counter = 0;
+
+        while (true) {
+            auto fn_iter
+                = lnav_data.ld_active_files.fc_file_names.find(display_name);
+            if (fn_iter == lnav_data.ld_active_files.fc_file_names.end()) {
+                break;
+            }
+            name_counter += 1;
+            display_name
+                = fmt::format(FMT_STRING("{} [{}]"), name_base, name_counter);
+        }
+
         auto create_piper_res
             = lnav::piper::create_looper(display_name,
                                          std::move(child_fds[0].read_end()),
@@ -4398,7 +4508,7 @@ com_sh(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
         if (create_piper_res.isErr()) {
             auto um
                 = lnav::console::user_message::error("unable to create piper")
-                      .with_reason(child_res.unwrapErr());
+                      .with_reason(create_piper_res.unwrapErr());
             ec.add_error_context(um);
             return Err(um);
         }
@@ -4591,26 +4701,12 @@ com_eval(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
         }
 
         auto src_guard = ec.enter_source(EVAL_SRC, 1, expanded_cmd);
-        std::string alt_msg;
-        switch (expanded_cmd[0]) {
-            case ':':
-                return execute_command(ec, expanded_cmd.substr(1));
-            case ';':
-                return execute_sql(ec, expanded_cmd.substr(1), alt_msg);
-            case '|':
-                return execute_file(ec, expanded_cmd.substr(1));
-            case '/': {
-                auto search_cmd = expanded_cmd.substr(1);
-                lnav_data.ld_view_stack.top() |
-                    [&search_cmd](auto tc) { tc->execute_search(search_cmd); };
-                break;
-            }
-            default:
-                return ec.make_error(
-                    "expecting argument to start with ':', ';', '/', "
-                    "or '|' to signify a command, SQL query, or script to "
-                    "execute");
+        auto content = string_fragment::from_str(expanded_cmd);
+        multiline_executor me(ec, ":eval");
+        for (auto line : content.split_lines()) {
+            TRY(me.push_back(line));
         }
+        retval = TRY(me.final());
     } else {
         return ec.make_error("expecting a command or query to evaluate");
     }
@@ -5193,25 +5289,40 @@ com_prompt(exec_context& ec,
 
     if (args.empty()) {
     } else if (!ec.ec_dry_run) {
-        args.clear();
+        static const intern_string_t SRC = intern_string::lookup("flags");
 
         auto lexer = shlex(cmdline);
-        lexer.split(args, ec.create_resolver());
+        auto split_args_res = lexer.split(ec.create_resolver());
+        if (split_args_res.isErr()) {
+            auto split_err = split_args_res.unwrapErr();
+            auto um = lnav::console::user_message::error(
+                          "unable to parse file name")
+                          .with_reason(split_err.te_msg)
+                          .with_snippet(lnav::console::snippet::from(
+                              SRC, lexer.to_attr_line(split_err)));
+
+            return Err(um);
+        }
+
+        auto split_args = split_args_res.unwrap()
+            | lnav::itertools::map(
+                              [](const auto& elem) { return elem.se_value; });
 
-        auto alt_flag = std::find(args.begin(), args.end(), "--alt");
+        auto alt_flag
+            = std::find(split_args.begin(), split_args.end(), "--alt");
         auto is_alt = false;
-        if (alt_flag != args.end()) {
-            args.erase(alt_flag);
+        if (alt_flag != split_args.end()) {
+            split_args.erase(alt_flag);
             is_alt = true;
         }
 
-        auto prompter = PROMPT_TYPES.find(args[1]);
+        auto prompter = PROMPT_TYPES.find(split_args[1]);
 
         if (prompter == PROMPT_TYPES.end()) {
-            return ec.make_error("Unknown prompt type: {}", args[1]);
+            return ec.make_error("Unknown prompt type: {}", split_args[1]);
         }
 
-        prompter->second(args);
+        prompter->second(split_args);
         lnav_data.ld_rl_view->set_alt_focus(is_alt);
     }
     return Ok(std::string());
@@ -6058,6 +6169,8 @@ readline_context::command_t STD_COMMANDS[] = {
         help_text(":sh")
             .with_summary("Execute the given command-line and display the "
                           "captured output")
+            .with_parameter(help_text(
+                "--name=<name>", "The name to give to the captured output"))
             .with_parameter(
                 help_text("cmdline", "The command-line to execute."))
             .with_tags({"scripting"}),

src/lnav_config.cc

@@ -1364,7 +1364,7 @@ static const struct json_path_container url_scheme_handlers = {
 };
 
 static const struct json_path_container url_handlers = {
-    yajlpp::pattern_property_handler(R"((?<url_scheme>\w+))")
+    yajlpp::pattern_property_handler(R"((?<url_scheme>[a-z][\w\-\+\.]+))")
         .with_description("Definition of a custom URL scheme")
         .with_obj_provider<lnav::url_handler::scheme, _lnav_config>(
             [](const yajlpp_provider_context& ypc, _lnav_config* root) {

src/log_format.cc

@@ -1301,7 +1301,7 @@ external_log_format::scan(logfile& lf,
                 }
             }
 
-            log_debug("%s:%d:date-time re-locked to %d",
+            log_debug("%s:%d: date-time re-locked to %d",
                       lf.get_unique_path().c_str(),
                       dst.size(),
                       this->lf_date_time.dts_fmt_lock);
@@ -1474,10 +1474,12 @@ external_log_format::scan(logfile& lf,
         if (orig_lock != curr_fmt) {
             uint32_t lock_line;
 
-            log_debug("%zu: changing pattern lock %d -> %d",
+            log_debug("%s:%zu: changing pattern lock %d -> (%d)%s",
+                      lf.get_unique_path().c_str(),
                       dst.size() - 1,
                       orig_lock,
-                      curr_fmt);
+                      curr_fmt,
+                      this->elf_pattern_order[curr_fmt]->p_name.c_str());
             if (this->lf_pattern_locks.empty()) {
                 lock_line = 0;
             } else {
@@ -2648,6 +2650,8 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
                     .with_snippets(this->get_snippets()));
         }
         if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
+            this->lf_multiline = true;
+            this->lf_structured = true;
             this->jlf_parse_context
                 = std::make_shared<yajlpp_parse_context>(this->elf_name);
             this->jlf_yajl_handle.reset(
@@ -2658,7 +2662,6 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
             yajl_config(
                 this->jlf_yajl_handle.get(), yajl_dont_validate_strings, 1);
         }
-
     } else {
         if (this->elf_patterns.empty()) {
             errors.emplace_back(lnav::console::user_message::error(

src/log_format.hh

@@ -542,6 +542,7 @@ public:
     std::string lf_description;
     uint8_t lf_mod_index{0};
     bool lf_multiline{true};
+    bool lf_structured{false};
     date_time_scanner lf_date_time;
     date_time_scanner lf_time_scanner;
     std::vector<pattern_for_lines> lf_pattern_locks;

src/logfile.cc

@@ -389,10 +389,16 @@ logfile::process_prefix(shared_buffer_ref& sbr,
 
             for (size_t lpc = 0; lpc < this->lf_index.size() - 1; lpc++) {
                 if (this->lf_format->lf_multiline) {
+                    if (this->lf_format->lf_structured) {
+                        this->lf_index[lpc].set_ignore(true);
+                    } else {
+                        this->lf_index[lpc].set_time(last_line.get_time());
+                        this->lf_index[lpc].set_millis(last_line.get_millis());
+                    }
+                } else {
                     this->lf_index[lpc].set_time(last_line.get_time());
                     this->lf_index[lpc].set_millis(last_line.get_millis());
-                } else {
-                    this->lf_index[lpc].set_ignore(true);
+                    this->lf_index[lpc].set_level(LEVEL_INVALID);
                 }
             }
 

src/logfile_sub_source.cc

@@ -728,6 +728,7 @@ logfile_sub_source::rebuild_index(
     if (force) {
         log_debug("forced to full rebuild");
         retval = rebuild_result::rr_full_rebuild;
+        full_sort = true;
     }
 
     std::vector<size_t> file_order(this->lss_files.size());
@@ -765,6 +766,7 @@ logfile_sub_source::rebuild_index(
                           ld.ld_file_index);
                 force = true;
                 retval = rebuild_result::rr_full_rebuild;
+                full_sort = true;
             }
         } else {
             if (time_left && deadline && ui_clock::now() > deadline.value()) {
@@ -850,7 +852,7 @@ logfile_sub_source::rebuild_index(
 
     if (this->lss_index.reserve(total_lines)) {
         // The index array was reallocated, just do a full sort/rebuild since
-        // its been cleared out.
+        // it's been cleared out.
         log_debug("expanding index capacity %zu", this->lss_index.ba_capacity);
         force = true;
         retval = rebuild_result::rr_full_rebuild;

src/ptimec.hh

@@ -282,10 +282,13 @@ ptime_S(struct exttm* dst, const char* str, off_t& off_inout, ssize_t len)
         }
         dst->et_tm.tm_sec
             = (str[off_inout] - '0') * 10 + (str[off_inout + 1] - '0');
+        if (dst->et_tm.tm_sec < 0 || dst->et_tm.tm_sec >= 60) {
+            return false;
+        }
         dst->et_flags |= ETF_SECOND_SET;
     });
 
-    return (dst->et_tm.tm_sec >= 0 && dst->et_tm.tm_sec <= 59);
+    return true;
 }
 
 inline void

src/readline_callbacks.cc

@@ -773,13 +773,14 @@ rl_callback_int(readline_curses* rc, bool is_alt)
                     }
                 }
 
+                tm current_tm;
                 struct stat st;
 
                 if (fstat(fd_copy, &st) != -1 && st.st_size > 0) {
                     strftime(timestamp,
                              sizeof(timestamp),
                              "%a %b %d %H:%M:%S %Z",
-                             localtime(&current_time));
+                             localtime_r(&current_time, &current_tm));
                     snprintf(desc,
                              sizeof(desc),
                              "Output of %s (%s)",

src/readline_curses.cc

@@ -56,6 +56,7 @@
 
 #include "base/ansi_scrubber.hh"
 #include "base/auto_mem.hh"
+#include "base/itertools.hh"
 #include "base/lnav_log.hh"
 #include "base/paths.hh"
 #include "base/string_util.hh"
@@ -374,7 +375,6 @@ readline_context::attempted_completion(const char* text, int start, int end)
     } else {
         char* space;
         std::string cmd;
-        std::vector<std::string> prefix;
         int point = rl_point;
         while (point > 0 && rl_line_buffer[point] != ' ') {
             point -= 1;
@@ -384,7 +384,11 @@ readline_context::attempted_completion(const char* text, int start, int end)
 
         arg_possibilities = nullptr;
         rl_completion_append_character = 0;
-        if (lexer.split(prefix, scoped_resolver{&scope})) {
+        auto split_res = lexer.split(scoped_resolver{&scope});
+        if (split_res.isOk()) {
+            auto prefix = split_res.unwrap()
+                | lnav::itertools::map(
+                              [](const auto& elem) { return elem.se_value; });
             auto prefix2
                 = fmt::format(FMT_STRING("{}"), fmt::join(prefix, "\x1f"));
             auto prefix_iter = loaded_context->rc_prefixes.find(prefix2);
@@ -422,88 +426,101 @@ readline_context::attempted_completion(const char* text, int start, int end)
                     arg_possibilities = nullptr;
                 } else if (proto[0] == "dirname") {
                     shlex fn_lexer(rl_line_buffer, rl_point);
-                    std::vector<std::string> fn_list;
+                    auto split_res = fn_lexer.split(scoped_resolver{&scope});
+                    if (split_res.isOk()) {
+                        auto fn_list = split_res.unwrap();
+                        const auto& last_fn = fn_list.size() <= 1
+                            ? ""
+                            : fn_list.back().se_value;
 
-                    fn_lexer.split(fn_list, scoped_resolver{&scope});
+                        static std::set<std::string> dir_name_set;
 
-                    const auto& last_fn = fn_list.size() <= 1 ? ""
-                                                              : fn_list.back();
-
-                    static std::set<std::string> dir_name_set;
-
-                    dir_name_set.clear();
-                    auto_mem<char> completed_fn;
-                    int fn_state = 0;
+                        dir_name_set.clear();
+                        auto_mem<char> completed_fn;
+                        int fn_state = 0;
 
-                    while ((completed_fn = rl_filename_completion_function(
-                                last_fn.c_str(), fn_state))
-                           != nullptr)
-                    {
-                        dir_name_set.insert(completed_fn.in());
-                        fn_state += 1;
+                        while ((completed_fn = rl_filename_completion_function(
+                                    last_fn.c_str(), fn_state))
+                               != nullptr)
+                        {
+                            dir_name_set.insert(completed_fn.in());
+                            fn_state += 1;
+                        }
+                        arg_possibilities = &dir_name_set;
+                        arg_needs_shlex = true;
+                    } else {
+                        arg_possibilities = nullptr;
                     }
-                    arg_possibilities = &dir_name_set;
-                    arg_needs_shlex = true;
                 } else if (proto[0] == "filename") {
                     shlex fn_lexer(rl_line_buffer, rl_point);
-                    std::vector<std::string> fn_list;
                     int found = 0;
 
-                    fn_lexer.split(fn_list, scoped_resolver{&scope});
-
-                    const auto& last_fn = fn_list.size() <= 1 ? ""
-                                                              : fn_list.back();
-
-                    if (last_fn.find(':') != std::string::npos) {
-                        auto rp_iter = loaded_context->rc_possibilities.find(
-                            "remote-path");
-                        if (rp_iter != loaded_context->rc_possibilities.end()) {
-                            for (const auto& poss : rp_iter->second) {
-                                if (startswith(poss, last_fn.c_str())) {
-                                    found += 1;
+                    auto split_res = fn_lexer.split(scoped_resolver{&scope});
+                    if (split_res.isOk()) {
+                        auto fn_list = split_res.unwrap();
+                        const auto& last_fn = fn_list.size() <= 1
+                            ? ""
+                            : fn_list.back().se_value;
+
+                        if (last_fn.find(':') != std::string::npos) {
+                            auto rp_iter
+                                = loaded_context->rc_possibilities.find(
+                                    "remote-path");
+                            if (rp_iter
+                                != loaded_context->rc_possibilities.end())
+                            {
+                                for (const auto& poss : rp_iter->second) {
+                                    if (startswith(poss, last_fn.c_str())) {
+                                        found += 1;
+                                    }
+                                }
+                                if (found) {
+                                    arg_possibilities = &rp_iter->second;
+                                    arg_needs_shlex = false;
                                 }
                             }
-                            if (found) {
-                                arg_possibilities = &rp_iter->second;
-                                arg_needs_shlex = false;
+                            if (!found
+                                || (endswith(last_fn, "/") && found == 1))
+                            {
+                                char msg[2048];
+
+                                snprintf(
+                                    msg, sizeof(msg), "\t:%s", last_fn.c_str());
+                                sendstring(child_this->rc_command_pipe[1],
+                                           msg,
+                                           strlen(msg));
                             }
                         }
-                        if (!found || (endswith(last_fn, "/") && found == 1)) {
-                            char msg[2048];
-
-                            snprintf(
-                                msg, sizeof(msg), "\t:%s", last_fn.c_str());
-                            sendstring(child_this->rc_command_pipe[1],
-                                       msg,
-                                       strlen(msg));
-                        }
-                    }
-                    if (!found) {
-                        static std::set<std::string> file_name_set;
-
-                        file_name_set.clear();
-                        auto_mem<char> completed_fn;
-                        int fn_state = 0;
-                        auto recent_netlocs_iter
-                            = loaded_context->rc_possibilities.find(
-                                "recent-netlocs");
-
-                        if (recent_netlocs_iter
-                            != loaded_context->rc_possibilities.end())
-                        {
-                            file_name_set.insert(
-                                recent_netlocs_iter->second.begin(),
-                                recent_netlocs_iter->second.end());
-                        }
-                        while ((completed_fn = rl_filename_completion_function(
-                                    last_fn.c_str(), fn_state))
-                               != nullptr)
-                        {
-                            file_name_set.insert(completed_fn.in());
-                            fn_state += 1;
+                        if (!found) {
+                            static std::set<std::string> file_name_set;
+
+                            file_name_set.clear();
+                            auto_mem<char> completed_fn;
+                            int fn_state = 0;
+                            auto recent_netlocs_iter
+                                = loaded_context->rc_possibilities.find(
+                                    "recent-netlocs");
+
+                            if (recent_netlocs_iter
+                                != loaded_context->rc_possibilities.end())
+                            {
+                                file_name_set.insert(
+                                    recent_netlocs_iter->second.begin(),
+                                    recent_netlocs_iter->second.end());
+                            }
+                            while (
+                                (completed_fn = rl_filename_completion_function(
+                                     last_fn.c_str(), fn_state))
+                                != nullptr)
+                            {
+                                file_name_set.insert(completed_fn.in());
+                                fn_state += 1;
+                            }
+                            arg_possibilities = &file_name_set;
+                            arg_needs_shlex = true;
                         }
-                        arg_possibilities = &file_name_set;
-                        arg_needs_shlex = true;
+                    } else {
+                        arg_possibilities = nullptr;
                     }
                 } else {
                     arg_possibilities

src/readline_highlighters.cc

@@ -306,63 +306,77 @@ readline_shlex_highlighter_int(attr_line_t& al, int x, line_range sub)
 {
     attr_line_builder alb(al);
     const auto& str = al.get_string();
-    string_fragment cap;
-    shlex_token_t token;
     nonstd::optional<int> quote_start;
     shlex lexer(string_fragment{al.al_string.data(), sub.lr_start, sub.lr_end});
+    bool done = false;
+
+    while (!done) {
+        auto tokenize_res = lexer.tokenize();
+        if (tokenize_res.isErr()) {
+            auto te = tokenize_res.unwrapErr();
+
+            alb.overlay_attr(line_range(sub.lr_start + te.te_source.sf_begin,
+                                        sub.lr_start + te.te_source.sf_end),
+                             VC_STYLE.value(text_attrs{A_REVERSE}));
+            alb.overlay_attr(line_range(sub.lr_start + te.te_source.sf_begin,
+                                        sub.lr_start + te.te_source.sf_end),
+                             VC_ROLE.value(role_t::VCR_ERROR));
+            return;
+        }
 
-    while (lexer.tokenize(cap, token)) {
-        switch (token) {
-            case shlex_token_t::ST_ERROR:
-                alb.overlay_attr(line_range(sub.lr_start + cap.sf_begin,
-                                            sub.lr_start + cap.sf_end),
-                                 VC_STYLE.value(text_attrs{A_REVERSE}));
-                alb.overlay_attr(line_range(sub.lr_start + cap.sf_begin,
-                                            sub.lr_start + cap.sf_end),
-                                 VC_ROLE.value(role_t::VCR_ERROR));
+        auto token = tokenize_res.unwrap();
+        switch (token.tr_token) {
+            case shlex_token_t::eof:
+                done = true;
                 break;
-            case shlex_token_t::ST_TILDE:
-            case shlex_token_t::ST_ESCAPE:
-                alb.overlay_attr(line_range(sub.lr_start + cap.sf_begin,
-                                            sub.lr_start + cap.sf_end),
-                                 VC_ROLE.value(role_t::VCR_SYMBOL));
+            case shlex_token_t::tilde:
+            case shlex_token_t::escape:
+                alb.overlay_attr(
+                    line_range(sub.lr_start + token.tr_frag.sf_begin,
+                               sub.lr_start + token.tr_frag.sf_end),
+                    VC_ROLE.value(role_t::VCR_SYMBOL));
                 break;
-            case shlex_token_t::ST_DOUBLE_QUOTE_START:
-            case shlex_token_t::ST_SINGLE_QUOTE_START:
-                quote_start = sub.lr_start + cap.sf_begin;
+            case shlex_token_t::double_quote_start:
+            case shlex_token_t::single_quote_start:
+                quote_start = sub.lr_start + token.tr_frag.sf_begin;
                 break;
-            case shlex_token_t::ST_DOUBLE_QUOTE_END:
-            case shlex_token_t::ST_SINGLE_QUOTE_END:
+            case shlex_token_t::double_quote_end:
+            case shlex_token_t::single_quote_end:
                 alb.overlay_attr(
-                    line_range(quote_start.value(), sub.lr_start + cap.sf_end),
+                    line_range(quote_start.value(),
+                               sub.lr_start + token.tr_frag.sf_end),
                     VC_ROLE.value(role_t::VCR_STRING));
                 quote_start = nonstd::nullopt;
                 break;
-            case shlex_token_t::ST_VARIABLE_REF:
-            case shlex_token_t::ST_QUOTED_VARIABLE_REF: {
-                int extra = token == shlex_token_t::ST_VARIABLE_REF ? 0 : 1;
-                auto ident = str.substr(sub.lr_start + cap.sf_begin + 1 + extra,
-                                        cap.length() - 1 - extra * 2);
+            case shlex_token_t::variable_ref:
+            case shlex_token_t::quoted_variable_ref: {
+                int extra = token.tr_token == shlex_token_t::variable_ref ? 0
+                                                                          : 1;
+                auto ident = str.substr(
+                    sub.lr_start + token.tr_frag.sf_begin + 1 + extra,
+                    token.tr_frag.length() - 1 - extra * 2);
                 alb.overlay_attr(
-                    line_range(sub.lr_start + cap.sf_begin,
-                               sub.lr_start + cap.sf_begin + 1 + extra),
+                    line_range(
+                        sub.lr_start + token.tr_frag.sf_begin,
+                        sub.lr_start + token.tr_frag.sf_begin + 1 + extra),
                     VC_ROLE.value(role_t::VCR_SYMBOL));
                 alb.overlay_attr(
-                    line_range(sub.lr_start + cap.sf_begin + 1 + extra,
-                               sub.lr_start + cap.sf_end - extra),
-                    VC_ROLE.value(
-                        x == sub.lr_start + cap.sf_end
-                                || (cap.sf_begin <= x && x < cap.sf_end)
-                            ? role_t::VCR_SYMBOL
-                            : role_t::VCR_IDENTIFIER));
+                    line_range(
+                        sub.lr_start + token.tr_frag.sf_begin + 1 + extra,
+                        sub.lr_start + token.tr_frag.sf_end - extra),
+                    VC_ROLE.value(x == sub.lr_start + token.tr_frag.sf_end
+                                          || (token.tr_frag.sf_begin <= x
+                                              && x < token.tr_frag.sf_end)
+                                      ? role_t::VCR_SYMBOL
+                                      : role_t::VCR_IDENTIFIER));
                 if (extra) {
                     alb.overlay_attr_for_char(
-                        sub.lr_start + cap.sf_end - 1,
+                        sub.lr_start + token.tr_frag.sf_end - 1,
                         VC_ROLE.value(role_t::VCR_SYMBOL));
                 }
                 break;
             }
-            case shlex_token_t::ST_WHITESPACE:
+            case shlex_token_t::whitespace:
                 break;
         }
     }

src/root-config.json

@@ -96,6 +96,9 @@
             "docker": {
                 "handler": "docker-url-handler"
             },
+            "docker-compose": {
+                "handler": "docker-compose-url-handler"
+            },
             "piper": {
                 "handler": "piper-url-handler"
             }

src/scripts/docker-compose-url-handler.lnav

@@ -1,15 +0,0 @@
-#
-# @synopsis: docker-compose-url-handler
-# @description: Internal script to handle opening docker-compose URLs
-#
-
-;SELECT st_name FROM fstat('compose.yml')
- UNION
- SELECT st_name FROM fstat('compose.yaml')
- UNION
- SELECT st_name FROM fstat('docker-compose.yml')
- UNION
- SELECT st_name FROM fstat('docker-compose.yaml')
-
-;SELECT group_concat(compose_services.key, ' ')
-   FROM fstat($st_name), json_each(yaml_to_json(data), '$.services') as compose_services

src/scripts/docker-url-handler.lnav

@@ -3,15 +3,29 @@
 # @description: Internal script to handle opening docker URLs
 #
 
-;SELECT CASE path
-    WHEN '/' THEN
-        'docker logs -f ' || hostname
-    ELSE
-        'docker exec ' || hostname || ' tail -n +0 -F "' || path || '"'
-    END AS cmd
-    FROM (SELECT
-            jget(url, '/host') AS hostname,
-            jget(url, '/path') AS path
-            FROM (SELECT parse_url($1) AS url))
+;SELECT jget(url, '/host') AS docker_hostname,
+        jget(url, '/path') AS docker_path
+   FROM (SELECT parse_url($1) AS url)
 
-:sh eval $cmd
+;SELECT CASE
+        $docker_hostname
+        WHEN 'compose' THEN (
+            SELECT group_concat(
+                    printf(
+                        ':sh --name=%s docker compose logs --no-log-prefix -f %s',
+                        compose_services.key,
+                        compose_services.key
+                    ),
+                    char(10)
+                ) AS cmds
+            FROM fstat(substr($docker_path, 2)),
+                 json_each(yaml_to_json(data), '$.services') as compose_services
+        )
+        ELSE CASE
+            $docker_path
+            WHEN '/' THEN ':sh docker logs -f ' || $docker_hostname
+            ELSE ':sh docker exec ' || $docker_hostname || ' tail -n +0 -F "' || $docker_path || '"'
+        END
+    END AS cmds
+
+:eval ${cmds}

src/scripts/pcap_log-converter.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Check that tshark is installed and return a nice message.
-if ! command -v tshark; then
+if ! command -v tshark > /dev/null; then
   echo "pcap support requires 'tshark' v3+ to be installed" > /dev/stderr
   exit 1
 fi

src/scripts/scripts.am

@@ -2,7 +2,6 @@
 BUILTIN_LNAVSCRIPTS = \
     $(srcdir)/scripts/dhclient-summary.lnav \
     $(srcdir)/scripts/docker-url-handler.lnav \
-    $(srcdir)/scripts/docker-compose-url-handler.lnav \
     $(srcdir)/scripts/lnav-pop-view.lnav \
     $(srcdir)/scripts/partition-by-boot.lnav \
     $(srcdir)/scripts/piper-url-handler.lnav \

src/shlex.cc

@@ -36,40 +36,59 @@
 #include "config.h"
 #include "shlex.hh"
 
-bool
-shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out)
+using namespace lnav::roles::literals;
+
+attr_line_t
+shlex::to_attr_line(const shlex::tokenize_error_t& te) const
 {
+    return attr_line_t()
+        .append(string_fragment::from_bytes(this->s_str, this->s_len))
+        .append("\n")
+        .pad_to(te.te_source.sf_begin)
+        .append("^"_snippet_border);
+}
+
+Result<shlex::tokenize_result_t, shlex::tokenize_error_t>
+shlex::tokenize()
+{
+    tokenize_result_t retval;
+
+    retval.tr_frag.sf_string = this->s_str;
     while (this->s_index < this->s_len) {
         switch (this->s_str[this->s_index]) {
             case '\\':
-                cap_out.sf_begin = this->s_index;
+                retval.tr_frag.sf_begin = this->s_index;
                 if (this->s_index + 1 < this->s_len) {
-                    token_out = shlex_token_t::ST_ESCAPE;
+                    retval.tr_token = shlex_token_t::escape;
                     this->s_index += 2;
-                    cap_out.sf_end = this->s_index;
+                    retval.tr_frag.sf_end = this->s_index;
                 } else {
                     this->s_index += 1;
-                    cap_out.sf_end = this->s_index;
-                    token_out = shlex_token_t::ST_ERROR;
+                    retval.tr_frag.sf_end = this->s_index;
+
+                    return Err(tokenize_error_t{
+                        "invalid escape",
+                        retval.tr_frag,
+                    });
                 }
-                return true;
+                return Ok(retval);
             case '\"':
                 if (!this->s_ignore_quotes) {
                     switch (this->s_state) {
                         case state_t::STATE_NORMAL:
-                            cap_out.sf_begin = this->s_index;
+                            retval.tr_frag.sf_begin = this->s_index;
                             this->s_index += 1;
-                            cap_out.sf_end = this->s_index;
-                            token_out = shlex_token_t::ST_DOUBLE_QUOTE_START;
+                            retval.tr_frag.sf_end = this->s_index;
+                            retval.tr_token = shlex_token_t::double_quote_start;
                             this->s_state = state_t::STATE_IN_DOUBLE_QUOTE;
-                            return true;
+                            return Ok(retval);
                         case state_t::STATE_IN_DOUBLE_QUOTE:
-                            cap_out.sf_begin = this->s_index;
+                            retval.tr_frag.sf_begin = this->s_index;
                             this->s_index += 1;
-                            cap_out.sf_end = this->s_index;
-                            token_out = shlex_token_t::ST_DOUBLE_QUOTE_END;
+                            retval.tr_frag.sf_end = this->s_index;
+                            retval.tr_token = shlex_token_t::double_quote_end;
                             this->s_state = state_t::STATE_NORMAL;
-                            return true;
+                            return Ok(retval);
                         default:
                             break;
                     }
@@ -79,19 +98,19 @@ shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out)
                 if (!this->s_ignore_quotes) {
                     switch (this->s_state) {
                         case state_t::STATE_NORMAL:
-                            cap_out.sf_begin = this->s_index;
+                            retval.tr_frag.sf_begin = this->s_index;
                             this->s_index += 1;
-                            cap_out.sf_end = this->s_index;
-                            token_out = shlex_token_t::ST_SINGLE_QUOTE_START;
+                            retval.tr_frag.sf_end = this->s_index;
+                            retval.tr_token = shlex_token_t::single_quote_start;
                             this->s_state = state_t::STATE_IN_SINGLE_QUOTE;
-                            return true;
+                            return Ok(retval);
                         case state_t::STATE_IN_SINGLE_QUOTE:
-                            cap_out.sf_begin = this->s_index;
+                            retval.tr_frag.sf_begin = this->s_index;
                             this->s_index += 1;
-                            cap_out.sf_end = this->s_index;
-                            token_out = shlex_token_t::ST_SINGLE_QUOTE_END;
+                            retval.tr_frag.sf_end = this->s_index;
+                            retval.tr_token = shlex_token_t::single_quote_end;
                             this->s_state = state_t::STATE_NORMAL;
-                            return true;
+                            return Ok(retval);
                         default:
                             break;
                     }
@@ -100,9 +119,10 @@ shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out)
             case '$':
                 switch (this->s_state) {
                     case state_t::STATE_NORMAL:
-                    case state_t::STATE_IN_DOUBLE_QUOTE:
-                        this->scan_variable_ref(cap_out, token_out);
-                        return true;
+                    case state_t::STATE_IN_DOUBLE_QUOTE: {
+                        auto rc = TRY(this->scan_variable_ref());
+                        return Ok(rc);
+                    }
                     default:
                         break;
                 }
@@ -110,7 +130,7 @@ shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out)
             case '~':
                 switch (this->s_state) {
                     case state_t::STATE_NORMAL:
-                        cap_out.sf_begin = this->s_index;
+                        retval.tr_frag.sf_begin = this->s_index;
                         this->s_index += 1;
                         while (this->s_index < this->s_len
                                && (isalnum(this->s_str[this->s_index])
@@ -119,9 +139,9 @@ shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out)
                         {
                             this->s_index += 1;
                         }
-                        cap_out.sf_end = this->s_index;
-                        token_out = shlex_token_t::ST_TILDE;
-                        return true;
+                        retval.tr_frag.sf_end = this->s_index;
+                        retval.tr_token = shlex_token_t::tilde;
+                        return Ok(retval);
                     default:
                         break;
                 }
@@ -130,13 +150,13 @@ shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out)
             case '\t':
                 switch (this->s_state) {
                     case state_t::STATE_NORMAL:
-                        cap_out.sf_begin = this->s_index;
+                        retval.tr_frag.sf_begin = this->s_index;
                         while (isspace(this->s_str[this->s_index])) {
                             this->s_index += 1;
                         }
-                        cap_out.sf_end = this->s_index;
-                        token_out = shlex_token_t::ST_WHITESPACE;
-                        return true;
+                        retval.tr_frag.sf_end = this->s_index;
+                        retval.tr_token = shlex_token_t::whitespace;
+                        return Ok(retval);
                     default:
                         break;
                 }
@@ -148,29 +168,47 @@ shlex::tokenize(string_fragment& cap_out, shlex_token_t& token_out)
         this->s_index += 1;
     }
 
-    return false;
+    if (this->s_state != state_t::STATE_NORMAL) {
+        retval.tr_frag.sf_begin = this->s_index;
+        retval.tr_frag.sf_end = this->s_len;
+        return Err(tokenize_error_t{
+            "non-terminated string",
+            retval.tr_frag,
+        });
+    }
+
+    retval.tr_frag.sf_begin = this->s_len;
+    retval.tr_frag.sf_end = this->s_len;
+    retval.tr_token = shlex_token_t::eof;
+    return Ok(retval);
 }
 
-void
-shlex::scan_variable_ref(string_fragment& cap_out, shlex_token_t& token_out)
+Result<shlex::tokenize_result_t, shlex::tokenize_error_t>
+shlex::scan_variable_ref()
 {
-    cap_out.sf_begin = this->s_index;
+    tokenize_result_t retval;
+
+    retval.tr_frag.sf_string = this->s_str;
+
+    retval.tr_frag.sf_begin = this->s_index;
     this->s_index += 1;
     if (this->s_index >= this->s_len) {
-        cap_out.sf_end = this->s_index;
-        token_out = shlex_token_t::ST_ERROR;
-        return;
+        retval.tr_frag.sf_end = this->s_index;
+        return Err(tokenize_error_t{
+            "invalid variable reference",
+            retval.tr_frag,
+        });
     }
 
     if (this->s_str[this->s_index] == '{') {
-        token_out = shlex_token_t::ST_QUOTED_VARIABLE_REF;
+        retval.tr_token = shlex_token_t::quoted_variable_ref;
         this->s_index += 1;
     } else {
-        token_out = shlex_token_t::ST_VARIABLE_REF;
+        retval.tr_token = shlex_token_t::variable_ref;
     }
 
     while (this->s_index < this->s_len) {
-        if (token_out == shlex_token_t::ST_VARIABLE_REF) {
+        if (retval.tr_token == shlex_token_t::variable_ref) {
             if (isalnum(this->s_str[this->s_index])
                 || this->s_str[this->s_index] == '#'
                 || this->s_str[this->s_index] == '_')
@@ -188,14 +226,19 @@ shlex::scan_variable_ref(string_fragment& cap_out, shlex_token_t& token_out)
         }
     }
 
-    cap_out.sf_end = this->s_index;
-    if (token_out == shlex_token_t::ST_QUOTED_VARIABLE_REF
+    retval.tr_frag.sf_end = this->s_index;
+    if (retval.tr_token == shlex_token_t::quoted_variable_ref
         && this->s_str[this->s_index - 1] != '}')
     {
-        cap_out.sf_begin += 1;
-        cap_out.sf_end = cap_out.sf_begin + 1;
-        token_out = shlex_token_t::ST_ERROR;
+        retval.tr_frag.sf_begin += 1;
+        retval.tr_frag.sf_end = retval.tr_frag.sf_begin + 1;
+        return Err(tokenize_error_t{
+            "missing closing curly-brace in variable reference",
+            retval.tr_frag,
+        });
     }
+
+    return Ok(retval);
 }
 
 void
@@ -222,27 +265,36 @@ shlex::eval(std::string& result, const scoped_resolver& vars)
 {
     result.clear();
 
-    string_fragment cap;
-    shlex_token_t token;
     int last_index = 0;
+    bool done = false;
+
+    while (!done) {
+        auto tokenize_res = this->tokenize();
+        if (tokenize_res.isErr()) {
+            return false;
+        }
+        auto token = tokenize_res.unwrap();
 
-    while (this->tokenize(cap, token)) {
-        result.append(&this->s_str[last_index], cap.sf_begin - last_index);
-        switch (token) {
-            case shlex_token_t::ST_ERROR:
-                return false;
-            case shlex_token_t::ST_ESCAPE:
-                result.append(1, this->s_str[cap.sf_begin + 1]);
+        result.append(&this->s_str[last_index],
+                      token.tr_frag.sf_begin - last_index);
+        switch (token.tr_token) {
+            case shlex_token_t::eof:
+                done = true;
                 break;
-            case shlex_token_t::ST_WHITESPACE:
-                result.append(&this->s_str[cap.sf_begin], cap.length());
+            case shlex_token_t::escape:
+                result.append(1, this->s_str[token.tr_frag.sf_begin + 1]);
                 break;
-            case shlex_token_t::ST_VARIABLE_REF:
-            case shlex_token_t::ST_QUOTED_VARIABLE_REF: {
-                int extra = token == shlex_token_t::ST_VARIABLE_REF ? 0 : 1;
+            case shlex_token_t::whitespace:
+                result.append(&this->s_str[token.tr_frag.sf_begin],
+                              token.tr_frag.length());
+                break;
+            case shlex_token_t::variable_ref:
+            case shlex_token_t::quoted_variable_ref: {
+                int extra = token.tr_token == shlex_token_t::variable_ref ? 0
+                                                                          : 1;
                 const std::string var_name(
-                    &this->s_str[cap.sf_begin + 1 + extra],
-                    cap.length() - 1 - extra * 2);
+                    &this->s_str[token.tr_frag.sf_begin + 1 + extra],
+                    token.tr_frag.length() - 1 - extra * 2);
                 auto local_var = vars.find(var_name);
                 const char* var_value = getenv(var_name.c_str());
 
@@ -253,21 +305,21 @@ shlex::eval(std::string& result, const scoped_resolver& vars)
                 }
                 break;
             }
-            case shlex_token_t::ST_TILDE:
-                this->resolve_home_dir(result, cap);
+            case shlex_token_t::tilde:
+                this->resolve_home_dir(result, token.tr_frag);
                 break;
-            case shlex_token_t::ST_DOUBLE_QUOTE_START:
-            case shlex_token_t::ST_DOUBLE_QUOTE_END:
+            case shlex_token_t::double_quote_start:
+            case shlex_token_t::double_quote_end:
                 result.append("\"");
                 break;
-            case shlex_token_t::ST_SINGLE_QUOTE_START:
-            case shlex_token_t::ST_SINGLE_QUOTE_END:
+            case shlex_token_t::single_quote_start:
+            case shlex_token_t::single_quote_end:
                 result.append("'");
                 break;
             default:
                 break;
         }
-        last_index = cap.sf_end;
+        last_index = token.tr_frag.sf_end;
     }
 
     result.append(&this->s_str[last_index], this->s_len - last_index);
@@ -275,66 +327,89 @@ shlex::eval(std::string& result, const scoped_resolver& vars)
     return true;
 }
 
-bool
-shlex::split(std::vector<std::string>& result, const scoped_resolver& vars)
+Result<std::vector<shlex::split_element_t>, shlex::tokenize_error_t>
+shlex::split(const scoped_resolver& vars)
 {
-    result.clear();
-
-    string_fragment cap;
-    shlex_token_t token;
+    std::vector<split_element_t> retval;
     int last_index = 0;
     bool start_new = true;
+    bool done = false;
 
-    while (isspace(this->s_str[this->s_index])) {
+    while (this->s_index < this->s_len && isspace(this->s_str[this->s_index])) {
         this->s_index += 1;
     }
-    while (this->tokenize(cap, token)) {
+    if (this->s_index == this->s_len) {
+        return Ok(retval);
+    }
+    while (!done) {
+        auto tokenize_res = TRY(this->tokenize());
+
         if (start_new) {
-            result.emplace_back("");
+            retval.emplace_back(split_element_t{
+                string_fragment::from_byte_range(
+                    this->s_str, last_index, tokenize_res.tr_frag.sf_begin),
+                "",
+            });
             start_new = false;
+        } else if (tokenize_res.tr_token != shlex_token_t::whitespace) {
+            retval.back().se_origin.sf_end = tokenize_res.tr_frag.sf_end;
+        } else {
+            retval.back().se_origin.sf_end = tokenize_res.tr_frag.sf_begin;
         }
-        result.back().append(&this->s_str[last_index],
-                             cap.sf_begin - last_index);
-        switch (token) {
-            case shlex_token_t::ST_ERROR:
-                return false;
-            case shlex_token_t::ST_ESCAPE:
-                result.back().append(1, this->s_str[cap.sf_begin + 1]);
+        retval.back().se_value.append(
+            &this->s_str[last_index],
+            tokenize_res.tr_frag.sf_begin - last_index);
+        switch (tokenize_res.tr_token) {
+            case shlex_token_t::eof:
+                done = true;
+                break;
+            case shlex_token_t::escape:
+                retval.back().se_value.append(
+                    1, this->s_str[tokenize_res.tr_frag.sf_begin + 1]);
                 break;
-            case shlex_token_t::ST_WHITESPACE:
+            case shlex_token_t::whitespace:
                 start_new = true;
                 break;
-            case shlex_token_t::ST_VARIABLE_REF:
-            case shlex_token_t::ST_QUOTED_VARIABLE_REF: {
-                int extra = token == shlex_token_t::ST_VARIABLE_REF ? 0 : 1;
-                std::string var_name(&this->s_str[cap.sf_begin + 1 + extra],
-                                     cap.length() - 1 - extra * 2);
+            case shlex_token_t::variable_ref:
+            case shlex_token_t::quoted_variable_ref: {
+                int extra = tokenize_res.tr_token == shlex_token_t::variable_ref
+                    ? 0
+                    : 1;
+                std::string var_name(
+                    &this->s_str[tokenize_res.tr_frag.sf_begin + 1 + extra],
+                    tokenize_res.tr_frag.length() - 1 - extra * 2);
                 auto local_var = vars.find(var_name);
                 const char* var_value = getenv(var_name.c_str());
 
                 if (local_var != vars.end()) {
-                    result.back().append(fmt::to_string(local_var->second));
+                    retval.back().se_value.append(
+                        fmt::to_string(local_var->second));
                 } else if (var_value != nullptr) {
-                    result.back().append(var_value);
+                    retval.back().se_value.append(var_value);
                 }
                 break;
             }
-            case shlex_token_t::ST_TILDE:
-                this->resolve_home_dir(result.back(), cap);
+            case shlex_token_t::tilde:
+                this->resolve_home_dir(retval.back().se_value,
+                                       tokenize_res.tr_frag);
                 break;
             default:
                 break;
         }
-        last_index = cap.sf_end;
+        last_index = tokenize_res.tr_frag.sf_end;
     }
 
     if (last_index < this->s_len) {
-        if (start_new || result.empty()) {
-            result.emplace_back("");
+        if (start_new || retval.empty()) {
+            retval.emplace_back(split_element_t{
+                string_fragment::from_byte_range(
+                    this->s_str, last_index, this->s_len),
+                "",
+            });
         }
-        result.back().append(&this->s_str[last_index],
-                             this->s_len - last_index);
+        retval.back().se_value.append(&this->s_str[last_index],
+                                      this->s_len - last_index);
     }
 
-    return true;
+    return Ok(retval);
 }

src/shlex.hh

@@ -32,32 +32,34 @@
 #ifndef LNAV_SHLEX_HH_H
 #define LNAV_SHLEX_HH_H
 
+#include <functional>
 #include <map>
 #include <string>
 #include <vector>
 
 #include <pwd.h>
 
+#include "base/attr_line.hh"
 #include "base/intern_string.hh"
 #include "base/opt_util.hh"
 #include "shlex.resolver.hh"
 
 enum class shlex_token_t {
-    ST_ERROR,
-    ST_WHITESPACE,
-    ST_ESCAPE,
-    ST_DOUBLE_QUOTE_START,
-    ST_DOUBLE_QUOTE_END,
-    ST_SINGLE_QUOTE_START,
-    ST_SINGLE_QUOTE_END,
-    ST_VARIABLE_REF,
-    ST_QUOTED_VARIABLE_REF,
-    ST_TILDE,
+    eof,
+    whitespace,
+    escape,
+    double_quote_start,
+    double_quote_end,
+    single_quote_start,
+    single_quote_end,
+    variable_ref,
+    quoted_variable_ref,
+    tilde,
 };
 
 class shlex {
 public:
-    shlex(const char* str, size_t len) : s_str(str), s_len(len){};
+    shlex(const char* str, size_t len) : s_str(str), s_len(len) {}
 
     explicit shlex(const string_fragment& sf)
         : s_str(sf.data()), s_len(sf.length())
@@ -65,7 +67,9 @@ public:
     }
 
     explicit shlex(const std::string& str)
-        : s_str(str.c_str()), s_len(str.size()){};
+        : s_str(str.c_str()), s_len(str.size())
+    {
+    }
 
     shlex& with_ignore_quotes(bool val)
     {
@@ -73,11 +77,27 @@ public:
         return *this;
     }
 
-    bool tokenize(string_fragment& cap_out, shlex_token_t& token_out);
+    struct tokenize_result_t {
+        shlex_token_t tr_token;
+        string_fragment tr_frag;
+    };
+
+    struct tokenize_error_t {
+        const char* te_msg{nullptr};
+        string_fragment te_source;
+    };
+
+    Result<tokenize_result_t, tokenize_error_t> tokenize();
 
     bool eval(std::string& result, const scoped_resolver& vars);
 
-    bool split(std::vector<std::string>& result, const scoped_resolver& vars);
+    struct split_element_t {
+        string_fragment se_origin;
+        std::string se_value;
+    };
+
+    Result<std::vector<split_element_t>, tokenize_error_t> split(
+        const scoped_resolver& vars);
 
     void reset()
     {
@@ -85,10 +105,12 @@ public:
         this->s_state = state_t::STATE_NORMAL;
     }
 
-    void scan_variable_ref(string_fragment& cap_out, shlex_token_t& token_out);
+    Result<tokenize_result_t, tokenize_error_t> scan_variable_ref();
 
     void resolve_home_dir(std::string& result, string_fragment cap) const;
 
+    attr_line_t to_attr_line(const tokenize_error_t& te) const;
+
     enum class state_t {
         STATE_NORMAL,
         STATE_IN_DOUBLE_QUOTE,

src/sql_commands.cc

@@ -30,6 +30,7 @@
 #include "base/auto_mem.hh"
 #include "base/fs_util.hh"
 #include "base/injector.bind.hh"
+#include "base/itertools.hh"
 #include "base/lnav_log.hh"
 #include "bound_tags.hh"
 #include "command_executor.hh"
@@ -88,6 +89,7 @@ sql_cmd_read(exec_context& ec,
              std::string cmdline,
              std::vector<std::string>& args)
 {
+    static const intern_string_t SRC = intern_string::lookup("cmdline");
     static auto& lnav_db = injector::get<auto_sqlite3&>();
     static auto& lnav_flags = injector::get<unsigned long&, lnav_flags_tag>();
 
@@ -102,13 +104,23 @@ sql_cmd_read(exec_context& ec,
         return ec.make_error("{} -- unavailable in secure mode", args[0]);
     }
 
-    std::vector<std::string> split_args;
     shlex lexer(cmdline);
 
-    if (!lexer.split(split_args, ec.create_resolver())) {
-        return ec.make_error("unable to parse arguments");
+    auto split_args_res = lexer.split(ec.create_resolver());
+    if (split_args_res.isErr()) {
+        auto split_err = split_args_res.unwrapErr();
+        auto um
+            = lnav::console::user_message::error("unable to parse file name")
+                  .with_reason(split_err.te_msg)
+                  .with_snippet(lnav::console::snippet::from(
+                      SRC, lexer.to_attr_line(split_err)));
+
+        return Err(um);
     }
 
+    auto split_args = split_args_res.unwrap()
+        | lnav::itertools::map([](const auto& elem) { return elem.se_value; });
+
     for (size_t lpc = 1; lpc < split_args.size(); lpc++) {
         auto read_res = lnav::filesystem::read_file(split_args[lpc]);
 

src/textfile_sub_source.cc

@@ -636,6 +636,11 @@ textfile_sub_source::rescan_files(
             continue;
         }
 
+        if (!this->tss_completed_last_scan && lf->size() > 0) {
+            ++iter;
+            continue;
+        }
+
         try {
             const auto& st = lf->get_stat();
             uint32_t old_size = lf->size();
@@ -650,11 +655,6 @@ textfile_sub_source::rescan_files(
                 continue;
             }
 
-            if (!this->tss_completed_last_scan && lf->size() > 0) {
-                ++iter;
-                continue;
-            }
-
             bool new_data = false;
             switch (new_text_data) {
                 case logfile::rebuild_result_t::NEW_LINES:

src/time_formats.am

@@ -68,6 +68,7 @@ TIME_FORMATS = \
     "%m/%e/%Y %l:%M:%S%p" \
     "%m/%d/%y %H:%M:%S" \
     "%m/%d/%Y %H:%M:%S" \
+    "%d/%b/%Y %H:%M:%S" \
     "%d/%b/%y %H:%M:%S" \
     "%m%d %H:%M:%S" \
     "%Y%m%d %H:%M:%S" \

src/top_status_source.cc

@@ -64,12 +64,13 @@ top_status_source::update_time(const timeval& current_time)
 {
     auto& sf = this->tss_fields[TSF_TIME];
     char buffer[32];
+    tm current_tm;
 
     buffer[0] = ' ';
     strftime(&buffer[1],
              sizeof(buffer) - 1,
              this->tss_config.tssc_clock_format.c_str(),
-             localtime(&current_time.tv_sec));
+             localtime_r(&current_time.tv_sec, &current_tm));
     sf.set_value(buffer);
 }
 

test/drive_shlexer.cc

@@ -35,7 +35,7 @@
 using namespace std;
 
 const char* ST_TOKEN_NAMES[] = {
-    "err",
+    "eof",
     "wsp",
     "esc",
     "dst",
@@ -47,6 +47,22 @@ const char* ST_TOKEN_NAMES[] = {
     "til",
 };
 
+static void
+put_underline(FILE* file, string_fragment frag)
+{
+    for (int lpc = 0; lpc < frag.sf_end; lpc++) {
+        if (lpc == frag.sf_begin) {
+            fputc('^', stdout);
+        } else if (lpc == (frag.sf_end - 1)) {
+            fputc('^', stdout);
+        } else if (lpc > frag.sf_begin) {
+            fputc('-', stdout);
+        } else {
+            fputc(' ', stdout);
+        }
+    }
+}
+
 int
 main(int argc, char* argv[])
 {
@@ -56,25 +72,26 @@ main(int argc, char* argv[])
     }
 
     shlex lexer(argv[1], strlen(argv[1]));
-    string_fragment cap;
-    shlex_token_t token;
+    bool done = false;
 
     printf("    %s\n", argv[1]);
-    while (lexer.tokenize(cap, token)) {
-        int lpc;
+    while (!done) {
+        auto tokenize_res = lexer.tokenize();
+        if (tokenize_res.isErr()) {
+            auto te = tokenize_res.unwrapErr();
+
+            printf("err ");
+            put_underline(stdout, te.te_source);
+            printf(" -- %s\n", te.te_msg);
+            break;
+        }
 
-        printf("%s ", ST_TOKEN_NAMES[(int) token]);
-        for (lpc = 0; lpc < cap.sf_end; lpc++) {
-            if (lpc == cap.sf_begin) {
-                fputc('^', stdout);
-            } else if (lpc == (cap.sf_end - 1)) {
-                fputc('^', stdout);
-            } else if (lpc > cap.sf_begin) {
-                fputc('-', stdout);
-            } else {
-                fputc(' ', stdout);
-            }
+        auto tr = tokenize_res.unwrap();
+        if (tr.tr_token == shlex_token_t::eof) {
+            done = true;
         }
+        printf("%s ", ST_TOKEN_NAMES[(int) tr.tr_token]);
+        put_underline(stdout, tr.tr_frag);
         printf("\n");
     }
 
@@ -85,11 +102,14 @@ main(int argc, char* argv[])
         printf("eval -- %s\n", result.c_str());
     }
     lexer.reset();
-    std::vector<std::string> sresult;
-    if (lexer.split(sresult, scoped_resolver{&vars})) {
+    auto split_res = lexer.split(scoped_resolver{&vars});
+    if (split_res.isOk()) {
+        auto sresult = split_res.unwrap();
         printf("split:\n");
         for (size_t lpc = 0; lpc < sresult.size(); lpc++) {
-            printf("  %zu -- %s\n", lpc, sresult[lpc].c_str());
+            printf("% 3zu ", lpc);
+            put_underline(stdout, sresult[lpc].se_origin);
+            printf(" -- %s\n", sresult[lpc].se_value.c_str());
         }
     }
 

test/expected/expected.am

@@ -206,6 +206,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_cmds.sh_ca66660c973f76a3c2a147c7f5035bcb4e8a8bbc.out \
     $(srcdir)/%reldir%/test_cmds.sh_ccd326da92d1cacda63501cd1a3077381a18e8f2.err \
     $(srcdir)/%reldir%/test_cmds.sh_ccd326da92d1cacda63501cd1a3077381a18e8f2.out \
+    $(srcdir)/%reldir%/test_cmds.sh_d0d0ff9b68adc17136329f457fe52d5addcb12c0.err \
+    $(srcdir)/%reldir%/test_cmds.sh_d0d0ff9b68adc17136329f457fe52d5addcb12c0.out \
     $(srcdir)/%reldir%/test_cmds.sh_d1afefacbdd387f02562c8633968b0162a588502.err \
     $(srcdir)/%reldir%/test_cmds.sh_d1afefacbdd387f02562c8633968b0162a588502.out \
     $(srcdir)/%reldir%/test_cmds.sh_d3b69abdfb39e4bfa5828c2f9593e2b2b7ed4d5d.err \

test/expected/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out

@@ -105,6 +105,9 @@
             "docker": {
                 "handler": "docker-url-handler"
             },
+            "docker-compose": {
+                "handler": "docker-compose-url-handler"
+            },
             "hw": {
                 "handler": "hw-url-handler"
             },

test/expected/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.out

@@ -48,9 +48,10 @@
 /tuning/remote/ssh/config/ConnectTimeout -> root-config.json:35
 /tuning/remote/ssh/start-command -> root-config.json:37
 /tuning/remote/ssh/transfer-command -> root-config.json:38
+/tuning/url-scheme/docker-compose/handler -> root-config.json:100
 /tuning/url-scheme/docker/handler -> root-config.json:97
 /tuning/url-scheme/hw/handler -> {test_dir}/configs/installed/hw-url-handler.json:6
-/tuning/url-scheme/piper/handler -> root-config.json:100
+/tuning/url-scheme/piper/handler -> root-config.json:103
 /ui/clock-format -> root-config.json:4
 /ui/default-colors -> root-config.json:6
 /ui/dim-text -> root-config.json:5

test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out

@@ -1444,11 +1444,13 @@ For support questions, email:
    
 
 
-:sh cmdline
+:sh --name=<name> cmdline
 ══════════════════════════════════════════════════════════════════════
   Execute the given command-line and display the captured output
-Parameter
-  cmdline   The command-line to execute.
+Parameters
+  --name=<name>   The name to give to the captured
+                  output
+  cmdline         The command-line to execute.
 See Also
   :alt-msg, :cd, :echo, :eval, :export-session-to, :rebuild, 
   :redirect-to, :write-csv-to, :write-json-to, :write-jsonlines-to, 

test/expected/test_cmds.sh_d0d0ff9b68adc17136329f457fe52d5addcb12c0.err

@@ -0,0 +1,6 @@
+✘ error: invalid epoch time: 16120724091612072409 -- Value too large to be stored in data type
+ --> command-option:1
+ | :unix-time 16120724091612072409         
+ = help: :unix-time seconds
+         ══════════════════════════════════════════════════════════════════════
+           Convert epoch time to a human-readable form

test/expected/test_shlexer.sh_14dd967cb2af90899c9e5e45d00b676b5a3163aa.out

@@ -1,7 +1,8 @@
     ~ foo
 til ^
 wsp  ^
+eof     ^
 eval -- ../test foo
 split:
-  0 -- ../test
-  1 -- foo
+  0 ^ -- ../test
+  1   ^-^ -- foo

test/expected/test_shlexer.sh_2781f5dd570580cbe746ad91b58a28b8371283b3.out

@@ -1,7 +1,8 @@
     ~nonexistent/bar baz
 til ^----------^
 wsp                 ^
+eof                    ^
 eval -- ~nonexistent/bar baz
 split:
-  0 -- ~nonexistent/bar
-  1 -- baz
+  0 ^--------------^ -- ~nonexistent/bar
+  1                  ^-^ -- baz

test/expected/test_shlexer.sh_2af44d06fc137a77bc230be86376ccad23a2806b.out

@@ -1,2 +1,2 @@
     \
-err ^
+err ^ -- invalid escape

test/expected/test_shlexer.sh_6858e530a8ecb77cbaec1a7507768dd5a1942ac9.out

@@ -1,5 +1,6 @@
     ${FOO}
 qrf ^----^
+eof      ^
 eval -- bar
 split:
-  0 -- bar
+  0 ^----^ -- bar

test/expected/test_shlexer.sh_7f31e16ea2469da7a4328c93c7bcc8e109f84d2f.out

@@ -2,6 +2,7 @@
 dst ^
 qrf      ^----^
 den                ^
+eof                ^
 eval -- "abc xyz 123"
 split:
-  0 -- abc xyz 123
+  0 ^--------------^ -- abc xyz 123

test/expected/test_shlexer.sh_8aeebcdef56edd783579eaaddaff7c5cc127bb86.out

@@ -1,6 +1,7 @@
     'abc $DEF 123'
 sst ^
 sen              ^
+eof              ^
 eval -- 'abc $DEF 123'
 split:
-  0 -- abc $DEF 123
+  0 ^------------^ -- abc $DEF 123

test/expected/test_shlexer.sh_8e9addb0e5b6f4254d81dd89ecf12783109644bb.out

@@ -2,6 +2,7 @@
 dst ^
 ref      ^--^
 den              ^
+eof              ^
 eval -- "abc xyz 123"
 split:
-  0 -- abc xyz 123
+  0 ^------------^ -- abc xyz 123

test/expected/test_shlexer.sh_90961e6728e96d0a44535a6c9907cc990c10316c.out

@@ -1,6 +1,7 @@
     "def"
 dst ^
 den     ^
+eof     ^
 eval -- "def"
 split:
-  0 -- def
+  0 ^---^ -- def

test/expected/test_shlexer.sh_95c4e861804a5434900fdb4d67b149d1baa2edf4.out

@@ -1,5 +1,6 @@
     $FOO
 ref ^--^
+eof    ^
 eval -- bar
 split:
-  0 -- bar
+  0 ^--^ -- bar

test/expected/test_shlexer.sh_d7fe5f6b8fc9ba00539fad0fa0bfb08319d8b04b.out

@@ -1,6 +1,7 @@
     'abc'
 sst ^
 sen     ^
+eof     ^
 eval -- 'abc'
 split:
-  0 -- abc
+  0 ^---^ -- abc

test/expected/test_shlexer.sh_d9d46422a913e3a06ddbd262933ef5352c30e68f.out

@@ -2,8 +2,9 @@
 wsp    ^
 ref     ^--^
 wsp         ^^
+eof             ^
 eval -- abc xyz  123
 split:
-  0 -- abc
-  1 -- xyz
-  2 -- 123
+  0 ^-^ -- abc
+  1     ^--^ -- xyz
+  2           ^-^ -- 123

test/expected/test_shlexer.sh_e0599f0b53d1bd27af767113853f8e84291f137d.out

@@ -1,6 +1,7 @@
     '"'
 sst ^
 sen   ^
+eof   ^
 eval -- '"'
 split:
-  0 -- "
+  0 ^-^ -- "

test/expected/test_shlexer.sh_e8fa2239ab17e7563d0c524f5400a79d6ff8bfda.out

@@ -1,6 +1,7 @@
     "'"
 dst ^
 den   ^
+eof   ^
 eval -- "'"
 split:
-  0 -- '
+  0 ^-^ -- '

test/formats/jsontest/lnav-logstash.json

@@ -7,40 +7,48 @@
         "json": true,
         "hide-extra": false,
         "file-pattern": "\\.clog.*",
-        "multiline": false,
         "line-format": [
-            { "field" : "@timestamp" },
+            {
+                "field": "@timestamp"
+            },
             " ",
-            { "field" : "ipaddress" },
+            {
+                "field": "ipaddress"
+            },
             " ",
-            { "field" : "message" },
+            {
+                "field": "message"
+            },
             " ",
-            { "field" : "stack_trace", "default-value" : "" }
+            {
+                "field": "stack_trace",
+                "default-value": ""
+            }
         ],
-        "timestamp-field" : "@timestamp",
-        "body-field" : "message",        
-        "level-field" : "level",
-        "level" : {
-            "trace" : "TRACE",
-            "debug" : "DEBUG",
-            "info" : "INFO",
-            "error" : "ERROR",
-            "warning" : "WARN"
+        "timestamp-field": "@timestamp",
+        "body-field": "message",
+        "level-field": "level",
+        "level": {
+            "trace": "TRACE",
+            "debug": "DEBUG",
+            "info": "INFO",
+            "error": "ERROR",
+            "warning": "WARN"
         },
-        "value" : {
-            "logger_name" : {
-                "kind" : "string",
-                "identifier" : true
+        "value": {
+            "logger_name": {
+                "kind": "string",
+                "identifier": true
             },
-            "ipaddress" : {
-                "kind" : "string",
-                "identifier" : true
+            "ipaddress": {
+                "kind": "string",
+                "identifier": true
             },
-            "level_value" : {
+            "level_value": {
                 "hidden": true
             },
-            "stack_trace" : {
-                "kind" : "string"
+            "stack_trace": {
+                "kind": "string"
             }
         }
     }

test/lnav_doctests.cc

@@ -37,6 +37,7 @@
 #include "lnav_util.hh"
 #include "ptimec.hh"
 #include "relative_time.hh"
+#include "shlex.hh"
 #include "unique_path.hh"
 
 using namespace std;
@@ -61,6 +62,66 @@ TEST_CASE("overwritten-logfile") {
 }
 #endif
 
+TEST_CASE("shlex::eval")
+{
+    std::string cmdline1 = "${semantic_highlight_color}";
+
+    shlex lexer(cmdline1);
+
+    std::map<std::string, scoped_value_t> vars = {
+        {"semantic_highlight_color", "foo"},
+    };
+
+    std::string out;
+    auto rc = lexer.eval(out, scoped_resolver{&vars});
+    CHECK(rc);
+    CHECK(out == "foo");
+}
+
+TEST_CASE("shlex::split")
+{
+    {
+        std::string cmdline1 = "";
+
+        std::map<std::string, scoped_value_t> vars;
+        shlex lexer(cmdline1);
+        auto split_res = lexer.split(scoped_resolver{&vars});
+        CHECK(split_res.isOk());
+        auto args = split_res.unwrap();
+        CHECK(args.empty());
+    }
+    {
+        std::string cmdline1 = ":sh --name=\"foo $BAR\" echo Hello!";
+
+        std::map<std::string, scoped_value_t> vars;
+        shlex lexer(cmdline1);
+        auto split_res = lexer.split(scoped_resolver{&vars});
+        CHECK(split_res.isOk());
+        auto args = split_res.unwrap();
+        for (const auto& se : args) {
+            printf(" range %d:%d -- %s\n",
+                   se.se_origin.sf_begin,
+                   se.se_origin.sf_end,
+                   se.se_value.c_str());
+        }
+    }
+    {
+        std::string cmdline1 = "abc def $FOO ghi";
+
+        std::map<std::string, scoped_value_t> vars;
+        shlex lexer(cmdline1);
+        auto split_res = lexer.split(scoped_resolver{&vars});
+        CHECK(split_res.isOk());
+        auto args = split_res.unwrap();
+        for (const auto& se : args) {
+            printf(" range %d:%d -- %s\n",
+                   se.se_origin.sf_begin,
+                   se.se_origin.sf_end,
+                   se.se_value.c_str());
+        }
+    }
+}
+
 TEST_CASE("byte_array")
 {
     using my_array_t = byte_array<8>;

test/test_cmds.sh

@@ -48,6 +48,10 @@ run_cap_test env TZ=UTC ${lnav_test} -n \
     -c ":unix-time 1612072409" \
     "${test_dir}/logfile_access_log.*"
 
+run_cap_test env TZ=UTC ${lnav_test} -n \
+    -c ":unix-time 16120724091612072409" \
+    "${test_dir}/logfile_access_log.*"
+
 run_cap_test env TZ=UTC ${lnav_test} -n \
     -c ":current-time" \
     "${test_dir}/logfile_access_log.*"