[cmd] add pipe-to and pipe-line-to commands

NEWS

@@ -1,4 +1,9 @@
 
+lnav v0.7.3:
+     * Add 'pipe-to' and 'pipe-line-to' commands that pipe the currently
+       marked lines or the current log message to a shell command,
+       respectively.
+
 lnav v0.7.2:
      * Added log formats for vdsm, openstack, and the vmkernel.
      * Added a "lo-fi" mode (L hotkey) that dumps the displayed log lines

docs/source/commands.rst

@@ -64,3 +64,7 @@ Output
   Use '-' to write the lines to the terminal.
 * write-json-to <file> - Write SQL query results to the given file in JSON
   format.  Use '-' to write the lines to the terminal.
+* pipe-to <shell-cmd> - Pipe the bookmarked lines in the current view to a
+  shell command and open the output in lnav.
+* pipe-line-to <shell-cmd> - Pipe the top line in the current view to a shell
+  command and open the output in lnav.

src/auto_fd.hh

@@ -198,6 +198,13 @@ public:
         }
     };
 
+    void close_on_exec() const {
+        if (this->af_fd == -1) {
+            return;
+        }
+        log_perror(fcntl(this->af_fd, F_SETFD, FD_CLOEXEC));
+    }
+
 private:
     int af_fd;  /*< The managed file descriptor. */
 };

src/help.txt

@@ -440,6 +440,14 @@ COMMANDS
                     to write to standard out.  Use '-' to write the data to
                     the terminal.
 
+  pipe-to <shell-cmd>
+                    Send the currently marked lines to the given shell command
+                    for processing and open the resulting file for viewing.
+
+  pipe-line-to <shell-cmd>
+                    Send the top log message to the given shell command
+                    for processing and open the resulting file for viewing.
+
   session <cmd>     Add the given command to the session file
                     (~/.lnav/session).  Any commands listed in the session file
                     are executed on startup.  Only the highlight, word-wrap, and

src/lnav.cc

@@ -3442,6 +3442,19 @@ void update_hits(void *dummy, textview_curses *tc)
     }
 }
 
+static void gather_pipers(void)
+{
+    for (std::list<piper_proc *>::iterator iter = lnav_data.ld_pipers.begin();
+         iter != lnav_data.ld_pipers.end(); ) {
+        if ((*iter)->has_exited()) {
+            delete *iter;
+            iter = lnav_data.ld_pipers.erase(iter);
+        } else {
+            ++iter;
+        }
+    }
+}
+
 static void looper(void)
 {
     try {
@@ -3482,6 +3495,7 @@ static void looper(void)
         (void)signal(SIGTERM, sigint);
         (void)signal(SIGWINCH, sigwinch);
         (void)signal(SIGCHLD, sigchld);
+        (void)signal(SIGPIPE, SIG_IGN);
 
         screen_curses sc;
         lnav_behavior lb;
@@ -3778,15 +3792,7 @@ static void looper(void)
                     iter = lnav_data.ld_children.erase(iter);
                 }
 
-                for (std::list<piper_proc *>::iterator iter = lnav_data.ld_pipers.begin();
-                    iter != lnav_data.ld_pipers.end(); ) {
-                    if ((*iter)->has_exited()) {
-                        delete *iter;
-                        iter = lnav_data.ld_pipers.erase(iter);
-                    } else {
-                        ++iter;
-                    }
-                }
+                gather_pipers();
             }
         }
     }
@@ -4428,6 +4434,16 @@ int main(int argc, char *argv[])
                 lnav_data.ld_views[LNV_LOG].set_top(vis_line_t(0));
 
                 execute_init_commands(msgs);
+                for (;;) {
+                    gather_pipers();
+                    if (lnav_data.ld_pipers.empty()) {
+                        break;
+                    }
+                    else {
+                        usleep(10000);
+                        rebuild_indexes(false);
+                    }
+                }
                 rebuild_indexes(false);
 
                 for (msg_iter = msgs.begin();

src/lnav_commands.cc

@@ -468,6 +468,120 @@ static string com_save_to(string cmdline, vector<string> &args)
     return "";
 }
 
+static string com_pipe_to(string cmdline, vector<string> &args)
+{
+    string retval = "error: expecting command to execute";
+
+    if (args.size() == 0) {
+        args.push_back("filename");
+        return "";
+    }
+
+    if (args.size() < 2) {
+        return retval;
+    }
+
+    textview_curses *            tc = lnav_data.ld_view_stack.top();
+    bookmark_vector<vis_line_t> &bv =
+            tc->get_bookmarks()[&textview_curses::BM_USER];
+
+    string cmd = trim(cmdline.substr(cmdline.find(args[1], args[0].size())));
+    auto_pipe in_pipe(STDIN_FILENO);
+    auto_pipe out_pipe(STDOUT_FILENO);
+
+    in_pipe.open();
+    out_pipe.open();
+
+    pid_t child_pid = fork();
+
+    in_pipe.after_fork(child_pid);
+    out_pipe.after_fork(child_pid);
+
+    switch (child_pid) {
+        case -1:
+            return "error: unable to fork child process -- " + string(strerror(errno));
+
+        case 0: {
+            const char *args[] = {
+                    "sh", "-c", cmd.c_str(), NULL,
+            };
+            vector<std::string> path_v;
+            string path;
+
+            dup2(STDOUT_FILENO, STDERR_FILENO);
+            path_v.push_back(dotlnav_path("formats/default"));
+            setenv("PATH", build_path(path_v).c_str(), 1);
+            execvp(args[0], (char *const *) args);
+            _exit(1);
+            break;
+        }
+
+        default:
+            bookmark_vector<vis_line_t>::iterator iter;
+            static int exec_count = 0;
+            string line;
+
+            in_pipe.read_end().close_on_exec();
+            in_pipe.write_end().close_on_exec();
+
+            lnav_data.ld_children.push_back(child_pid);
+            if (out_pipe.read_end() != -1) {
+                piper_proc *pp = new piper_proc(out_pipe.read_end(), false);
+                char desc[128];
+
+                lnav_data.ld_pipers.push_back(pp);
+                snprintf(desc,
+                        sizeof(desc), "[%d] Output of %s",
+                        exec_count++,
+                        cmdline.c_str());
+                lnav_data.ld_file_names.insert(make_pair(
+                        desc,
+                        pp->get_fd()));
+                lnav_data.ld_files_to_front.push_back(make_pair(desc, 0));
+                if (lnav_data.ld_rl_view != NULL) {
+                    lnav_data.ld_rl_view->set_alt_value(HELP_MSG_1(
+                            X, "to close the file"));
+                }
+            }
+
+            if (args[0] == "pipe-line-to") {
+                if (tc == &lnav_data.ld_views[LNV_LOG]) {
+                    logfile_sub_source &lss = lnav_data.ld_log_source;
+                    content_line_t cl = lss.at(tc->get_top());
+                    logfile *lf = lss.find(cl);
+                    shared_buffer_ref sbr;
+                    lf->read_full_message(lf->message_start(lf->begin() + cl), sbr);
+                    if (write(in_pipe.write_end(), sbr.get_data(), sbr.length()) == -1) {
+                        return "error: Unable to write to pipe -- " + string(strerror(errno));
+                    }
+                    write(in_pipe.write_end(), "\n", 1);
+                }
+                else {
+                    tc->grep_value_for_line(tc->get_top(), line);
+                    if (write(in_pipe.write_end(), line.c_str(), line.size()) == -1) {
+                        return "error: Unable to write to pipe -- " + string(strerror(errno));
+                    }
+                    write(in_pipe.write_end(), "\n", 1);
+                }
+            }
+            else {
+                for (iter = bv.begin(); iter != bv.end(); iter++) {
+                    tc->grep_value_for_line(*iter, line);
+                    if (write(in_pipe.write_end(), line.c_str(), line.size()) == -1) {
+                        return "error: Unable to write to pipe -- " + string(strerror(errno));
+                    }
+                    write(in_pipe.write_end(), "\n", 1);
+                }
+            }
+            in_pipe.close();
+
+            retval = "";
+            break;
+    }
+
+    return retval;
+}
+
 static string com_highlight(string cmdline, vector<string> &args)
 {
     string retval = "error: expecting regular expression to highlight";
@@ -1494,6 +1608,8 @@ void init_lnav_commands(readline_context::command_map_t &cmd_map)
     cmd_map["write-to"]             = com_save_to;
     cmd_map["write-csv-to"]         = com_save_to;
     cmd_map["write-json-to"]        = com_save_to;
+    cmd_map["pipe-to"]              = com_pipe_to;
+    cmd_map["pipe-line-to"]         = com_pipe_to;
     cmd_map["enable-filter"]        = com_enable_filter;
     cmd_map["disable-filter"]       = com_disable_filter;
     cmd_map["enable-word-wrap"]     = com_enable_word_wrap;

src/lnav_log.hh

@@ -32,9 +32,11 @@
 #ifndef __lnav_log_hh
 #define __lnav_log_hh
 
+#include <errno.h>
 #include <stdio.h>
 #include <termios.h>
 #include <stdlib.h>
+#include <string.h>
 #include <sys/cdefs.h>
 
 #ifndef __dead2
@@ -90,4 +92,9 @@ extern enum lnav_log_level_t lnav_log_level;
 #define __ensure(e, file, line) \
     (log_msg(LOG_LEVEL_ERROR, file, line, "failed postcondition `%s'", e), log_abort(), 1)
 
+#define log_perror(e)  \
+    ((void) ((e != -1) ? 0 : __log_perror (#e, __FILE__, __LINE__)))
+#define __log_perror(e, file, line) \
+    (log_msg(LOG_LEVEL_ERROR, file, line, "syscall failed `%s' -- %s", e, strerror(errno)), 1)
+
 #endif

src/lnav_util.cc

@@ -42,6 +42,8 @@
 #include "auto_fd.hh"
 #include "lnav_util.hh"
 
+using namespace std;
+
 std::string hash_string(const std::string &str)
 {
     byte_array<2, uint64> hash;
@@ -532,3 +534,22 @@ size_t strtonum<long>(long &num_out, const char *string, size_t len);
 
 template
 size_t strtonum<int>(int &num_out, const char *string, size_t len);
+
+string build_path(const vector<string> &paths)
+{
+    string retval;
+
+    for (vector<string>::const_iterator path_iter = paths.begin();
+         path_iter != paths.end();
+         ++path_iter) {
+        if (path_iter->empty()) {
+            continue;
+        }
+        if (!retval.empty()) {
+            retval += ":";
+        }
+        retval += *path_iter;
+    }
+    retval += ":" + string(getenv("PATH"));
+    return retval;
+}

src/lnav_util.hh

@@ -40,6 +40,7 @@
 #include "spookyhash/SpookyV2.h"
 
 #include <string>
+#include <vector>
 
 #include "ptimec.hh"
 #include "byte_array.hh"
@@ -170,6 +171,8 @@ inline bool is_glob(const char *fn)
             strchr(fn, '[') != NULL);
 };
 
+std::string build_path(const std::vector<std::string> &paths);
+
 /**
  * Convert the time stored in a 'tm' struct into epoch time.
  *

src/log_vtab_impl.cc

@@ -603,6 +603,17 @@ static int vt_update(sqlite3_vtab *tab,
         int val = sqlite3_value_int(argv[2 + VT_COL_MARK]);
 
         vt->tc->set_user_mark(&textview_curses::BM_USER, vis_line_t(rowid), val);
+        rowid += 1;
+        while (rowid < vt->lss->text_line_count()) {
+            vis_line_t vl(rowid);
+            content_line_t cl = vt->lss->at(vl);
+            logline *ll = vt->lss->find_line(cl);
+            if (!ll->is_continued()) {
+                break;
+            }
+            vt->tc->set_user_mark(&textview_curses::BM_USER, vl, val);
+            rowid += 1;
+        }
 
         retval = SQLITE_OK;
     }

src/logfile.cc

@@ -88,9 +88,7 @@ throw (error)
             throw error(filename, errno);
         }
 
-        if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1) {
-            throw error(filename, errno);
-        }
+        fd.close_on_exec();
 
         this->lf_valid_filename = true;
     }
@@ -388,7 +386,9 @@ void logfile::read_full_message(logfile::iterator ll,
             /* ... */
         }
         ++ll;
-        max_lines -= 1;
+        if (max_lines != -1) {
+            max_lines -= 1;
+        }
     } while (ll != this->end() && ll->is_continued() &&
              (max_lines == -1 || max_lines > 0));
 

src/logfile.hh

@@ -244,6 +244,17 @@ public:
         return retval;
     };
 
+    iterator message_start(iterator ll) {
+        iterator retval = ll;
+
+        while (retval != this->begin() &&
+                (retval->get_sub_offset() != 0 || retval->is_continued())) {
+            --retval;
+        }
+
+        return retval;
+    }
+
     size_t line_length(iterator ll) {
         iterator next_line = ll;
         size_t retval;

src/piper_proc.cc

@@ -111,7 +111,21 @@ piper_proc::piper_proc(int pipefd, bool timestamp, const char *filename)
         int nullfd;
 
         nullfd = open("/dev/null", O_RDWR);
+        dup2(nullfd, STDIN_FILENO);
         dup2(nullfd, STDOUT_FILENO);
+        for (int fd_to_close = 0; fd_to_close < 1024; fd_to_close++) {
+            int flags;
+
+            if (fd_to_close == this->pp_fd.get()) {
+                continue;
+            }
+            if ((flags = fcntl(fd_to_close, F_GETFD)) == -1) {
+                continue;
+            }
+            if (flags & FD_CLOEXEC) {
+                close(fd_to_close);
+            }
+        }
         fcntl(infd.get(), F_SETFL, O_NONBLOCK);
         lb.set_fd(infd);
         do {

test/test_cmds.sh

@@ -259,6 +259,31 @@ check_output "write-json-to is not working" <<EOF
 EOF
 
 
+run_test ${lnav_test} -n \
+    -c ";update generic_log set log_mark=1" \
+    -c ":pipe-to sed -e 's/World!/Bork!/g'" \
+    ${test_dir}/logfile_multiline.0
+check_output "pipe-to is not working" <<EOF
+2009-07-20 22:59:27,672:DEBUG:Hello, Bork!
+  How are you today?
+2009-07-20 22:59:27,672:DEBUG:Hello, World!
+  How are you today?
+2009-07-20 22:59:30,221:ERROR:Goodbye, Bork!
+2009-07-20 22:59:30,221:ERROR:Goodbye, World!
+EOF
+
+run_test ${lnav_test} -n \
+    -c ":goto 2" \
+    -c ":pipe-line-to sed -e 's/World!/Bork!/g'" \
+    ${test_dir}/logfile_multiline.0
+check_output "pipe-line-to is not working" <<EOF
+2009-07-20 22:59:27,672:DEBUG:Hello, World!
+  How are you today?
+2009-07-20 22:59:30,221:ERROR:Goodbye, Bork!
+2009-07-20 22:59:30,221:ERROR:Goodbye, World!
+EOF
+
+
 run_test ${lnav_test} -n \
     -c ":set-min-log-level error" \
     ${test_dir}/logfile_access_log.0