[sql] add a log_search table

NEWS

@@ -4,7 +4,10 @@ lnav v0.8.0:
      * Integration with "papertrailapp.com" for querying and tailing
        server log and syslog messages.  See the Papertrail section in
        the online help for more details.
-     * Remote files can be opened when linked with libcurl.
+     * Remote files can be opened when lnav is built with libcurl.
+     * SQL queries can now be done on lines that match a regular expression
+       using the "log_search" table or by creating custom tables with the
+       ":create-search-table" command.
      * Log formats that are "containers" for other log formats, like
        syslog, are now supported.  See the online help for more
        information.

docs/source/commands.rst

@@ -63,6 +63,23 @@ Display
 
 * redraw - Redraw the window to correct any corruption.
 
+
+SQL
+---
+
+* create-logline-table <table-name> - Create an SQL table using the top line
+  of the log view as a template.  See the :ref:`data-ext` section for more information.
+
+* delete-logline-table <table-name> - Delete a table created by create-logline-table.
+
+* create-search-table <table-name> [regex] - Create an SQL table that
+  extracts information from logs using the provided regular expression or the
+  last search that was done.  Any captures in the expression will be used as
+  columns in the SQL table.  If the capture is named, that name will be used as
+  the column name, otherwise the column name will be of the form 'col_N'.
+* delete-search-table <table-name> - Delete a table that was created with create-search-table.
+
+
 Output
 ------
 

src/CMakeLists.txt

@@ -96,6 +96,7 @@ set(diag_STAT_SRCS
     log_data_helper.hh
     log_data_table.hh
     log_format_impls.cc
+    log_search_table.hh
     logfile_stats.hh
     plain_text_source.hh
     pretty_printer.hh

src/Makefile.am

@@ -137,6 +137,7 @@ noinst_HEADERS = \
     log_data_table.hh \
 	log_format.hh \
 	log_format_loader.hh \
+	log_search_table.hh \
 	logfile.hh \
 	logfile_sub_source.hh \
 	papertrail_proc.hh \

src/help.txt

@@ -487,6 +487,17 @@ COMMANDS
                     Delete an SQL table created by the 'create-logline-table'
                     command.
 
+  create-search-table <table-name> [<regex>]
+                    Create an SQL table that extracts information from logs
+                    using the provided regular expression or the last search
+                    that was done.  Any captures in the expression will be
+                    used as columns in the SQL table.  If the capture is named,
+                    that name will be used as the column name, otherwise the
+                    column name will be of the form 'col_N'.
+
+  delete-search-table <table-name>
+                    Delete a table that was created with create-search-table.
+
   switch-to-view <view-name>
                     Switch the display to the given view, which can be one of:
                     help, log, text, histogram, db, and schema.

src/lnav.cc

@@ -132,6 +132,7 @@
 #include "readline_possibilities.hh"
 #include "field_overlay_source.hh"
 #include "url_loader.hh"
+#include "log_search_table.hh"
 
 using namespace std;
 
@@ -936,10 +937,10 @@ void execute_search(lnav_view_t view, const std::string &regex_orig)
     auto_ptr<grep_highlighter> &gc = lnav_data.ld_search_child[view];
     textview_curses &           tc = lnav_data.ld_views[view];
     std::string regex = regex_orig;
+    pcre *      code = NULL;
 
     if ((gc.get() == NULL) || (regex != lnav_data.ld_last_search[view])) {
         const char *errptr;
-        pcre *      code = NULL;
         int         eoff;
         bool quoted = false;
 
@@ -1004,6 +1005,16 @@ void execute_search(lnav_view_t view, const std::string &regex_orig)
                 new grep_highlighter(gp, "$search", hm));
             gc = gh;
         }
+
+        if (view == LNV_LOG) {
+            static intern_string_t log_search_name = intern_string::lookup("log_search");
+
+            lnav_data.ld_vtab_manager->unregister_vtab(log_search_name);
+            if (code != NULL) {
+                lnav_data.ld_vtab_manager->register_vtab(new log_search_table(
+                        regex.c_str(), log_search_name));
+            }
+        }
     }
 
     lnav_data.ld_last_search[view] = regex;

src/lnav_commands.cc

@@ -49,6 +49,7 @@
 #include "command_executor.hh"
 #include "url_loader.hh"
 #include "readline_curses.hh"
+#include "log_search_table.hh"
 
 using namespace std;
 
@@ -1014,7 +1015,7 @@ static string com_disable_word_wrap(string cmdline, vector<string> &args)
     return retval;
 }
 
-static std::vector<string> custom_logline_tables;
+static std::set<string> custom_logline_tables;
 
 static string com_create_logline_table(string cmdline, vector<string> &args)
 {
@@ -1035,6 +1036,7 @@ static string com_create_logline_table(string cmdline, vector<string> &args)
 
             errmsg = lnav_data.ld_vtab_manager->register_vtab(ldt);
             if (errmsg.empty()) {
+                custom_logline_tables.insert(args[1]);
                 if (lnav_data.ld_rl_view != NULL) {
                     lnav_data.ld_rl_view->add_possibility(LNM_COMMAND,
                       "custom-table",
@@ -1043,6 +1045,7 @@ static string com_create_logline_table(string cmdline, vector<string> &args)
                 retval = "info: created new log table -- " + args[1];
             }
             else {
+                delete ldt;
                 retval = "error: unable to create table -- " + errmsg;
             }
         }
@@ -1059,6 +1062,10 @@ static string com_delete_logline_table(string cmdline, vector<string> &args)
         args.push_back("custom-table");
     }
     else if (args.size() == 2) {
+        if (custom_logline_tables.find(args[1]) == custom_logline_tables.end()) {
+            return "error: unknown logline table -- " + args[1];
+        }
+
         string rc = lnav_data.ld_vtab_manager->unregister_vtab(
                 intern_string::lookup(args[1]));
 
@@ -1078,6 +1085,85 @@ static string com_delete_logline_table(string cmdline, vector<string> &args)
     return retval;
 }
 
+static std::set<string> custom_search_tables;
+
+static string com_create_search_table(string cmdline, vector<string> &args)
+{
+    string retval = "error: expecting a table name";
+
+    if (args.size() == 0) {
+
+    }
+    else if (args.size() >= 2) {
+        log_search_table *lst;
+        string regex;
+
+        if (args.size() >= 3) {
+            regex = cmdline.substr(cmdline.find(args[2], args[0].size() + args[1].size()));
+        }
+        else {
+            regex = lnav_data.ld_last_search[LNV_LOG];
+        }
+
+        try {
+            lst = new log_search_table(regex.c_str(),
+                                       intern_string::lookup(args[1]));
+        } catch (pcrepp::error &e) {
+            return "error: unable to compile regex -- " + regex;
+        }
+
+        string          errmsg;
+
+        errmsg = lnav_data.ld_vtab_manager->register_vtab(lst);
+        if (errmsg.empty()) {
+            custom_search_tables.insert(args[1]);
+            if (lnav_data.ld_rl_view != NULL) {
+                lnav_data.ld_rl_view->add_possibility(LNM_COMMAND,
+                                                      "search-table",
+                                                      args[1]);
+            }
+            retval = "info: created new search table -- " + args[1];
+        }
+        else {
+            delete lst;
+            retval = "error: unable to create table -- " + errmsg;
+        }
+    }
+
+    return retval;
+}
+
+static string com_delete_search_table(string cmdline, vector<string> &args)
+{
+    string retval = "error: expecting a table name";
+
+    if (args.size() == 0) {
+        args.push_back("search-table");
+    }
+    else if (args.size() == 2) {
+        if (custom_search_tables.find(args[1]) == custom_search_tables.end()) {
+            return "error: unknown search table -- " + args[1];
+        }
+
+        string rc = lnav_data.ld_vtab_manager->unregister_vtab(
+                intern_string::lookup(args[1]));
+
+        if (rc.empty()) {
+            if (lnav_data.ld_rl_view != NULL) {
+                lnav_data.ld_rl_view->rem_possibility(LNM_COMMAND,
+                                                      "search-table",
+                                                      args[1]);
+            }
+            retval = "info: deleted search table";
+        }
+        else {
+            retval = "error: " + rc;
+        }
+    }
+
+    return retval;
+}
+
 static string com_session(string cmdline, vector<string> &args)
 {
     string retval = "error: expecting a command to save to the session file";
@@ -1989,6 +2075,18 @@ readline_context::command_t STD_COMMANDS[] = {
         "Delete a table created with create-logline-table",
         com_delete_logline_table,
     },
+    {
+        "create-search-table",
+        "<table-name> [<regex>]",
+        "Create an SQL table based on a regex search",
+        com_create_search_table,
+    },
+    {
+        "delete-search-table",
+        "<table-name>",
+        "Delete a table created with create-search-table",
+        com_delete_search_table,
+    },
     {
         "open",
         "<filename>",

src/log_data_table.hh

@@ -27,9 +27,6 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * @file log_data_table.hh
- *
- * XXX This file has become a dumping ground for code and needs to be broken up
- * a bit.
  */
 
 #ifndef _log_data_table_hh

src/log_search_table.hh

@@ -0,0 +1,132 @@
+/**
+ * Copyright (c) 2015, Timothy Stack
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of Timothy Stack nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * @file log_search_table.hh
+ */
+
+#ifndef _log_search_table_hh
+#define _log_search_table_hh
+
+#include <string>
+#include <vector>
+
+#include "lnav.hh"
+#include "logfile.hh"
+#include "data_parser.hh"
+#include "column_namer.hh"
+#include "log_vtab_impl.hh"
+
+class log_search_table : public log_vtab_impl {
+public:
+
+    log_search_table(const char *regex, intern_string_t table_name)
+        : log_vtab_impl(table_name),
+          lst_regex(regex) {
+    };
+
+    void get_columns(std::vector<vtab_column> &cols)
+    {
+        column_namer cn;
+
+        for (int lpc = 0; lpc < this->lst_regex.get_capture_count(); lpc++) {
+            // TODO: it would be nice to figure out the type and collator
+            // We could test the regex with some canned data to figure out
+            // what it will and will not accept.  That way we don't have to
+            // check actual log data.
+            cols.push_back(vtab_column(cn.add_column(
+                    this->lst_regex.name_for_capture(lpc))));
+        }
+    };
+
+    bool next(log_cursor &lc, logfile_sub_source &lss)
+    {
+        lc.lc_curr_line = lc.lc_curr_line + vis_line_t(1);
+        lc.lc_sub_index = 0;
+
+        if (lc.lc_curr_line == (int)lss.text_line_count()) {
+            return true;
+        }
+
+        content_line_t cl;
+
+        cl = lss.at(lc.lc_curr_line);
+        logfile *         lf      = lss.find(cl);
+        logfile::iterator lf_iter = lf->begin() + cl;
+
+        if (lf_iter->is_continued()) {
+            return false;
+        }
+
+        string_attrs_t             sa;
+        std::vector<logline_value> line_values;
+
+        lf->read_full_message(lf_iter, this->lst_current_line);
+        lf->get_format()->annotate(this->lst_current_line, sa, line_values);
+        this->lst_body = find_string_attr_range(sa, &textview_curses::SA_BODY);
+        if (this->lst_body.lr_end == -1 || this->lst_body.length() == 0) {
+            return false;
+        }
+
+        pcre_input pi(&this->lst_current_line.get_data()[this->lst_body.lr_start],
+                      0,
+                      this->lst_body.length());
+
+        return this->lst_regex.match(this->lst_match_context, pi);
+    };
+
+    void extract(logfile *lf,
+                 shared_buffer_ref &line,
+                 std::vector<logline_value> &values)
+    {
+        pcre_input pi(&this->lst_current_line.get_data()[this->lst_body.lr_start],
+                      0,
+                      this->lst_body.length());
+        int next_column = 0;
+
+        for (int lpc = 0; lpc < this->lst_regex.get_capture_count(); lpc++) {
+            pcre_context::capture_t *cap = this->lst_match_context[lpc];
+            shared_buffer_ref value_sbr;
+
+            value_sbr.subset(line,
+                             this->lst_body.lr_start + cap->c_begin,
+                             cap->length());
+            values.push_back(logline_value(intern_string::lookup("", 0),
+                                           logline_value::VALUE_TEXT,
+                                           value_sbr));
+            values.back().lv_column = next_column++;
+        }
+    };
+
+private:
+    pcrepp lst_regex;
+    shared_buffer_ref lst_current_line;
+    struct line_range lst_body;
+    pcre_context_static<128> lst_match_context;
+};
+
+#endif

src/log_vtab_impl.cc

@@ -78,9 +78,7 @@ std::string log_vtab_impl::get_table_statement(void)
         auto_mem<char, sqlite3_free> coldecl;
         auto_mem<char, sqlite3_free> colname;
 
-        require(iter->vc_name != NULL);
+        colname = sql_quote_ident(iter->vc_name.c_str());
-
-        colname = sql_quote_ident(iter->vc_name);
         coldecl = sqlite3_mprintf("  %s %s %s collate %Q,\n",
                                   colname.in(),
                                   type_to_string(iter->vc_type),
@@ -671,7 +669,7 @@ string log_vtab_manager::register_vtab(log_vtab_impl *vi)
 
     if (this->vm_impls.find(vi->get_name()) == this->vm_impls.end()) {
         auto_mem<char> errmsg(sqlite3_free);
-        char *         sql;
+        auto_mem<char> sql(sqlite3_free);
         int            rc;
 
         this->vm_impls[vi->get_name()] = vi;
@@ -688,8 +686,6 @@ string log_vtab_manager::register_vtab(log_vtab_impl *vi)
         if (rc != SQLITE_OK) {
             retval = errmsg;
         }
-
-        sqlite3_free(sql);
     }
     else {
         retval = "a table with the given name already exists";
@@ -706,7 +702,7 @@ string log_vtab_manager::unregister_vtab(intern_string_t name)
         retval = "unknown log line table -- " + name.to_string();
     }
     else {
-        char *sql;
+        auto_mem<char> sql(sqlite3_free);
         int   rc;
 
         sql = sqlite3_mprintf("DROP TABLE %s ", name.get());
@@ -717,8 +713,6 @@ string log_vtab_manager::unregister_vtab(intern_string_t name)
                            NULL);
         require(rc == SQLITE_OK);
 
-        sqlite3_free(sql);
-
         this->vm_impls.erase(name);
     }
 

src/log_vtab_impl.hh

@@ -70,13 +70,13 @@ struct log_cursor {
 class log_vtab_impl {
 public:
     struct vtab_column {
-        vtab_column(const char *name = NULL,
+        vtab_column(const std::string name = "",
                     int type = SQLITE3_TEXT,
                     const char *collator = NULL,
                     bool hidden = false)
             : vc_name(name), vc_type(type), vc_collator(collator), vc_hidden(hidden) { };
 
-        const char *vc_name;
+        std::string vc_name;
         int         vc_type;
         const char *vc_collator;
         bool vc_hidden;

src/pcrepp.hh

@@ -380,12 +380,28 @@ public:
     int name_index(const char *name) const {
         int retval = pcre_get_stringnumber(this->p_code, name);
 
-        if (retval == PCRE_ERROR_NOSUBSTRING)
+        if (retval == PCRE_ERROR_NOSUBSTRING) {
             return retval;
+        }
 
         return retval - 1;
     };
 
+    const char *name_for_capture(int index) {
+        for (pcre_named_capture::iterator iter = this->named_begin();
+             iter != this->named_end();
+             ++iter) {
+            if (iter->index() == index) {
+                return iter->pnc_name;
+            }
+        }
+        return "";
+    };
+
+    int get_capture_count() const {
+        return this->p_capture_count;
+    };
+
     bool match(pcre_context &pc, pcre_input &pi, int options = 0) const
     {
         int         length, startoffset, filtered_options = options;
@@ -511,6 +527,10 @@ private:
             pcre_assign_jit_stack(extra, NULL, jit_stack());
 #endif
         }
+        pcre_fullinfo(this->p_code,
+                      this->p_code_extra,
+                      PCRE_INFO_CAPTURECOUNT,
+                      &this->p_capture_count);
         pcre_fullinfo(this->p_code,
                       this->p_code_extra,
                       PCRE_INFO_NAMECOUNT,
@@ -527,6 +547,7 @@ private:
 
     pcre *p_code;
     auto_mem<pcre_extra> p_code_extra;
+    int p_capture_count;
     int p_named_count;
     int p_name_len;
     pcre_named_capture *p_named_entries;

src/readline_callbacks.cc

@@ -34,6 +34,7 @@
 #include "plain_text_source.hh"
 #include "command_executor.hh"
 #include "readline_curses.hh"
+#include "log_search_table.hh"
 
 using namespace std;
 

test/test_sql.sh

@@ -519,3 +519,53 @@ check_output "multiline data is not right?" <<EOF
     }
 ]
 EOF
+
+
+run_test ${lnav_test} -n \
+    -c ":create-search-table search_test1 (\w+), World!" \
+    -c ";select col_0 from search_test1" \
+    -c ":write-csv-to -" \
+    ${test_dir}/logfile_multiline.0
+
+check_output "create-search-table is not working?" <<EOF
+col_0
+Hello
+Goodbye
+EOF
+
+run_test ${lnav_test} -n \
+    -c ":create-search-table search_test1 (?<word>\w+), World!" \
+    -c ";select word from search_test1" \
+    -c ":write-csv-to -" \
+    ${test_dir}/logfile_multiline.0
+
+check_output "create-search-table is not working?" <<EOF
+word
+Hello
+Goodbye
+EOF
+
+run_test ${lnav_test} -n \
+    -c ":delete-search-table search_test1" \
+    ${test_dir}/logfile_multiline.0
+
+check_error_output "able to delete unknown table?" <<EOF
+error: unknown search table -- search_test1
+EOF
+
+run_test ${lnav_test} -n \
+    -c ":create-logline-table search_test1" \
+    -c ":delete-search-table search_test1" \
+    ${test_dir}/logfile_multiline.0
+
+check_error_output "able to delete logline table?" <<EOF
+error: unknown search table -- search_test1
+EOF
+
+run_test ${lnav_test} -n \
+    -c ":create-search-table search_test1 bad(" \
+    ${test_dir}/logfile_multiline.0
+
+check_error_output "able to create table with a bad regex?" <<EOF
+error: unable to compile regex -- bad(
+EOF