[cmd] add summarize command

src/bottom_status_source.hh

@@ -77,6 +77,7 @@ public:
         this->bss_fields[BSF_ERRORS].set_width(10);
         this->bss_fields[BSF_ERRORS].set_role(view_colors::VCR_ALERT_STATUS);
         this->bss_fields[BSF_FILTERED].set_width(20);
+        this->bss_fields[BSF_FILTERED].set_role(view_colors::VCR_BOLD_STATUS);
         this->bss_fields[BSF_LOADING].set_width(13);
         this->bss_fields[BSF_LOADING].set_cylon(true);
         this->bss_fields[BSF_LOADING].right_justify(true);

src/data_parser.cc

@@ -53,6 +53,10 @@ data_format_state_t dfs_prefix_next(data_format_state_t state,
         case DT_HEX_NUMBER:
         case DT_NUMBER:
         case DT_WHITE:
+        case DT_LSQUARE:
+        case DT_RSQUARE:
+        case DT_LANGLE:
+        case DT_RANGLE:
             break;
         default:
             retval = DFS_ERROR;

src/extension-functions.c

@@ -1781,6 +1781,7 @@ int RegisterExtensionFunctions(sqlite3 *db){
     void (*xFinalize)(sqlite3_context*);
   } aAggs[] = {
     { "stdev",            1, 0, 0, varianceStep, stdevFinalize  },
+    { "stddev",            1, 0, 0, varianceStep, stdevFinalize  },
     { "variance",         1, 0, 0, varianceStep, varianceFinalize  },
     { "mode",             1, 0, 0, modeStep,     modeFinalize  },
     { "median",           1, 0, 0, modeStep,     medianFinalize  },

src/histview_curses.hh

@@ -32,6 +32,7 @@
 
 #include <map>
 #include <string>
+#include <vector>
 
 #include "strong_int.hh"
 #include "listview_curses.hh"
@@ -92,9 +93,9 @@ public:
     void reload_data(void);
 
 private:
-    typedef vector<bucket_count_t> buckets_t;
+    typedef std::vector<bucket_count_t> buckets_t;
 
-    map<bucket_group_t, buckets_t> hv_groups;
+    std::map<bucket_group_t, buckets_t> hv_groups;
     int hv_bucket_size; /* hours */
     int hv_group_size;  /* days */
     hist_data_source * hv_data_source;

src/lnav.cc

@@ -582,6 +582,26 @@ static void walk_sqlite_metadata(sqlite3 *db)
                 lnav_data.ld_db_key_names.end());
 }
 
+bool setup_logline_table()
+{
+    textview_curses &log_view = lnav_data.ld_views[LNV_LOG];
+    bool retval = false;
+
+    if (log_view.get_inner_height()) {
+        vis_line_t     vl = log_view.get_top();
+        content_line_t cl = lnav_data.ld_log_source.at(vl);
+
+        lnav_data.ld_vtab_manager->unregister_vtab("logline");
+        lnav_data.ld_vtab_manager->register_vtab(new log_data_table(cl));
+
+        retval = true;
+    }
+
+    walk_sqlite_metadata(lnav_data.ld_db.in());
+
+    return retval;
+}
+
 /**
  * Observer for loading progress that updates the bottom status bar.
  */
@@ -1546,6 +1566,39 @@ static void handle_paging_key(int ch)
         break;
 
     case ':':
+        if (lnav_data.ld_views[LNV_LOG].get_inner_height() > 0) {
+            logfile_sub_source &lss = lnav_data.ld_log_source;
+            textview_curses &log_view = lnav_data.ld_views[LNV_LOG];
+            content_line_t    cl   = lss.at(log_view.get_top());
+            logfile *         lf   = lss.find(cl);
+            std::string       line = lf->read_line(lf->begin() + cl);
+            struct line_range body;
+            string_attrs_t    sa;
+            std::vector<logline_value> line_values;
+
+            lf->get_format()->annotate(line, sa, line_values);
+
+            body = find_string_attr_range(sa, "body");
+            if (body.lr_end != -1) {
+                line = line.substr(body.lr_start);
+            }
+
+            data_scanner ds(line);
+            data_parser dp(&ds);
+            dp.parse();
+
+            column_namer namer;
+
+            lnav_data.ld_rl_view->clear_possibilities(LNM_COMMAND, "colname");
+            for (data_parser::element_list_t::iterator iter = dp.dp_pairs.begin();
+                 iter != dp.dp_pairs.end();
+                 ++iter) {
+                std::string colname = dp.get_element_string(iter->e_sub_elements->front());
+
+                colname            = namer.add_column(colname);
+                lnav_data.ld_rl_view->add_possibility(LNM_COMMAND, "colname", colname);
+            }
+        }
         lnav_data.ld_mode = LNM_COMMAND;
         lnav_data.ld_rl_view->focus(LNM_COMMAND, ":");
         break;
@@ -1563,17 +1616,9 @@ static void handle_paging_key(int ch)
             textview_curses &log_view = lnav_data.ld_views[LNV_LOG];
 
             lnav_data.ld_mode = LNM_SQL;
-            walk_sqlite_metadata(lnav_data.ld_db.in());
+            setup_logline_table();
             lnav_data.ld_rl_view->focus(LNM_SQL, ";");
 
-            if (log_view.get_inner_height()) {
-                vis_line_t     vl = log_view.get_top();
-                content_line_t cl = lnav_data.ld_log_source.at(vl);
-
-                lnav_data.ld_vtab_manager->unregister_vtab("logline");
-                lnav_data.ld_vtab_manager->register_vtab(new log_data_table(cl));
-            }
-
             lnav_data.ld_bottom_source.update_loading(0, 0);
             lnav_data.ld_status[LNS_BOTTOM].do_update();
 
@@ -1838,7 +1883,7 @@ static void execute_file(string path)
     }
 }
 
-static int sql_callback(sqlite3_stmt *stmt)
+int sql_callback(sqlite3_stmt *stmt)
 {
     logfile_sub_source &lss = lnav_data.ld_log_source;
     db_label_source &   dls = lnav_data.ld_db_rows;
@@ -2045,7 +2090,7 @@ static void rl_callback(void *dummy, readline_curses *rc)
         lnav_data.ld_bottom_source.grep_error("");
         hs.clear();
         dls.clear();
-        retcode = sqlite3_prepare_v2(lnav_data.ld_db,
+        retcode = sqlite3_prepare_v2(lnav_data.ld_db.in(),
                                      rc->get_value().c_str(),
                                      -1,
                                      stmt.out(),

src/lnav.hh

@@ -186,4 +186,7 @@ bool toggle_view(textview_curses *toggle_tc);
 
 std::string execute_command(std::string cmdline);
 
+bool setup_logline_table();
+int sql_callback(sqlite3_stmt *stmt);
+
 #endif

src/lnav_commands.cc

@@ -35,6 +35,8 @@
 #include <vector>
 #include <fstream>
 
+#include <pcrecpp.h>
+
 #include "lnav.hh"
 #include "lnav_util.hh"
 #include "auto_mem.hh"
@@ -581,6 +583,187 @@ static string com_session(string cmdline, vector<string> &args)
     return retval;
 }
 
+static string com_summarize(string cmdline, vector<string> &args)
+{
+    static pcrecpp::RE db_column_converter("\"");
+
+    string retval = "";
+
+    if (args.size() == 0) {
+        args.push_back("colname");
+        return retval;
+    }
+    else if (!setup_logline_table()) {
+        retval = "error: no log data available";
+    }
+    else {
+        auto_mem<char, sqlite3_free> query_frag;
+        std::vector<string> other_columns;
+        std::vector<string> num_columns;
+        string query;
+
+        for (size_t lpc = 1; lpc < args.size(); lpc++) {
+            string quoted_name = args[lpc];
+            const char *datatype;
+            int rc;
+
+            rc = sqlite3_table_column_metadata(
+                lnav_data.ld_db.in(),
+                "main",
+                "logline",
+                args[lpc].c_str(),
+                &datatype,
+                NULL,
+                NULL,
+                NULL,
+                NULL);
+            if (rc != SQLITE_OK) {
+                return "error: bad column name -- " + args[lpc];
+            }
+
+            db_column_converter.GlobalReplace("\"\"", &quoted_name);
+
+            fprintf(stderr, "dt = %s\n", datatype);
+            if (strcasecmp(datatype, "float") == 0) {
+                num_columns.push_back(quoted_name);
+            }
+            else {
+                other_columns.push_back(quoted_name);
+            }
+        }
+
+        query = "SELECT";
+        for (std::vector<string>::iterator iter = other_columns.begin();
+             iter != other_columns.end();
+             ++iter) {
+            if (iter != other_columns.begin())
+                query += ",";
+            query_frag = sqlite3_mprintf(" \"%s\", count(*) as \"count_%s\"",
+                                         iter->c_str(),
+                                         iter->c_str());
+            query += query_frag;
+        }
+
+        if (!other_columns.empty() && !num_columns.empty())
+            query += ", ";
+
+        for (std::vector<string>::iterator iter = num_columns.begin();
+             iter != num_columns.end();
+             ++iter) {
+            if (iter != num_columns.begin())
+                query += ",";
+            query_frag = sqlite3_mprintf(" sum(\"%s\"), "
+                                         " min(\"%s\"), "
+                                         " avg(\"%s\"), "
+                                         " median(\"%s\"), "
+                                         " stddev(\"%s\"), "
+                                         " max(\"%s\") ",
+                                         iter->c_str(),
+                                         iter->c_str(),
+                                         iter->c_str(),
+                                         iter->c_str(),
+                                         iter->c_str(),
+                                         iter->c_str());
+            query += query_frag;
+        }
+
+        query += " FROM logline";
+
+        for (std::vector<string>::iterator iter = other_columns.begin();
+             iter != other_columns.end();
+             ++iter) {
+            if (iter == other_columns.begin())
+                query += " GROUP BY ";
+            else
+                query += ",";
+            query_frag = sqlite3_mprintf(" \"%s\"", iter->c_str());
+            query += query_frag;
+        }
+
+        for (std::vector<string>::iterator iter = other_columns.begin();
+             iter != other_columns.end();
+             ++iter) {
+            if (iter == other_columns.begin())
+                query += " ORDER BY ";
+            else
+                query += ",";
+            query_frag = sqlite3_mprintf(" \"count_%s\" desc, \"%s\" asc",
+                                         iter->c_str(),
+                                         iter->c_str());
+            query += query_frag;
+        }
+
+        fprintf(stderr, "query %s\n", query.c_str());
+
+        db_label_source &      dls = lnav_data.ld_db_rows;
+        hist_source &          hs  = lnav_data.ld_db_source;
+        auto_mem<sqlite3_stmt> stmt(sqlite3_finalize);
+        int retcode;
+
+        hs.clear();
+        dls.clear();
+        retcode = sqlite3_prepare_v2(lnav_data.ld_db.in(),
+                                     query.c_str(),
+                                     -1,
+                                     stmt.out(),
+                                     NULL);
+
+        if (retcode != SQLITE_OK) {
+            const char *errmsg = sqlite3_errmsg(lnav_data.ld_db);
+
+            retval = "error: " + string(errmsg);
+        }
+        else if (stmt == NULL) {
+            retval = "";
+        }
+        else {
+            bool done = false;
+
+            while (!done) {
+                retcode = sqlite3_step(stmt.in());
+
+                switch (retcode) {
+                case SQLITE_OK:
+                case SQLITE_DONE:
+                    done = true;
+                    break;
+
+                case SQLITE_ROW:
+                    sql_callback(stmt.in());
+                    break;
+
+                default:
+                {
+                    const char *errmsg;
+
+                    fprintf(stderr, "code %d\n", retcode);
+                    errmsg = sqlite3_errmsg(lnav_data.ld_db);
+                    retval = "error: " + string(errmsg);
+                    done = true;
+                }
+                break;
+                }
+            }
+
+            if (retcode == SQLITE_DONE) {
+                hs.analyze();
+                lnav_data.ld_views[LNV_LOG].reload_data();
+                lnav_data.ld_views[LNV_DB].reload_data();
+                lnav_data.ld_views[LNV_DB].set_left(0);
+
+                if (dls.dls_rows.size() > 0) {
+                    ensure_view(&lnav_data.ld_views[LNV_DB]);
+                }
+            }
+
+            lnav_data.ld_bottom_source.update_loading(0, 0);
+            lnav_data.ld_status[LNS_BOTTOM].do_update();
+        }
+    }
+
+    return retval;
+}
+
 static string com_add_test(string cmdline, vector<string> &args)
 {
     string retval = "";
@@ -641,6 +824,7 @@ void init_lnav_commands(readline_context::command_map_t &cmd_map)
     cmd_map["create-logline-table"] = com_create_logline_table;
     cmd_map["delete-logline-table"] = com_delete_logline_table;
     cmd_map["session"]        = com_session;
+    cmd_map["summarize"] = com_summarize;
 
     if (getenv("LNAV_SRC") != NULL) {
         cmd_map["add-test"] = com_add_test;

src/log_data_table.hh

@@ -175,7 +175,19 @@ public:
             std::string tmp = dp.get_element_string(
                 pair_iter->e_sub_elements->back());
 
-            values.push_back(logline_value("", tmp));
+            fprintf(stderr, "data %s\n", tmp.c_str());
+            switch(pair_iter->e_sub_elements->back().e_token) {
+                case DT_NUMBER: {
+                    double d = 0;
+
+                    sscanf(tmp.c_str(), "%lf", &d);
+                    values.push_back(logline_value("", d));
+                }
+                break;
+                default:
+                values.push_back(logline_value("", tmp));
+                break;
+            }
         }
     };
 

src/log_format.cc

@@ -105,7 +105,7 @@ const char *logline::level_names[LEVEL__MAX] = {
 
 static int strcasestr_i(const char *s1, const char *s2)
 {
-    return strcasestr(s1, s2) != NULL;
+    return strcasestr(s1, s2) == NULL;
 }
 
 logline::level_t logline::string2level(const char *levelstr, bool exact)
@@ -118,28 +118,28 @@ logline::level_t logline::string2level(const char *levelstr, bool exact)
     else
         cmpfunc = strcasestr_i;
 
-    if (strcasestr(levelstr, "TRACE") == 0) {
+    if (cmpfunc(levelstr, "TRACE") == 0) {
         retval = logline::LEVEL_TRACE;
     }
-    else if (strcasestr(levelstr, "VERBOSE") == 0) {
+    else if (cmpfunc(levelstr, "VERBOSE") == 0) {
         retval = logline::LEVEL_DEBUG;
     }
-    else if (strcasestr(levelstr, "DEBUG") == 0) {
+    else if (cmpfunc(levelstr, "DEBUG") == 0) {
         retval = logline::LEVEL_DEBUG;
     }
-    else if (strcasestr(levelstr, "INFO") == 0) {
+    else if (cmpfunc(levelstr, "INFO") == 0) {
         retval = logline::LEVEL_INFO;
     }
-    else if (strcasestr(levelstr, "WARNING") == 0) {
+    else if (cmpfunc(levelstr, "WARNING") == 0) {
         retval = logline::LEVEL_WARNING;
     }
-    else if (strcasestr(levelstr, "ERROR") == 0) {
+    else if (cmpfunc(levelstr, "ERROR") == 0) {
         retval = logline::LEVEL_ERROR;
     }
-    else if (strcasestr(levelstr, "CRITICAL") == 0) {
+    else if (cmpfunc(levelstr, "CRITICAL") == 0) {
         retval = logline::LEVEL_CRITICAL;
     }
-    else if (strcasestr(levelstr, "FATAL") == 0) {
+    else if (cmpfunc(levelstr, "FATAL") == 0) {
         retval = logline::LEVEL_FATAL;
     }
 

src/log_format_impls.cc

@@ -481,7 +481,7 @@ class generic_log_format : public log_format {
     static const char **get_log_formats()
     {
         static const char *log_fmt[] = {
-            "%63[0-9: ,.-]%63[^:]%n",
+            "%63[0-9TZ: ,.-]%63[^:]%n",
             "%63[a-zA-Z0-9:-+/.] [%*x %63[^\n]%n",
             "%63[a-zA-Z0-9:.,-] %63[^\n]%n",
             "%63[a-zA-Z0-9: .,-] [%*[^]]]%63[^:]%n",

src/view_curses.cc

@@ -204,6 +204,8 @@ view_colors::view_colors()
         COLOR_PAIR(VC_GREEN_ON_WHITE);
     this->vc_role_colors[VCR_ACTIVE_STATUS2] =
         COLOR_PAIR(VC_GREEN_ON_WHITE) | A_BOLD;
+    this->vc_role_colors[VCR_BOLD_STATUS] =
+        COLOR_PAIR(VC_BLACK_ON_WHITE) | A_BOLD;
 
     this->vc_role_colors[VCR_DIFF_DELETE]  = COLOR_PAIR(VC_RED);
     this->vc_role_colors[VCR_DIFF_ADD]     = COLOR_PAIR(VC_GREEN);

src/view_curses.hh

@@ -339,6 +339,7 @@ public:
         VCR_ALERT_STATUS,       /*< Alert status line text. */
         VCR_ACTIVE_STATUS,      /*< */
         VCR_ACTIVE_STATUS2,     /*< */
+        VCR_BOLD_STATUS,
 
         VCR_DIFF_DELETE,        /*< Deleted line in a diff. */
         VCR_DIFF_ADD,           /*< Added line in a diff. */