[all_logs] add a .msgformats SQL command

NEWS

@@ -23,7 +23,9 @@ lnav v0.8.0:
      * The '-C' option will now try to check any specified log files to
        make sure the format(s) match all of the lines.
      * Added an "all_logs" SQLite table that contains the message format
-       extracted from each log line.
+       extracted from each log line.  Also added a ';.msgformat' SQL command
+       that executes a query that returns the counts for each format and the
+       first line where the format was seen.
      * Added an "lnav_views" SQLite table that can be used to query and
        change the lnav view state.
      * When typing in a command, the status bar will display a short

src/command_executor.cc

@@ -38,6 +38,10 @@
 
 using namespace std;
 
+static const string MSG_FORMAT_STMT =
+        "SELECT count(*) as total, min(log_line) as log_line, log_msg_format "
+                "FROM all_logs GROUP BY log_msg_format ORDER BY total desc";
+
 static int sql_progress(const struct log_cursor &lc)
 {
     static sig_atomic_t sql_counter = 0;
@@ -115,6 +119,9 @@ string execute_sql(string sql, string &alt_msg)
         lnav_data.ld_mode = LNM_PAGING;
         return "";
     }
+    else if (stmt_str == ".msgformats") {
+        stmt_str = MSG_FORMAT_STMT;
+    }
 
     hs.clear();
     hs.get_displayed_buckets().clear();

src/data_scanner_re.re

@@ -142,8 +142,8 @@ bool data_scanner::tokenize2(pcre_context &pc, data_token_t &token_out)
            cap[1].c_end -= 1;
            return true;
        }
-       [a-zA-Z0-9]+"://"[^\x00\r\n\t '"\[\](){}]+[/a-zA-Z0-9\-=&] { RET(DT_URL); }
-       ("/"|"./"|"../")[a-zA-Z0-9_\.\-_\~/]* { RET(DT_PATH); }
+       [a-zA-Z0-9]+"://"[^\x00\r\n\t '"\[\](){}]+[/a-zA-Z0-9\-=&?%] { RET(DT_URL); }
+       ("/"|"./"|"../")[a-zA-Z0-9_\.\-\~/!@#$%^&*()]* { RET(DT_PATH); }
        (SPACE|NUM)NUM":"NUM{2}/[^:] { RET(DT_TIME); }
        (SPACE|NUM)NUM":"NUM{2}":"NUM{2}("."NUM{3,6})?/[^:] { RET(DT_TIME); }
        [0-9a-fA-F][0-9a-fA-F](":"[0-9a-fA-F][0-9a-fA-F])+ {
@@ -205,7 +205,7 @@ bool data_scanner::tokenize2(pcre_context &pc, data_token_t &token_out)
        ("true"|"True"|"TRUE"|"false"|"False"|"FALSE"|"None"|"null"|"NULL") { RET(DT_CONSTANT); }
        ("re-")?[a-zA-Z][a-z']+/([\r\n\t \(\)!\*:;'\"\?,]|[\.\!,\?]SPACE|EOF) { RET(DT_WORD); }
 
-       [^\x00"; \t\r\n:=,\(\)\{\}\[\]\+#!@%\^&\*'\?<>\~`\|\\]+("::"[^\x00"; \r\n\t:=,\(\)\{\}\[\]\+#!@%\^&\*'\?<>\~`\|\\]+)* {
+       [^\x00"; \t\r\n:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\\]+("::"[^\x00"; \r\n\t:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\\]+)* {
            RET(DT_SYMBOL);
        }
 

src/lnav.cc

@@ -246,6 +246,7 @@ bool setup_logline_table()
 
     static const char *commands[] = {
         ".schema",
+        ".msgformats",
 
         NULL
     };