[tidy] use anon namespaces and some other stuff

NEWS.md

@@ -122,6 +122,7 @@ Features:
 * Added a `log_msg_values` column to the `all_logs` SQL
   table that contains a JSON object with the top 5 values
   for the fields extracted from the log message.
+* Added Nextcloud log format from Adam Monsen.
 
 Bug Fixes:
 * Binary data piped into stdin should now be treated the same

src/curl_looper.cc

@@ -108,6 +108,8 @@ curl_url_strerror(CURLUcode error)
 }
 #    endif
 
+namespace {
+
 struct curl_request_eq {
     explicit curl_request_eq(const std::string& name) : cre_name(name){};
 
@@ -125,6 +127,8 @@ struct curl_request_eq {
     const std::string& cre_name;
 };
 
+}  // namespace
+
 int
 curl_request::debug_cb(
     CURL* handle, curl_infotype type, char* data, size_t size, void* userp)

src/data_scanner_re.re

@@ -202,13 +202,37 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
            return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
        }
        <init, bol> "/*" ([^\x00*]|"*"+[^\x00/])* "*"+ "/" {
-           RET(DT_COMMENT);
+           CAPTURE(DT_COMMENT);
+           if (tf == text_format_t::TF_DIFF) {
+               auto sf = this->to_string_fragment(cap_all);
+               auto split_res = sf.split_when(string_fragment::tag1{'\n'});
+               cap_all.c_end = split_res.first.sf_end;
+               cap_inner.c_end = split_res.first.sf_end;
+               this->ds_next_offset = cap_all.c_end;
+           }
+           return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
        }
        <init, bol> "<!--" ([^\x00*]|"-"+[^\x00>])* "-"{2,} ">" {
-           RET(DT_COMMENT);
+           CAPTURE(DT_COMMENT);
+           if (tf == text_format_t::TF_DIFF) {
+               auto sf = this->to_string_fragment(cap_all);
+               auto split_res = sf.split_when(string_fragment::tag1{'\n'});
+               cap_all.c_end = split_res.first.sf_end;
+               cap_inner.c_end = split_res.first.sf_end;
+               this->ds_next_offset = cap_all.c_end;
+           }
+           return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
        }
        <init, bol> "#[" "="* "[" ([^\x00\]]|"]" [^\x00=\]])* "]" "="* "]" {
-           RET(DT_COMMENT);
+           CAPTURE(DT_COMMENT);
+           if (tf == text_format_t::TF_DIFF) {
+               auto sf = this->to_string_fragment(cap_all);
+               auto split_res = sf.split_when(string_fragment::tag1{'\n'});
+               cap_all.c_end = split_res.first.sf_end;
+               cap_inner.c_end = split_res.first.sf_end;
+               this->ds_next_offset = cap_all.c_end;
+           }
+           return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
        }
 
        <init, bol> ("f"|"u"|"r")?"'"('\\'[^\x00]|"''"|[^\x00\x16\x1b\n'\\])*"'"/[^sS] {

src/environ_vtab.cc

@@ -46,6 +46,8 @@ CREATE TABLE environ (
 );
 )";
 
+namespace {
+
 struct env_vtab {
     sqlite3_vtab base;
     sqlite3* db;
@@ -316,6 +318,8 @@ static sqlite3_module vtab_module = {
     nullptr, /* xFindFunction - function overloading */
 };
 
+}  // namespace
+
 int
 register_environ_vtab(sqlite3* db)
 {

src/file_vtab.cc

@@ -42,6 +42,8 @@
 #include "vtab_module.hh"
 #include "vtab_module_json.hh"
 
+namespace {
+
 struct lnav_file : public tvt_iterator_cursor<lnav_file> {
     using iterator = std::vector<std::shared_ptr<logfile>>::iterator;
 
@@ -379,3 +381,5 @@ static auto file_binder
 
 static auto file_meta_binder = injector::bind_multiple<vtab_module_base>()
                                    .add<injectable_lnav_file_metadata>();
+
+}  // namespace

src/formats/esx_syslog_log.json

@@ -83,6 +83,9 @@
             },
             {
                 "line": "2022-05-17T07:39:38.357Z In(9) watchdog-vobd[1001390409]: Executing '/usr/lib/vmware/vob/bin/vobd ++securitydom=vobdDom'"
+            },
+            {
+                "line": "2023-11-07T19:17:28.030Z In(14) settingsd[2099680]: [Ticket] Deleted ticket /var/run/vmware/tickets/vmtck-31182534-c078-88"
             }
         ]
     }

src/fstat_vtab.cc

@@ -46,6 +46,8 @@
 #include "sql_util.hh"
 #include "vtab_module.hh"
 
+namespace {
+
 enum {
     FSTAT_COL_PARENT,
     FSTAT_COL_NAME,
@@ -470,6 +472,8 @@ rcFilter(sqlite3_vtab_cursor* pVtabCursor,
     return SQLITE_OK;
 }
 
+}  // namespace
+
 int
 register_fstat_vtab(sqlite3* db)
 {

src/json-extension-functions.cc

@@ -45,6 +45,8 @@
 
 #define JSON_SUBTYPE 74 /* Ascii for "J" */
 
+namespace {
+
 class sql_json_op : public json_op {
 public:
     explicit sql_json_op(json_ptr& ptr) : json_op(ptr){};
@@ -750,6 +752,8 @@ sql_json_group_array_final(sqlite3_context* context)
     }
 }
 
+}  // namespace
+
 int
 json_extension_functions(struct FuncDef** basic_funcs,
                          struct FuncDefAgg** agg_funcs)

src/lnav.cc

@@ -882,9 +882,7 @@ handle_key(int ch)
         default: {
             switch (lnav_data.ld_mode) {
                 case ln_mode_t::PAGING:
-                    if (ch == KEY_ENTER || ch == '\n' || ch == '\r'
+                    if (ch == '`') {
-                        || ch == '`')
-                    {
                         breadcrumb_view.focus();
                         lnav_data.ld_mode = ln_mode_t::BREADCRUMBS;
                         return true;

src/regexp_vtab.cc

@@ -45,6 +45,8 @@
 #include "yajlpp/yajlpp.hh"
 #include "yajlpp/yajlpp_def.hh"
 
+namespace {
+
 enum {
     RC_COL_MATCH_INDEX,
     RC_COL_INDEX,
@@ -549,6 +551,8 @@ rcjFilter(sqlite3_vtab_cursor* pVtabCursor,
     return SQLITE_OK;
 }
 
+}  // namespace
+
 int
 register_regexp_vtab(sqlite3* db)
 {

src/static_file_vtab.cc

@@ -41,14 +41,7 @@
 #include "lnav.hh"
 #include "vtab_module.hh"
 
-const char* const STATIC_FILE_CREATE_STMT = R"(
+namespace {
--- Access static files in the lnav configuration directories
-CREATE TABLE lnav_static_files (
-    name TEXT PRIMARY KEY,
-    filepath TEXT,
-    content BLOB HIDDEN
-);
-)";
 
 struct static_file_vtab {
     sqlite3_vtab base;
@@ -307,6 +300,17 @@ static sqlite3_module static_file_vtab_module = {
     nullptr, /* xFindFunction - function overloading */
 };
 
+}  // namespace
+
+const char* const STATIC_FILE_CREATE_STMT = R"(
+-- Access static files in the lnav configuration directories
+CREATE TABLE lnav_static_files (
+    name TEXT PRIMARY KEY,
+    filepath TEXT,
+    content BLOB HIDDEN
+);
+)";
+
 int
 register_static_file_vtab(sqlite3* db)
 {

src/string-extension-functions.cc

@@ -48,6 +48,35 @@
 
 using namespace mapbox;
 
+enum class encode_algo {
+    base64,
+    hex,
+    uri,
+};
+
+template<>
+struct from_sqlite<encode_algo> {
+    inline encode_algo operator()(int argc, sqlite3_value** val, int argi)
+    {
+        const char* algo_name = (const char*) sqlite3_value_text(val[argi]);
+
+        if (strcasecmp(algo_name, "base64") == 0) {
+            return encode_algo::base64;
+        }
+        if (strcasecmp(algo_name, "hex") == 0) {
+            return encode_algo::hex;
+        }
+        if (strcasecmp(algo_name, "uri") == 0) {
+            return encode_algo::uri;
+        }
+
+        throw from_sqlite_conversion_error("value of 'base64', 'hex', or 'uri'",
+                                           argi);
+    }
+};
+
+namespace {
+
 struct cache_entry {
     std::shared_ptr<lnav::pcre2pp::code> re2;
     std::shared_ptr<column_namer> cn{
@@ -176,24 +205,7 @@ regexp_match(string_fragment re, string_fragment str)
 #endif
 }
 
-json_string
+static json_string
-extract(const char* str)
-{
-    data_scanner ds(str);
-    data_parser dp(&ds);
-
-    dp.parse();
-    // dp.print(stderr, dp.dp_pairs);
-
-    yajlpp_gen gen;
-    yajl_gen_config(gen, yajl_gen_beautify, false);
-
-    elements_to_json(gen, dp, &dp.dp_pairs);
-
-    return json_string(gen);
-}
-
-json_string
 logfmt2json(string_fragment line)
 {
     logfmt::parser p(line);
@@ -448,33 +460,6 @@ sql_gzip(sqlite3_value* val)
     return nonstd::nullopt;
 }
 
-enum class encode_algo {
-    base64,
-    hex,
-    uri,
-};
-
-template<>
-struct from_sqlite<encode_algo> {
-    inline encode_algo operator()(int argc, sqlite3_value** val, int argi)
-    {
-        const char* algo_name = (const char*) sqlite3_value_text(val[argi]);
-
-        if (strcasecmp(algo_name, "base64") == 0) {
-            return encode_algo::base64;
-        }
-        if (strcasecmp(algo_name, "hex") == 0) {
-            return encode_algo::hex;
-        }
-        if (strcasecmp(algo_name, "uri") == 0) {
-            return encode_algo::uri;
-        }
-
-        throw from_sqlite_conversion_error("value of 'base64', 'hex', or 'uri'",
-                                           argi);
-    }
-};
-
 #if defined(HAVE_LIBCURL)
 static CURL*
 get_curl_easy()
@@ -875,6 +860,25 @@ sql_unparse_url(string_fragment in)
     return retval;
 }
 
+}  // namespace
+
+json_string
+extract(const char* str)
+{
+    data_scanner ds(str);
+    data_parser dp(&ds);
+
+    dp.parse();
+    // dp.print(stderr, dp.dp_pairs);
+
+    yajlpp_gen gen;
+    yajl_gen_config(gen, yajl_gen_beautify, false);
+
+    elements_to_json(gen, dp, &dp.dp_pairs);
+
+    return json_string(gen);
+}
+
 int
 string_extension_functions(struct FuncDef** basic_funcs,
                            struct FuncDefAgg** agg_funcs)

src/text_format.cc

@@ -68,32 +68,38 @@ detect_text_format(string_fragment sf,
     static const auto MD_EXT = ghc::filesystem::path(".md");
     static const auto MARKDOWN_EXT = ghc::filesystem::path(".markdown");
 
+    static const auto DIFF_MATCHERS = lnav::pcre2pp::code::from_const(
+        R"(^--- .*\n\+\+\+ .*\n)", PCRE2_MULTILINE);
+
     static const auto MAN_MATCHERS = lnav::pcre2pp::code::from_const(
         R"(^[A-Za-z][A-Za-z\-_\+0-9]+\(\d\)\s+)", PCRE2_MULTILINE);
 
-    // XXX This is a pretty crude way of detecting format...
+    // XXX This is a pretty crude way of
+    // detecting format...
     static const auto PYTHON_MATCHERS = lnav::pcre2pp::code::from_const(
         "(?:"
-        "^\\s*def\\s+\\w+\\([^)]*\\):[^\\n]*$|"
+        "^\\s*def\\s+\\w+\\([^)]*\\):"
+        "[^\\n]*$|"
         "^\\s*try:[^\\n]*$"
         ")",
         PCRE2_MULTILINE);
 
-    static const auto RUST_MATCHERS
+    static const auto RUST_MATCHERS = lnav::pcre2pp::code::from_const(
-        = lnav::pcre2pp::code::from_const(R"(
+        R"(
 (?:
 ^\s*use\s+[\w+:\{\}]+;$|
 ^\s*(?:pub enum|pub const|(?:pub )?fn)\s+\w+.*$|
 ^\s*impl\s+\w+.*$
 )
 )",
-                                          PCRE2_MULTILINE);
+        PCRE2_MULTILINE);
 
     static const auto JAVA_MATCHERS = lnav::pcre2pp::code::from_const(
         "(?:"
         "^package\\s+|"
         "^import\\s+|"
-        "^\\s*(?:public)?\\s*class\\s*(\\w+\\s+)*\\s*{"
+        "^\\s*(?:public)?\\s*"
+        "class\\s*(\\w+\\s+)*\\s*{"
         ")",
         PCRE2_MULTILINE);
 
@@ -101,15 +107,18 @@ detect_text_format(string_fragment sf,
         "(?:"
         "^#\\s*include\\s+|"
         "^#\\s*define\\s+|"
-        "^\\s*if\\s+\\([^)]+\\)[^\\n]*$|"
+        "^\\s*if\\s+\\([^)]+\\)[^\\n]"
-        "^\\s*(?:\\w+\\s+)*class \\w+ {"
+        "*$|"
+        "^\\s*(?:\\w+\\s+)*class "
+        "\\w+ {"
         ")",
         PCRE2_MULTILINE);
 
     static const auto SQL_MATCHERS = lnav::pcre2pp::code::from_const(
         "(?:"
         "select\\s+.+\\s+from\\s+|"
-        "insert\\s+into\\s+.+\\s+values"
+        "insert\\s+into\\s+.+\\s+"
+        "values"
         ")",
         PCRE2_MULTILINE | PCRE2_CASELESS);
 
@@ -173,6 +182,10 @@ detect_text_format(string_fragment sf,
         }
     }
 
+    if (DIFF_MATCHERS.find_in(sf).ignore_error()) {
+        return text_format_t::TF_DIFF;
+    }
+
     if (MAN_MATCHERS.find_in(sf).ignore_error()) {
         return text_format_t::TF_MAN;
     }

src/text_format.hh

@@ -56,6 +56,7 @@ enum class text_format_t {
     TF_XML,
     TF_YAML,
     TF_TOML,
+    TF_DIFF,
 };
 
 namespace fmt {
@@ -111,6 +112,9 @@ struct formatter<text_format_t> : formatter<string_view> {
             case text_format_t::TF_TOML:
                 name = "application/toml";
                 break;
+            case text_format_t::TF_DIFF:
+                name = "text/x-diff";
+                break;
         }
         return formatter<string_view>::format(name, ctx);
     }

src/views_vtab.cc

@@ -122,6 +122,8 @@ struct from_sqlite<std::shared_ptr<lnav::pcre2pp::code>> {
     }
 };
 
+namespace {
+
 static const typed_json_path_container<breadcrumb::possibility>
     breadcrumb_possibility_handlers = {
         yajlpp::property_handler("display_value")
@@ -1274,11 +1276,6 @@ CREATE TABLE lnav_view_files (
     }
 };
 
-static const char* CREATE_FILTER_VIEW = R"(
-CREATE VIEW lnav_view_filters_and_stats AS
-  SELECT * FROM lnav_view_filters LEFT NATURAL JOIN lnav_view_filter_stats
-)";
-
 static auto a = injector::bind_multiple<vtab_module_base>()
                     .add<vtab_module<lnav_views>>()
                     .add<vtab_module<lnav_view_stack>>()
@@ -1286,9 +1283,16 @@ static auto a = injector::bind_multiple<vtab_module_base>()
                     .add<vtab_module<tvt_no_update<lnav_view_filter_stats>>>()
                     .add<vtab_module<lnav_view_files>>();
 
+}  // namespace
+
 int
 register_views_vtab(sqlite3* db)
 {
+    static const char* CREATE_FILTER_VIEW = R"(
+CREATE VIEW lnav_view_filters_and_stats AS
+  SELECT * FROM lnav_view_filters LEFT NATURAL JOIN lnav_view_filter_stats
+)";
+
     auto_mem<char> errmsg(sqlite3_free);
     if (sqlite3_exec(db, CREATE_FILTER_VIEW, nullptr, nullptr, errmsg.out())
         != SQLITE_OK)

test/Makefile.am

@@ -335,6 +335,7 @@ dist_noinst_DATA = \
 	logfile_logfmt.0 \
 	logfile_multiline.0 \
 	logfile_nested_json.json \
+	logfile_nextcloud.0 \
 	logfile_openam.0 \
 	logfile_plain.0 \
 	logfile_pretty.0 \

test/expected/expected.am

@@ -320,6 +320,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_json_format.sh_84a71e94dc34661a70bb9015b67ba00e93e9cfb5.out \
     $(srcdir)/%reldir%/test_json_format.sh_85d03b1b41a7f819af135d2521a8f2c59418e907.err \
     $(srcdir)/%reldir%/test_json_format.sh_85d03b1b41a7f819af135d2521a8f2c59418e907.out \
+    $(srcdir)/%reldir%/test_json_format.sh_895283eeb4c10e9c1702fafd13723c8085944f88.err \
+    $(srcdir)/%reldir%/test_json_format.sh_895283eeb4c10e9c1702fafd13723c8085944f88.out \
     $(srcdir)/%reldir%/test_json_format.sh_8f2ebcd319afc7966ef11e31f9dd646bf6f001dd.err \
     $(srcdir)/%reldir%/test_json_format.sh_8f2ebcd319afc7966ef11e31f9dd646bf6f001dd.out \
     $(srcdir)/%reldir%/test_json_format.sh_952297a90e312d2184fe3e4df795ddc731b096c9.err \

test/expected/test_json_format.sh_895283eeb4c10e9c1702fafd13723c8085944f88.out

@@ -0,0 +1,5 @@
+2023-10-01T02:39:09+00:00 twv 10.0.0.1        INFO files_antivirus Tried to scan non file
+2023-10-01T02:39:09+00:00 twv                 INFO files_antivirus Tried to scan non file
+2023-10-01T02:50:03+00:00 pZp                 INFO fulltextsearch_elasticsearch Request: PUT http://elasticsearch:9200/nextcloud/_doc/files%3A1780281
+2023-10-16T01:47:44+00:00 WO0 192.168.1.1     INFO memories Memories: Updated item successfully
+2023-10-16T01:50:18+00:00 Sbl 192.168.1.1     INFO files_versions Mark to expire /Documents/phat.txt next version should be 1697407738 or smaller. (prevTimestamp: 1697411338; step: 3600

test/test_json_format.sh

@@ -155,3 +155,6 @@ run_cap_test ${lnav_test} -n \
 run_cap_test ${lnav_test} -n \
     -c ':show-fields RayID' \
     ${test_dir}/logfile_cloudflare.json
+
+run_cap_test ${lnav_test} -n \
+    ${test_dir}/logfile_nextcloud.0

test/test_sql.sh

@@ -742,8 +742,8 @@ check_output "schema view is not working" <<EOF
 ATTACH DATABASE '' AS 'main';
 CREATE VIRTUAL TABLE environ USING environ_vtab_impl();
 CREATE VIRTUAL TABLE lnav_static_files USING lnav_static_file_vtab_impl();
-CREATE VIRTUAL TABLE lnav_views USING lnav_views_impl();
 CREATE VIRTUAL TABLE lnav_view_filter_stats USING lnav_view_filter_stats_impl();
+CREATE VIRTUAL TABLE lnav_views USING lnav_views_impl();
 CREATE VIRTUAL TABLE lnav_view_files USING lnav_view_files_impl();
 CREATE VIRTUAL TABLE lnav_view_stack USING lnav_view_stack_impl();
 CREATE VIRTUAL TABLE lnav_view_filters USING lnav_view_filters_impl();