[vtab] allow UPDATEs to log_annotations

src/command_executor.cc

@@ -736,6 +736,7 @@ execute_any(exec_context& ec, const std::string& cmdline_with_mode)
             (lnav_data.ld_flags & LNF_HEADLESS || ec.ec_path_stack.size() > 1))
         {
             rescan_files();
+            wait_for_pipers();
             rebuild_indexes_repeatedly();
         }
     });
@@ -832,6 +833,7 @@ execute_init_commands(
                 }
 
                 rescan_files();
+                wait_for_pipers();
                 rebuild_indexes_repeatedly();
             }
             if (dls.dls_rows.size() > 1) {

src/lnav.cc

@@ -991,7 +991,7 @@ gather_pipers()
     }
 }
 
-static void
+void
 wait_for_pipers()
 {
     static const auto MAX_SLEEP_TIME = std::chrono::milliseconds(300);

src/lnav.hh

@@ -283,5 +283,6 @@ extern const ssize_t ZOOM_COUNT;
 
 bool setup_logline_table(exec_context& ec);
 void wait_for_children();
+void wait_for_pipers();
 
 #endif

src/log_vtab_impl.cc

@@ -1938,6 +1938,8 @@ vt_update(sqlite3_vtab* tab,
             = sqlite3_value_text(argv[2 + VT_COL_LOG_COMMENT]);
         const auto log_tags = from_sqlite<nonstd::optional<string_fragment>>()(
             argc, argv, 2 + VT_COL_LOG_TAGS);
+        const auto log_annos = from_sqlite<nonstd::optional<string_fragment>>()(
+            argc, argv, 2 + VT_COL_LOG_ANNOTATIONS);
         bookmark_metadata tmp_bm;
 
         if (log_tags) {
@@ -1969,10 +1971,23 @@ vt_update(sqlite3_vtab* tab,
                 return SQLITE_ERROR;
             }
         }
+        if (log_annos) {
+            static const intern_string_t SRC
+                = intern_string::lookup("log_annotations");
+
+            auto parse_res = logmsg_annotations_handlers.parser_for(SRC).of(
+                log_annos.value());
+            if (parse_res.isErr()) {
+                set_vtable_errmsg(tab, parse_res.unwrapErr()[0]);
+                return SQLITE_ERROR;
+            }
+
+            tmp_bm.bm_annotations = parse_res.unwrap();
+        }
 
         auto& bv = vt->tc->get_bookmarks()[&textview_curses::BM_META];
         bool has_meta = part_name != nullptr || log_comment != nullptr
-            || log_tags.has_value();
+            || log_tags.has_value() || log_annos.has_value();
 
         if (binary_search(bv.begin(), bv.end(), vrowid) && !has_meta) {
             vt->tc->set_user_mark(&textview_curses::BM_META, vrowid, false);
@@ -2006,6 +2021,11 @@ vt_update(sqlite3_vtab* tab,
             } else {
                 line_meta.bm_tags.clear();
             }
+            if (log_annos) {
+                line_meta.bm_annotations = std::move(tmp_bm.bm_annotations);
+            } else {
+                line_meta.bm_annotations.la_pairs.clear();
+            }
 
             vt->lss->set_line_meta_changed();
         }

src/session.export.cc

@@ -42,6 +42,7 @@ struct log_message_session_state {
     bool lmss_mark;
     nonstd::optional<std::string> lmss_comment;
     nonstd::optional<std::string> lmss_tags;
+    nonstd::optional<std::string> lmss_annotations;
     std::string lmss_hash;
 };
 
@@ -57,7 +58,8 @@ struct from_sqlite<log_message_session_state> {
             from_sqlite<bool>()(argc, argv, argi + 2),
             from_sqlite<nonstd::optional<std::string>>()(argc, argv, argi + 3),
             from_sqlite<nonstd::optional<std::string>>()(argc, argv, argi + 4),
-            from_sqlite<std::string>()(argc, argv, argi + 5),
+            from_sqlite<nonstd::optional<std::string>>()(argc, argv, argi + 5),
+            from_sqlite<std::string>()(argc, argv, argi + 6),
         };
     }
 };
@@ -178,9 +180,12 @@ export_to(FILE* file)
     static auto& lnav_db = injector::get<auto_sqlite3&>();
 
     static const char* BOOKMARK_QUERY = R"(
-SELECT log_time_msecs, log_format, log_mark, log_comment, log_tags, log_line_hash
+SELECT log_time_msecs, log_format, log_mark, log_comment, log_tags, log_annotations, log_line_hash
    FROM all_logs
-   WHERE log_mark = 1 OR log_comment IS NOT NULL OR log_tags IS NOT NULL
+   WHERE log_mark = 1 OR
+         log_comment IS NOT NULL OR
+         log_tags IS NOT NULL OR
+         log_annotations IS NOT NULL
 )";
 
     static const char* FILTER_QUERY = R"(
@@ -320,13 +325,15 @@ SELECT content_id, format, time_offset FROM lnav_file
                            FMT_STRING(";UPDATE all_logs "
                                       "SET log_mark = {}, "
                                       "log_comment = {}, "
-                                      "log_tags = {} "
+                                      "log_tags = {}, "
+                                      "log_annotations = {} "
                                       "WHERE log_time_msecs = {} AND "
                                       "log_format = {} AND "
                                       "log_line_hash = {}\n"),
                            lmss.lmss_mark ? "1" : "0",
                            sqlitepp::quote(lmss.lmss_comment).in(),
                            sqlitepp::quote(lmss.lmss_tags).in(),
+                           sqlitepp::quote(lmss.lmss_annotations).in(),
                            lmss.lmss_time_msecs,
                            sqlitepp::quote(lmss.lmss_format).in(),
                            sqlitepp::quote(lmss.lmss_hash).in());

test/expected/expected.am

@@ -374,14 +374,16 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_logfile.sh_e840b674cd65936a72bd64b1dac1524d16fe44c3.out \
     $(srcdir)/%reldir%/test_logfile.sh_f171f265d8d45a2707e8b9f53e938f574c614d25.err \
     $(srcdir)/%reldir%/test_logfile.sh_f171f265d8d45a2707e8b9f53e938f574c614d25.out \
-    $(srcdir)/%reldir%/test_meta.sh_039c1f127c087ec2c6a23a0ecec4df7992cbc8b4.err \
-    $(srcdir)/%reldir%/test_meta.sh_039c1f127c087ec2c6a23a0ecec4df7992cbc8b4.out \
+    $(srcdir)/%reldir%/test_meta.sh_0ae5840c0758ec95a397493ac3c3d4fabd397a34.err \
+    $(srcdir)/%reldir%/test_meta.sh_0ae5840c0758ec95a397493ac3c3d4fabd397a34.out \
     $(srcdir)/%reldir%/test_meta.sh_154047fb52e4831aabf7d36512247bad6a6a2cf7.err \
     $(srcdir)/%reldir%/test_meta.sh_154047fb52e4831aabf7d36512247bad6a6a2cf7.out \
     $(srcdir)/%reldir%/test_meta.sh_3c9b5940f7533c5fc3d4956a6efce50a9e7132d4.err \
     $(srcdir)/%reldir%/test_meta.sh_3c9b5940f7533c5fc3d4956a6efce50a9e7132d4.out \
     $(srcdir)/%reldir%/test_meta.sh_41f643bb4f720130625b042563e9591bee4ae588.err \
     $(srcdir)/%reldir%/test_meta.sh_41f643bb4f720130625b042563e9591bee4ae588.out \
+    $(srcdir)/%reldir%/test_meta.sh_4283bf9128e0396c4ff141faf68279d5a3d8ac63.err \
+    $(srcdir)/%reldir%/test_meta.sh_4283bf9128e0396c4ff141faf68279d5a3d8ac63.out \
     $(srcdir)/%reldir%/test_meta.sh_45ff39a3d0ac0ca0c95aaca14d043450cec1cedd.err \
     $(srcdir)/%reldir%/test_meta.sh_45ff39a3d0ac0ca0c95aaca14d043450cec1cedd.out \
     $(srcdir)/%reldir%/test_meta.sh_48e85ba0c0945a5085fb4ee255771406061a9c17.err \
@@ -398,6 +400,12 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_meta.sh_83ac877aa9d38b25945cf96d6326a2468187c40f.out \
     $(srcdir)/%reldir%/test_meta.sh_a7489c1f0e001adc732b7e2ab31bb30960fda078.err \
     $(srcdir)/%reldir%/test_meta.sh_a7489c1f0e001adc732b7e2ab31bb30960fda078.out \
+    $(srcdir)/%reldir%/test_meta.sh_aae427fe704f131f8466fbfc9e7967c2e874530a.err \
+    $(srcdir)/%reldir%/test_meta.sh_aae427fe704f131f8466fbfc9e7967c2e874530a.out \
+    $(srcdir)/%reldir%/test_meta.sh_ad2f396f332d14ed6e95936983bc7e4aeea0bfac.err \
+    $(srcdir)/%reldir%/test_meta.sh_ad2f396f332d14ed6e95936983bc7e4aeea0bfac.out \
+    $(srcdir)/%reldir%/test_meta.sh_b622c04c2bede5371bfa6f2627b0474049ecc5e9.err \
+    $(srcdir)/%reldir%/test_meta.sh_b622c04c2bede5371bfa6f2627b0474049ecc5e9.out \
     $(srcdir)/%reldir%/test_meta.sh_c063f96398650f130941bbbf4cf63c1244fdbee5.err \
     $(srcdir)/%reldir%/test_meta.sh_c063f96398650f130941bbbf4cf63c1244fdbee5.out \
     $(srcdir)/%reldir%/test_meta.sh_c75128169049bd88d5eaf8b84a7f617e5ae5d936.err \

test/expected/test_meta.sh_0ae5840c0758ec95a397493ac3c3d4fabd397a34.err

@@ -0,0 +1,7 @@
+✘ error: unexpected JSON value
+ --> command-option:1
+ | ;UPDATE access_log SET log_annotations = '1' WHERE log_line = 0
+ --> log_annotations:1
+ | 1                                       
+ = help: Available Properties
+           (.*)

test/expected/test_meta.sh_4283bf9128e0396c4ff141faf68279d5a3d8ac63.out

@@ -0,0 +1,41 @@
+#!lnav -Nf
+# This file is an export of an lnav session.  You can type
+# '|/path/to/this/file' in lnav to execute this file and
+# restore the state of the session.
+
+;SELECT raise_error('This session export was made with a newer version of lnav, please upgrade to ' || '0.11.2' || ' or later')
+   WHERE lnav_version() < '0.11.2' COLLATE naturalcase
+
+# The files loaded into the session were:
+
+
+# Set this environment variable to override this value or edit this script.
+;INSERT OR IGNORE INTO environ (name, value) VALUES ('LOG_DIR_0', '{top_srcdir_parent}')
+:open $LOG_DIR_0/lnav/test/logfile_access_log.0
+
+:rebuild
+
+
+# The following SQL statements will restore the bookmarks,
+# comments, and tags that were added in the session.
+
+;SELECT total_changes() AS before_mark_changes
+;UPDATE all_logs SET log_mark = 0, log_comment = NULL, log_tags = NULL, log_annotations = '{"org.lnav.test":"Hello, <span style=\"color: #f00\">World</span>!\n"}' WHERE log_time_msecs = 1248130766000 AND log_format = 'access_log' AND log_line_hash = 'v1:3f7e0f10f2473f83b2b4eacccfc9b4e2'
+;UPDATE all_logs SET log_mark = 0, log_comment = NULL, log_tags = '["#foo"]', log_annotations = NULL WHERE log_time_msecs = 1248130769000 AND log_format = 'access_log' AND log_line_hash = 'v1:7ad4831dd06e0d6b0ffd965c7d65285f'
+
+;SELECT 2 - (total_changes() - $before_mark_changes) AS failed_mark_changes
+;SELECT echoln(printf('%sERROR%s: failed to restore %d bookmarks',
+                      $ansi_red, $ansi_norm, $failed_mark_changes))
+    WHERE $failed_mark_changes != 0
+
+
+# The following SQL statements will restore the filters that
+# were added in the session.
+
+;REPLACE INTO lnav_view_filters (view_name, enabled, type, language, pattern) VALUES ('log', 0, 'in', 'regex', 'credential status')
+
+
+# The following commands will restore the state of the LOG view.
+
+:switch-to-view log
+:goto 0

test/expected/test_meta.sh_65325d1bcb58e530a8d681685fea56803f67a2eb.out

@@ -0,0 +1,2 @@
+log_line   log_tags    
+      46 ["#bro-test"] 

test/expected/test_meta.sh_aae427fe704f131f8466fbfc9e7967c2e874530a.out

@@ -0,0 +1,2 @@
+log_line log_annotations 
+       0 {"abc":"def"}   

test/expected/test_meta.sh_ad2f396f332d14ed6e95936983bc7e4aeea0bfac.out

@@ -0,0 +1,7 @@
+192.168.202.254 - - [20/Jul/2009:22:59:26 +0000] "GET /vmw/cgi/tramp HTTP/1.0" 200 134 "-" "gPXE/0.9.7"
+ ├ org.lnav.test:
+ ╰ Hello, World!
+192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkboot.gz HTTP/1.0" 404 46210 "-" "gPXE/0.9.7"
+ 📝 Annotations available, focus on this line and use :annotate to apply them
+192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkernel.gz HTTP/1.0" 200 78929 "-" "gPXE/0.9.7"
+ 📝 Annotations available, focus on this line and use :annotate to apply them

test/expected/test_meta.sh_b622c04c2bede5371bfa6f2627b0474049ecc5e9.err

@@ -0,0 +1,6 @@
+✘ error: invalid JSON
+ reason: parse error: premature EOF
+ --> command-option:1
+ | ;UPDATE access_log SET log_annotations = '{"abc": "def"' WHERE log_line = 0
+ --> log_annotations:1
+ | {"abc": "def"                           

test/expected/test_meta.sh_f7c55356df0d71c51eb7629c5a81f347083ce1ef.out

@@ -0,0 +1,2 @@
+log_line   log_tags    
+      46 ["#bro-test"] 

test/expected/test_sessions.sh_6d87ff483d5785c58fb271a405ff1c35e4f83cd9.out

@@ -21,7 +21,7 @@
 # comments, and tags that were added in the session.
 
 ;SELECT total_changes() AS before_mark_changes
-;UPDATE all_logs SET log_mark = 1, log_comment = NULL, log_tags = NULL WHERE log_time_msecs = 1248130769000 AND log_format = 'access_log' AND log_line_hash = 'v1:b05c1bdfe75cde41e151c89087e31951'
+;UPDATE all_logs SET log_mark = 1, log_comment = NULL, log_tags = NULL, log_annotations = NULL WHERE log_time_msecs = 1248130769000 AND log_format = 'access_log' AND log_line_hash = 'v1:b05c1bdfe75cde41e151c89087e31951'
 
 ;SELECT 1 - (total_changes() - $before_mark_changes) AS failed_mark_changes
 ;SELECT echoln(printf('%sERROR%s: failed to restore %d bookmarks',

test/test_meta.sh

@@ -122,7 +122,26 @@ run_cap_test ${lnav_test} -n \
     -c ";SELECT log_line, log_tags FROM bro_http_log WHERE log_tags IS NOT NULL" \
     ${test_dir}/logfile_bro_http.log.0
 
+run_cap_test ${lnav_test} -n \
+    -c ";UPDATE access_log SET log_annotations = '1' WHERE log_line = 0" \
+    ${test_dir}/logfile_access_log.0
+
+run_cap_test ${lnav_test} -n \
+    -c ";UPDATE access_log SET log_annotations = '{\"abc\": \"def\"' WHERE log_line = 0" \
+    ${test_dir}/logfile_access_log.0
+
+run_cap_test ${lnav_test} -n \
+    -c ";UPDATE access_log SET log_annotations = '{\"abc\": \"def\"}' WHERE log_line = 0" \
+    -c ";SELECT log_line,log_annotations FROM access_log WHERE log_annotations IS NOT NULL" \
+    ${test_dir}/logfile_access_log.0
+
 export TEST_ANNO=1
 run_cap_test ${lnav_test} -d /tmp/lnav.err -I ${test_dir} -n \
     -c ':annotate' \
+    -c ':save-session' \
+    ${test_dir}/logfile_access_log.0
+
+run_cap_test ${lnav_test} -d /tmp/lnav.err -I ${test_dir} -n \
+    -c ':load-session' \
+    -c ':export-session-to -' \
     ${test_dir}/logfile_access_log.0

test/test_stubs.cc

@@ -69,6 +69,11 @@ rebuild_indexes_repeatedly()
 {
 }
 
+void
+wait_for_pipers()
+{
+}
+
 readline_context::command_map_t lnav_commands;
 
 namespace injector {