From a16a8cf3fa19fe24703fc6c52603781ca96b9ef9 Mon Sep 17 00:00:00 2001 From: Tim Stack Date: Sun, 30 Jul 2023 21:17:52 -0700 Subject: [PATCH] [log-annotations] add :annotate command --- NEWS.md | 6 + docs/schemas/config-v1.schema.json | 33 +- docs/source/config.rst | 36 +- docs/source/sqlext.rst | 2 + src/CMakeLists.txt | 5 +- src/Makefile.am | 3 + src/base/attr_line.hh | 18 + src/base/auto_fd.cc | 19 + src/base/auto_fd.hh | 22 + src/bookmarks.cc | 9 +- src/bookmarks.hh | 7 +- src/bookmarks.json.hh | 39 ++ src/field_overlay_source.cc | 89 +++- src/field_overlay_source.hh | 9 +- src/fs-extension-functions.cc | 123 +++--- src/internals/cmd-ref.rst | 23 +- src/listview_curses.hh | 2 + src/lnav.cc | 26 +- src/lnav_commands.cc | 94 +++- src/lnav_config.cc | 46 +- src/lnav_config.hh | 2 + src/log.annotate.cc | 406 ++++++++++++++++++ src/log.annotate.cfg.hh | 57 +++ src/log.annotate.hh | 53 +++ src/log_data_helper.cc | 1 + src/log_vtab_impl.cc | 18 + src/log_vtab_impl.hh | 1 + src/logfile_sub_source.cc | 36 ++ src/md2attr_line.cc | 10 +- src/md4cpp.cc | 32 ++ src/md4cpp.hh | 17 + src/root-config.json | 9 + src/scripts/com.vmware.btresolver.py | 57 +++ src/scripts/scripts.am | 1 + src/session_data.cc | 85 +++- test/Makefile.am | 1 + .../configs/invalid-annotation/config.json | 14 + test/configs/installed/anno-test.json | 12 + test/configs/installed/anno-test.sh | 3 + test/expected/expected.am | 4 + ...3639753916f71254e8c9cce4ebb8bfd9978d3e.out | 12 + ...06341dd560f927512e92c7c0985ed8b25827ae.out | 68 +-- ...92c5bc12f5e7aaa6d84c5ed47f0b9f96e36c6a.out | 3 + ...a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out | 16 +- ...22c3e94c536a1bfaeae0c40d271b5b4b08f4fc.out | 6 +- ...907769aba112d628e7ebe39c4ec252e5e0bc69.err | 15 + ...e861d2327512a721fd42ae51dc5427689e0bb6.out | 18 +- ...b0fd243e916546aea22029245ac590dae17a86.out | 28 +- ...223ac4742883f883ccc61044bfffd6e102cca6.out | 13 + ...be20faa161ab9fa77df7568fff84bf3e47e920.out | 8 +- ...d03b1b41a7f819af135d2521a8f2c59418e907.out | 28 +- ...ec34389274affb70a5a76ba4789d51fd60f602.out | 8 +- ...ff27a651650a04d93de9a06ab5480e94ce3a79.out | 8 +- ...8e14a26d8261c9f72747118a469266121d5459.out | 6 +- ...9c1f127c087ec2c6a23a0ecec4df7992cbc8b4.err | 0 ...9c1f127c087ec2c6a23a0ecec4df7992cbc8b4.out | 7 + ...af0b41066ca3be0bbce89c83c011f4ecfa516e.out | 2 +- ...429aed81d7edfd47b57e9cdb8a25c43aff35c4.out | 4 +- ...32c7a21e3f6b7df3aad10d7bdfbb7a812ae6c7.out | 4 +- ...69557a4e14e33e4a213932accbcd4803c947b4.err | 0 ...69557a4e14e33e4a213932accbcd4803c947b4.out | 2 + ...14ebb5e2e83bab11023354dea8a0885ddf64b4.out | 6 +- ...1a8e35f34a206e340a3880128b6ce137847872.out | 10 +- ...fd19d56a8cd1fc9c7eb9351270eabb491f8233.out | 10 +- ...707b6e856dbaab6f95e7e89b98dc3652021f85.out | 6 +- ...681c234d4f60df16c997a05163aeb058c52870.out | 10 +- ...41a0c09601e2419eeb99e85f7e286c889e4801.out | 54 +-- ...0d872ebc492fcecb2e79a0993170d5fc771a5b.out | 4 +- ...5f74863d065418bca5a000e6ad3d9344635164.out | 24 +- ...aae556ecb1661602f176215e28f661d3404032.out | 8 +- ...0fd242f57a96d40f466493938cda0789a094fa.out | 48 +-- ...9373a76853f345d06234f6e0fe11b5d40da27b.out | 12 +- test/test_meta.sh | 5 + test/test_sql.sh | 54 +-- test/test_sql_fs_func.sh | 2 + 75 files changed, 1603 insertions(+), 336 deletions(-) create mode 100644 src/bookmarks.json.hh create mode 100644 src/log.annotate.cc create mode 100644 src/log.annotate.cfg.hh create mode 100644 src/log.annotate.hh create mode 100755 src/scripts/com.vmware.btresolver.py create mode 100644 test/bad-config2/configs/invalid-annotation/config.json create mode 100644 test/configs/installed/anno-test.json create mode 100755 test/configs/installed/anno-test.sh create mode 100644 test/expected/test_meta.sh_039c1f127c087ec2c6a23a0ecec4df7992cbc8b4.err create mode 100644 test/expected/test_meta.sh_039c1f127c087ec2c6a23a0ecec4df7992cbc8b4.out create mode 100644 test/expected/test_sql_fs_func.sh_5a69557a4e14e33e4a213932accbcd4803c947b4.err create mode 100644 test/expected/test_sql_fs_func.sh_5a69557a4e14e33e4a213932accbcd4803c947b4.out diff --git a/NEWS.md b/NEWS.md index 862db4cb..0abf887b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -30,6 +30,12 @@ Features: the logs for a container (e.g. `docker://my-container`) or files within a container (e.g. `docker://my-serv/var/log/dpkg.log`). +* Added an `:annotate` command that can trigger a call-out + to a script to analyze a log message and generate an + annotation that is attached to the message. The script + is executed asynchronously, so it will not block input + and the result is saved in the session. Annotations are + defined in the `/log/annotations` configuration property. * Added the SQLite JSON functions to the online help. * Added `config get` and `config blame` management CLI commands to get the current configuration and the file diff --git a/docs/schemas/config-v1.schema.json b/docs/schemas/config-v1.schema.json index 97dde9da..374f378d 100644 --- a/docs/schemas/config-v1.schema.json +++ b/docs/schemas/config-v1.schema.json @@ -749,7 +749,7 @@ "title": "/log/watch-expressions", "type": "object", "patternProperties": { - "([\\w\\-]+)": { + "([\\w\\.\\-]+)": { "description": "A log message watch expression", "title": "/log/watch-expressions/", "type": "object", @@ -769,6 +769,37 @@ } }, "additionalProperties": false + }, + "annotations": { + "title": "/log/annotations", + "type": "object", + "patternProperties": { + "([\\w\\.\\-]+)": { + "title": "/log/annotations/", + "type": "object", + "properties": { + "description": { + "title": "/log/annotations//description", + "description": "A description of this annotation", + "type": "string" + }, + "condition": { + "title": "/log/annotations//condition", + "description": "The SQLite expression to execute for a log message that determines whether or not this annotation is applicable. The expression is evaluated the same way as a filter expression", + "type": "string", + "minLength": 1 + }, + "handler": { + "title": "/log/annotations//handler", + "description": "The script to execute to generate the annotation content. A JSON object with the log message content will be sent to the script on the standard input", + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false } }, "additionalProperties": false diff --git a/docs/source/config.rst b/docs/source/config.rst index ba03f8e4..3e103ba4 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -262,10 +262,44 @@ From there, you can create a SQLite trigger on the :code:`lnav_events` table that will examine the event contents and perform an action. See the :ref:`Events` section for more information on handling events. +Annotations (v0.12.0+) +^^^^^^^^^^^^^^^^^^^^^^ + +Annotations are content generated by a script for a given log message and +displayed along with the message, like comments and tags. Since the script +is run asynchronously, it can do complex analysis without delaying loading +or interrupting the viewing experience. An annotation is defined by a +condition and a handler in the **lnav** configuration. The condition is +tested against a log message to determine if the annotation is applicable. +If it is, the handler script will be executed for that log message when +the user runs the :ref:`:annotation` command. + +Conditions are SQLite expressions like the ones passed to +:ref:`:filter-expr` where the expression is appended to +:code:`SELECT 1 WHERE`. The expression can use bound variables that +correspond to the columns that would be in the format table and are +prefixed by a colon (:code:`:`). For example, the standard +:code:`log_opid` table column can be access by using :code:`:log_opid`. + +.. note:: The expression is executed with bound variables because it + can be applied to log messages from multiple formats. Writing an + expression that could handle different formats would be more + challenging. In this approach, variables for log message fields + that are not part of a format will evaluate to :code:`NULL`. + +Handlers are executable script files that should be co-located with +the configuration file that defined the annotation. The handler will +be executed and a JSON object with log message data fed in on the +standard input. The handler should then generate the annotation +content on the standard output. The output is treated as Markdown, +so the content can be styled as desired. + Reference ^^^^^^^^^ -.. jsonschema:: ../schemas/config-v1.schema.json#/properties/log/properties/watch-expressions/patternProperties/([\w\-]+) +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/log/properties/watch-expressions/patternProperties/([\w\.\-]+) +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/log/properties/annotations/patternProperties/([\w\.\-]+) + .. _tuning: diff --git a/docs/source/sqlext.rst b/docs/source/sqlext.rst index 221b100e..0a836fae 100644 --- a/docs/source/sqlext.rst +++ b/docs/source/sqlext.rst @@ -105,6 +105,8 @@ The following columns are builtin and included in a :code:`SELECT *`: an :code:`UPDATE` or the :ref:`:comment` command. :log_tags: A JSON list of tags for the message. This column can be changed by an :code:`UPDATE` or the :ref:`:tag` command. + :log_annotations: A JSON object of annotations for this message. + This column is populated by the :ref:`:annotate` command. :log_filters: A JSON list of filter IDs that matched this message The following columns are builtin and are hidden, so they will *not* be diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c8a7aa26..ef76d5a6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -208,7 +208,7 @@ add_custom_command( DEPENDS bin2c ${BUILTIN_LNAV_SCRIPTS}) list(APPEND GEN_SRCS builtin-scripts.h builtin-scripts.cc) -set(BUILTIN_SH_SCRIPTS scripts/dump-pid.sh scripts/pcap_log-converter.sh) +set(BUILTIN_SH_SCRIPTS scripts/com.vmware.btresolver.py scripts/dump-pid.sh scripts/pcap_log-converter.sh) set(BUILTIN_SH_SCRIPT_PATHS ${BUILTIN_SH_SCRIPTS}) @@ -378,6 +378,7 @@ add_library( lnav_commands.cc lnav_config.cc lnav_util.cc + log.annotate.cc log.watch.cc log_accel.cc log_actions.cc @@ -489,6 +490,8 @@ add_library( lnav_config.hh lnav_config_fwd.hh lnav_util.hh + log.annotate.hh + log.annotate.cfg.hh log.watch.hh log_actions.hh log_data_helper.hh diff --git a/src/Makefile.am b/src/Makefile.am index 437805a9..786e7f3d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -237,6 +237,8 @@ noinst_HEADERS = \ lnav_config.hh \ lnav_config_fwd.hh \ lnav_util.hh \ + log.annotate.hh \ + log.annotate.cfg.hh \ log.watch.hh \ log_accel.hh \ log_actions.hh \ @@ -419,6 +421,7 @@ libdiag_a_SOURCES = \ lnav_commands.cc \ lnav_config.cc \ lnav_util.cc \ + log.annotate.cc \ log.watch.cc \ log_accel.cc \ log_actions.cc \ diff --git a/src/base/attr_line.hh b/src/base/attr_line.hh index 8a62fcd4..9939118a 100644 --- a/src/base/attr_line.hh +++ b/src/base/attr_line.hh @@ -427,6 +427,24 @@ public: return *this; } + template + attr_line_t& insert(size_t index, + const std::pair& value) + { + size_t start_len = this->al_string.length(); + + this->insert(index, std::move(value.first)); + + line_range lr{ + (int) index, + (int) (index + (this->al_string.length() - start_len)), + }; + + this->al_attrs.emplace_back(lr, value.second); + + return *this; + } + template attr_line_t& add_header(Args... args) { diff --git a/src/base/auto_fd.cc b/src/base/auto_fd.cc index d5e2c489..bb96bb00 100644 --- a/src/base/auto_fd.cc +++ b/src/base/auto_fd.cc @@ -141,6 +141,25 @@ auto_fd::operator=(int fd) return *this; } +Result +auto_fd::write_fully(string_fragment sf) +{ + while (!sf.empty()) { + auto rc = write(this->af_fd, sf.data(), sf.length()); + + if (rc < 0) { + return Err( + fmt::format(FMT_STRING("failed to write {} bytes to FD {}"), + sf.length(), + this->af_fd)); + } + + sf = sf.substr(rc); + } + + return Ok(); +} + Result auto_pipe::for_child_fd(int child_fd) { diff --git a/src/base/auto_fd.hh b/src/base/auto_fd.hh index 5a2c334c..c53de768 100644 --- a/src/base/auto_fd.hh +++ b/src/base/auto_fd.hh @@ -36,6 +36,7 @@ #include +#include "base/intern_string.hh" #include "base/result.h" /** @@ -158,6 +159,8 @@ public: */ void reset(int fd = -1); + Result write_fully(string_fragment sf); + void close_on_exec() const; void non_blocking() const; @@ -170,6 +173,25 @@ class auto_pipe { public: static Result for_child_fd(int child_fd); + template + static Result, std::string> + for_child_fds(ARGS... args) + { + std::array retval; + + size_t index = 0; + for (const auto child_fd : {args...}) { + auto open_res = for_child_fd(child_fd); + if (open_res.isErr()) { + return Err(open_res.unwrapErr()); + } + + retval[index++] = open_res.unwrap(); + } + + return Ok(std::move(retval)); + } + explicit auto_pipe(int child_fd = -1, int child_flags = O_RDONLY); int open(); diff --git a/src/bookmarks.cc b/src/bookmarks.cc index 89637ad2..11a0edb6 100644 --- a/src/bookmarks.cc +++ b/src/bookmarks.cc @@ -32,10 +32,16 @@ #include "bookmarks.hh" #include "base/itertools.hh" +#include "bookmarks.json.hh" #include "config.h" std::unordered_set bookmark_metadata::KNOWN_TAGS; +typed_json_path_container logmsg_annotations_handlers = { + yajlpp::pattern_property_handler("(?.*)") + .for_field(&logmsg_annotations::la_pairs), +}; + void bookmark_metadata::add_tag(const std::string& tag) { @@ -61,7 +67,7 @@ bool bookmark_metadata::empty() const { return this->bm_name.empty() && this->bm_comment.empty() - && this->bm_tags.empty(); + && this->bm_tags.empty() && this->bm_annotations.la_pairs.empty(); } void @@ -69,6 +75,7 @@ bookmark_metadata::clear() { this->bm_comment.clear(); this->bm_tags.clear(); + this->bm_annotations.la_pairs.clear(); } nonstd::optional diff --git a/src/bookmarks.hh b/src/bookmarks.hh index 189e8308..73b0493c 100644 --- a/src/bookmarks.hh +++ b/src/bookmarks.hh @@ -34,17 +34,22 @@ #include #include -#include #include +#include #include #include "base/lnav_log.hh" +struct logmsg_annotations { + std::map la_pairs; +}; + struct bookmark_metadata { static std::unordered_set KNOWN_TAGS; std::string bm_name; std::string bm_comment; + logmsg_annotations bm_annotations; std::vector bm_tags; void add_tag(const std::string& tag); diff --git a/src/bookmarks.json.hh b/src/bookmarks.json.hh new file mode 100644 index 00000000..f3f267ab --- /dev/null +++ b/src/bookmarks.json.hh @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2023, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef lnav_bookmarks_json_hh +#define lnav_bookmarks_json_hh + +#include "bookmarks.hh" +#include "yajlpp/yajlpp_def.hh" + +extern typed_json_path_container + logmsg_annotations_handlers; + +#endif diff --git a/src/field_overlay_source.cc b/src/field_overlay_source.cc index 090d41e3..5512a0ad 100644 --- a/src/field_overlay_source.cc +++ b/src/field_overlay_source.cc @@ -33,6 +33,7 @@ #include "base/humanize.time.hh" #include "base/snippet_highlighters.hh" #include "config.h" +#include "log.annotate.hh" #include "log_format_ext.hh" #include "log_vtab_impl.hh" #include "md2attr_line.hh" @@ -41,10 +42,14 @@ #include "vtab_module.hh" #include "vtab_module_json.hh" +using namespace md4cpp::literals; +using namespace lnav::roles::literals; + json_string extract(const char* str); void -field_overlay_source::build_field_lines(const listview_curses& lv) +field_overlay_source::build_field_lines(const listview_curses& lv, + vis_line_t row) { auto& lss = this->fos_lss; auto& vc = view_colors::singleton(); @@ -57,7 +62,7 @@ field_overlay_source::build_field_lines(const listview_curses& lv) return; } - content_line_t cl = lss.at(lv.get_selection()); + content_line_t cl = lss.at(row); std::shared_ptr file = lss.find(cl); auto ll = file->begin() + cl; auto format = file->get_format(); @@ -72,13 +77,13 @@ field_overlay_source::build_field_lines(const listview_curses& lv) display = display || this->fos_contexts.top().c_show; } - this->build_meta_line(lv, this->fos_lines, lv.get_top()); + this->build_meta_line(lv, this->fos_lines, row); if (!display) { return; } - if (!this->fos_log_helper.parse_line(lv.get_selection())) { + if (!this->fos_log_helper.parse_line(row)) { return; } @@ -450,6 +455,23 @@ field_overlay_source::build_meta_line(const listview_curses& lv, { auto line_meta_opt = this->fos_lss.find_bookmark_metadata(row); + auto file_and_line = this->fos_lss.find_line_with_file(row); + if (file_and_line && !file_and_line->second->is_continued()) { + auto applicable_anno = lnav::log::annotate::applicable(row); + if (!applicable_anno.empty() + && (!line_meta_opt + || line_meta_opt.value()->bm_annotations.la_pairs.empty())) + { + auto anno_msg = attr_line_t(" ") + .append(":memo:"_emoji) + .append(" Annotations available, use ") + .append(":annotate"_quoted_code) + .append(" to apply them to this line"); + + dst.emplace_back(anno_msg); + } + } + if (!line_meta_opt) { return; } @@ -527,6 +549,61 @@ field_overlay_source::build_meta_line(const listview_curses& lv, } dst.emplace_back(al); } + if (!line_meta.bm_annotations.la_pairs.empty()) { + for (const auto& anno_pair : line_meta.bm_annotations.la_pairs) { + attr_line_t al; + md2attr_line mdal; + + dst.push_back( + attr_line_t() + .append(filename_width, ' ') + .appendf(FMT_STRING(" \u251c {}:"), anno_pair.first) + .with_attr_for_all(VC_ROLE.value(role_t::VCR_COMMENT))); + + auto parse_res = md4cpp::parse(anno_pair.second, mdal); + if (parse_res.isOk()) { + al.append(parse_res.unwrap()); + } else { + log_error("%d: cannot convert annotation to markdown: %s", + (int) row, + parse_res.unwrapErr().c_str()); + al.append(anno_pair.second); + } + + auto anno_lines = al.rtrim().split_lines(); + if (anno_lines.back().empty()) { + anno_lines.pop_back(); + } + for (size_t lpc = 0; lpc < anno_lines.size(); lpc++) { + auto& anno_line = anno_lines[lpc]; + + if (lpc == 0 && anno_line.empty()) { + continue; + } + // anno_line.with_attr_for_all(VC_ROLE.value(role_t::VCR_COMMENT)); + anno_line.insert(0, + lpc == anno_lines.size() - 1 + ? " \u2570 "_comment + : " \u2502 "_comment); + anno_line.insert(0, filename_width, ' '); + if (tc != nullptr) { + auto hl = tc->get_highlights(); + auto hl_iter + = hl.find({highlight_source_t::PREVIEW, "search"}); + + if (hl_iter != hl.end()) { + hl_iter->second.annotate(anno_line, filename_width); + } + } + + dst.emplace_back(anno_line); + } + } + } + + if (dst.size() > 30) { + dst.resize(30); + } } void @@ -550,7 +627,9 @@ field_overlay_source::list_value_for_overlay(const listview_curses& lv, attr_line_t& value_out) { if (y == 0) { - this->build_field_lines(lv); + this->fos_meta_lines.clear(); + this->fos_meta_lines_row = -1_vl; + this->build_field_lines(lv, row); return false; } diff --git a/src/field_overlay_source.hh b/src/field_overlay_source.hh index ab1d17fc..48f28b56 100644 --- a/src/field_overlay_source.hh +++ b/src/field_overlay_source.hh @@ -48,13 +48,20 @@ public: void add_key_line_attrs(int key_size, bool last_line = false); + void reset() override + { + this->fos_lines.clear(); + this->fos_meta_lines.clear(); + this->fos_meta_lines_row = -1_vl; + } + bool list_value_for_overlay(const listview_curses& lv, int y, int bottom, vis_line_t row, attr_line_t& value_out) override; - void build_field_lines(const listview_curses& lv); + void build_field_lines(const listview_curses& lv, vis_line_t row); void build_meta_line(const listview_curses& lv, std::vector& dst, vis_line_t row); diff --git a/src/fs-extension-functions.cc b/src/fs-extension-functions.cc index 8067ff2e..9a98469f 100644 --- a/src/fs-extension-functions.cc +++ b/src/fs-extension-functions.cc @@ -206,35 +206,24 @@ sql_shell_exec(const char* cmd, options = parse_res.unwrap(); } - auto in_pipe_res = auto_pipe::for_child_fd(STDIN_FILENO); - if (in_pipe_res.isErr()) { - throw lnav::console::user_message::error("cannot open input pipe") - .with_reason(in_pipe_res.unwrapErr()); + auto child_fds_res + = auto_pipe::for_child_fds(STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO); + if (child_fds_res.isErr()) { + throw lnav::console::user_message::error("cannot open child pipes") + .with_reason(child_fds_res.unwrapErr()); } - auto in_pipe = in_pipe_res.unwrap(); - auto out_pipe_res = auto_pipe::for_child_fd(STDOUT_FILENO); - if (out_pipe_res.isErr()) { - throw lnav::console::user_message::error("cannot open output pipe") - .with_reason(out_pipe_res.unwrapErr()); - } - auto out_pipe = out_pipe_res.unwrap(); - auto err_pipe_res = auto_pipe::for_child_fd(STDERR_FILENO); - if (err_pipe_res.isErr()) { - throw lnav::console::user_message::error("cannot open error pipe") - .with_reason(err_pipe_res.unwrapErr()); - } - auto err_pipe = err_pipe_res.unwrap(); auto child_pid_res = lnav::pid::from_fork(); if (child_pid_res.isErr()) { throw lnav::console::user_message::error("cannot fork()") .with_reason(child_pid_res.unwrapErr()); } + auto child_fds = child_fds_res.unwrap(); auto child_pid = child_pid_res.unwrap(); - in_pipe.after_fork(child_pid.in()); - out_pipe.after_fork(child_pid.in()); - err_pipe.after_fork(child_pid.in()); + for (auto& child_fd : child_fds) { + child_fd.after_fork(child_pid.in()); + } if (child_pid.in_child()) { const char* args[] = { @@ -256,64 +245,56 @@ sql_shell_exec(const char* cmd, _exit(EXIT_FAILURE); } - auto out_reader = std::async(std::launch::async, [&out_pipe]() { - auto buffer = auto_buffer::alloc(4096); - - while (true) { - if (buffer.available() < 4096) { - buffer.expand_by(4096); + auto out_reader = std::async( + std::launch::async, [out_fd = std::move(child_fds[1].read_end())]() { + auto buffer = auto_buffer::alloc(4096); + + while (true) { + if (buffer.available() < 4096) { + buffer.expand_by(4096); + } + + auto rc + = read(out_fd, buffer.next_available(), buffer.available()); + if (rc < 0) { + break; + } + if (rc == 0) { + break; + } + buffer.resize_by(rc); } - auto rc = read(out_pipe.read_end(), - buffer.next_available(), - buffer.available()); - if (rc < 0) { - break; - } - if (rc == 0) { - break; + return buffer; + }); + + auto err_reader = std::async( + std::launch::async, [err_fd = std::move(child_fds[2].read_end())]() { + auto buffer = auto_buffer::alloc(4096); + + while (true) { + if (buffer.available() < 4096) { + buffer.expand_by(4096); + } + + auto rc + = read(err_fd, buffer.next_available(), buffer.available()); + if (rc < 0) { + break; + } + if (rc == 0) { + break; + } + buffer.resize_by(rc); } - buffer.resize_by(rc); - } - - return buffer; - }); - auto err_reader = std::async(std::launch::async, [&err_pipe]() { - auto buffer = auto_buffer::alloc(4096); - - while (true) { - if (buffer.available() < 4096) { - buffer.expand_by(4096); - } - - auto rc = read(err_pipe.read_end(), - buffer.next_available(), - buffer.available()); - if (rc < 0) { - break; - } - if (rc == 0) { - break; - } - buffer.resize_by(rc); - } - - return buffer; - }); + return buffer; + }); if (input) { - auto sf = input.value(); - - while (!sf.empty()) { - auto rc = write(in_pipe.write_end(), sf.data(), sf.length()); - if (rc < 0) { - break; - } - sf = sf.substr(rc); - } - in_pipe.close(); + child_fds[0].write_end().write_fully(input.value()); } + child_fds[0].close(); auto retval = blob_auto_buffer{out_reader.get()}; diff --git a/src/internals/cmd-ref.rst b/src/internals/cmd-ref.rst index 4134fc46..f428a247 100644 --- a/src/internals/cmd-ref.rst +++ b/src/internals/cmd-ref.rst @@ -49,6 +49,19 @@ ---- +.. _annotate: + +:annotate +^^^^^^^^^ + + Analyze the focused log message and attach annotations + + **See Also** + :ref:`comment`, :ref:`tag` + +---- + + .. _append_to: :append-to *path* @@ -96,7 +109,7 @@ Clear the comment attached to the top log line **See Also** - :ref:`comment`, :ref:`tag` + :ref:`annotate`, :ref:`comment`, :ref:`tag` ---- @@ -190,7 +203,7 @@ :comment This is where it all went wrong **See Also** - :ref:`clear_comment`, :ref:`tag` + :ref:`annotate`, :ref:`clear_comment`, :ref:`tag` ---- @@ -370,7 +383,7 @@ :delete-tags #BUG123 #needs-review **See Also** - :ref:`comment`, :ref:`tag` + :ref:`annotate`, :ref:`comment`, :ref:`tag` ---- @@ -1362,7 +1375,7 @@ :tag #BUG123 #needs-review **See Also** - :ref:`comment`, :ref:`delete_tags`, :ref:`untag` + :ref:`annotate`, :ref:`comment`, :ref:`delete_tags`, :ref:`untag` ---- @@ -1440,7 +1453,7 @@ :untag #BUG123 #needs-review **See Also** - :ref:`comment`, :ref:`tag` + :ref:`annotate`, :ref:`comment`, :ref:`tag` ---- diff --git a/src/listview_curses.hh b/src/listview_curses.hh index bf375bc5..21bd4a14 100644 --- a/src/listview_curses.hh +++ b/src/listview_curses.hh @@ -105,6 +105,8 @@ class list_overlay_source { public: virtual ~list_overlay_source() = default; + virtual void reset() {} + virtual bool list_value_for_overlay(const listview_curses& lv, int y, int bottom, diff --git a/src/lnav.cc b/src/lnav.cc index d1bce0d8..9a6bd3c6 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -116,6 +116,7 @@ #include "log_vtab_impl.hh" #include "logfile.hh" #include "logfile_sub_source.hh" +#include "md4cpp.hh" #include "piper.looper.hh" #include "readline_curses.hh" #include "readline_highlighters.hh" @@ -170,6 +171,7 @@ using namespace std::literals::chrono_literals; using namespace lnav::roles::literals; +using namespace md4cpp::literals; static std::vector DEFAULT_FILES; static auto intern_lifetime = intern_string::get_table_lifetime(); @@ -707,28 +709,40 @@ make it easier to navigate through files quickly. .append("\n ") .append("\u2022"_list_glyph) .append(" Format files are read from:") - .append("\n \U0001F4C2 ") + .append("\n ") + .append(":open_file_folder:"_emoji) + .append(" ") .append(lnav::roles::file("/etc/lnav")) - .append("\n \U0001F4C2 ") + .append("\n ") + .append(":open_file_folder:"_emoji) + .append(" ") .append(lnav::roles::file(SYSCONFDIR "/lnav")) .append("\n ") .append("\u2022"_list_glyph) .append(" Configuration, session, and format files are stored in:\n") - .append(" \U0001F4C2 ") + .append(" ") + .append(":open_file_folder:"_emoji) + .append(" ") .append(lnav::roles::file(lnav::paths::dotlnav().string())) .append("\n\n ") .append("\u2022"_list_glyph) .append(" Local copies of remote files, files extracted from\n") .append(" archives, execution output, and so on are stored in:\n") - .append(" \U0001F4C2 ") + .append(" ") + .append(":open_file_folder:"_emoji) + .append(" ") .append(lnav::roles::file(lnav::paths::workdir().string())) .append("\n\n") .append("Documentation"_h1) .append(": https://docs.lnav.org\n") .append("Contact"_h1) .append("\n") - .append(" \U0001F4AC https://github.com/tstack/lnav/discussions\n") - .appendf(FMT_STRING(" \U0001F4EB {}\n"), PACKAGE_BUGREPORT) + .append(" ") + .append(":speech_balloon:"_emoji) + .append(" https://github.com/tstack/lnav/discussions\n") + .append(" ") + .append(":mailbox:"_emoji) + .appendf(FMT_STRING(" {}\n"), PACKAGE_BUGREPORT) .append("Version"_h1) .appendf(FMT_STRING(": {}"), VCS_PACKAGE_STRING); diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index d33f83e5..cd49be5c 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -62,6 +62,7 @@ #include "lnav_commands.hh" #include "lnav_config.hh" #include "lnav_util.hh" +#include "log.annotate.hh" #include "log_data_helper.hh" #include "log_data_table.hh" #include "log_search_table.hh" @@ -599,6 +600,40 @@ com_relative_goto(exec_context& ec, return Ok(retval); } +static Result +com_annotate(exec_context& ec, + std::string cmdline, + std::vector& args) +{ + std::string retval; + + if (args.empty()) { + } else if (!ec.ec_dry_run) { + auto* tc = *lnav_data.ld_view_stack.top(); + auto* lss = dynamic_cast(tc->get_sub_source()); + + if (lss != nullptr) { + auto sel = tc->get_selection(); + auto applicable_annos = lnav::log::annotate::applicable(sel); + + if (applicable_annos.empty()) { + return ec.make_error( + "no annotations available for this log message"); + } + + auto apply_res = lnav::log::annotate::apply(sel, applicable_annos); + if (apply_res.isErr()) { + return Err(apply_res.unwrapErr()); + } + } else { + return ec.make_error( + ":annotate is only supported for the LOG view"); + } + } + + return Ok(retval); +} + static Result com_mark(exec_context& ec, std::string cmdline, std::vector& args) { @@ -1419,24 +1454,42 @@ com_save_to(exec_context& ec, tc->set_word_wrap(wrapped); } else { auto* los = tc->get_overlay_source(); + auto* fos = dynamic_cast(los); std::vector rows(1); attr_line_t ov_al; size_t count = 0; + if (fos != nullptr) { + fos->fos_contexts.push(field_overlay_source::context{ + "", + false, + false, + }); + } + + los->reset(); for (auto iter = all_user_marks.begin(); iter != all_user_marks.end(); iter++, count++) { if (ec.ec_dry_run && count > 10) { break; } + auto y = 0_vl; + while (los != nullptr + && los->list_value_for_overlay( + *tc, y, tc->get_inner_height(), *iter, ov_al)) + { + write_line_to(outfile, ov_al); + ++y; + } tc->listview_value_for_rows(*tc, *iter, rows); + ++y; if (anonymize) { rows[0].al_attrs.clear(); rows[0].al_string = ta.next(rows[0].al_string); } write_line_to(outfile, rows[0]); - auto y = 1_vl; while (los != nullptr && los->list_value_for_overlay( *tc, y, tc->get_inner_height(), *iter, ov_al)) @@ -1447,6 +1500,10 @@ com_save_to(exec_context& ec, line_count += 1; } + + if (fos != nullptr) { + fos->fos_contexts.pop(); + } } fflush(outfile); @@ -4214,8 +4271,15 @@ com_sh(exec_context& ec, std::string cmdline, std::vector& args) log_info("executing: %s", carg.c_str()); - auto out_pipe_res = auto_pipe::for_child_fd(STDOUT_FILENO); - auto err_pipe_res = auto_pipe::for_child_fd(STDERR_FILENO); + auto child_fds_res + = auto_pipe::for_child_fds(STDOUT_FILENO, STDERR_FILENO); + if (child_fds_res.isErr()) { + auto um = lnav::console::user_message::error( + "unable to create child pipes") + .with_reason(child_fds_res.unwrapErr()); + ec.add_error_context(um); + return Err(um); + } auto child_res = lnav::pid::from_fork(); if (child_res.isErr()) { auto um @@ -4225,12 +4289,11 @@ com_sh(exec_context& ec, std::string cmdline, std::vector& args) return Err(um); } - auto out_pipe = out_pipe_res.unwrap(); - auto err_pipe = err_pipe_res.unwrap(); - + auto child_fds = child_fds_res.unwrap(); auto child = child_res.unwrap(); - out_pipe.after_fork(child.in()); - err_pipe.after_fork(child.in()); + for (auto& child_fd : child_fds) { + child_fd.after_fork(child.in()); + } if (child.in_child()) { auto dev_null = open("/dev/null", O_RDONLY | O_CLOEXEC); @@ -4271,8 +4334,8 @@ com_sh(exec_context& ec, std::string cmdline, std::vector& args) .fo_name; auto create_piper_res = lnav::piper::create_looper(display_name, - std::move(out_pipe.read_end()), - std::move(err_pipe.read_end())); + std::move(child_fds[0].read_end()), + std::move(child_fds[1].read_end())); if (create_piper_res.isErr()) { auto um @@ -5179,6 +5242,17 @@ readline_context::command_t STD_COMMANDS[] = { {"To move 10 percent back in the view", "-10%"}, }) .with_tags({"navigation"})}, + + { + "annotate", + com_annotate, + + help_text(":annotate") + .with_summary( + "Analyze the focused log message and attach annotations") + .with_tags({"metadata"}), + }, + {"mark", com_mark, diff --git a/src/lnav_config.cc b/src/lnav_config.cc index 45d9068e..ce41e583 100644 --- a/src/lnav_config.cc +++ b/src/lnav_config.cc @@ -102,6 +102,9 @@ static auto uh = injector::bind::to_instance( static auto lsc = injector::bind::to_instance( +[]() { return &lnav_config.lc_log_source; }); +static auto annoc = injector::bind::to_instance( + +[]() { return &lnav_config.lc_log_annotations; }); + static auto tssc = injector::bind::to_instance( +[]() { return &lnav_config.lc_top_status_cfg; }); @@ -1238,7 +1241,7 @@ static const struct json_path_container log_source_watch_expr_handlers = { }; static const struct json_path_container log_source_watch_handlers = { - yajlpp::pattern_property_handler("(?[\\w\\-]+)") + yajlpp::pattern_property_handler("(?[\\w\\.\\-]+)") .with_synopsis("") .with_description("A log message watch expression") .with_obj_provider") + .with_description("A description of this annotation") + .for_field(&lnav::log::annotate::annotation_def::a_description), + yajlpp::property_handler("condition") + .with_synopsis("") + .with_description( + "The SQLite expression to execute for a log message that " + "determines whether or not this annotation is applicable. The " + "expression is evaluated the same way as a filter expression") + .with_min_length(1) + .for_field(&lnav::log::annotate::annotation_def::a_condition), + yajlpp::property_handler("handler") + .with_synopsis("