diff --git a/.gitignore b/.gitignore index 51f95aeb..8c17e874 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ .lnav *.dat *.diff -*.err +test/*.err *.index *.log *.o diff --git a/NEWS b/NEWS index 9706864a..fb542fdf 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,10 @@ lnav v0.10.2: Features: + * Added an integration with regex101.com to make it easier to edit + log message regular expressions. Using the new "management CLI" + (activated by the -m option), a log format can be created from + a regular expression entry on regex101.com and existing patterns + can be edited. * Add initial support for pcap(3) files using tshark(1). * Add format for UniFi gateway. @@ -13,6 +18,7 @@ lnav v0.10.2: Fixes: * Toggling enabled/disabled filters when there is a SQL expression no longer causes a crash. + * Fix a crash related to long lines that are word wrapped. lnav v0.10.1: Features: diff --git a/TESTS_ENVIRONMENT.in b/TESTS_ENVIRONMENT.in index 4eacf186..3b68bebe 100644 --- a/TESTS_ENVIRONMENT.in +++ b/TESTS_ENVIRONMENT.in @@ -42,6 +42,8 @@ test_file_base=`basename $1` # The current test number for shell based tests. test_num=0 +test_hash="" + lnav_test="${top_builddir}/src/lnav-test" export lnav_test @@ -61,6 +63,10 @@ export HAVE_SQLITE3_VALUE_SUBTYPE LAST_TEST="" +LAST_CAP_TEST=() + +has_errors="" + # # Run a test case and capture its standard out and standard err. # @@ -73,11 +79,68 @@ LAST_TEST="" # run_test rktimes -V # run_test() { - LAST_TEST="test: $@" + printf "%s \033[0;35m=============================================================\033[0m\n" $(date -Iseconds) + LAST_TEST=("test: $@") + echo "${LAST_TEST[@]}" export test_num=`expr ${test_num} \+ 1` "$@" > ${test_file_base}_${test_num}.tmp 2> ${test_file_base}_${test_num}.err } +run_cap_test() { + LAST_CAP_TEST=("test: $@") + local full_cmd=$(echo "${LAST_CAP_TEST[@]}" | sed -e "s;${test_dir};{test_dir};g") + export test_hash=$(echo "${full_cmd}" | shasum | cut -f 1 -d ' ') + echo "${full_cmd}" > ${test_file_base}_${test_hash}.cmd + "$@" > ${test_file_base}_${test_hash}.out 2> ${test_file_base}_${test_hash}.err + + sed -ibak \ + -e "s;${builddir};{builddir};g" \ + -e "s;${test_dir};{test_dir};g" \ + -e "s;${top_srcdir};{top_srcdir};g" \ + ${test_file_base}_${test_hash}.out + echo + printf "%s \033[0;35m=============================================================\033[0m\n" $(date -Iseconds) + printf '\033[0;35mCommand\033[0m: %s\n' "${full_cmd}" + printf '\033[0;32mBEGIN\033[0m %s\n' "${test_file_base}_${test_hash}.out" + cat "${test_file_base}_${test_hash}.out" + printf '\033[0;32mEND\033[0m %s\n' "${test_file_base}_${test_hash}.out" + if test -f ${srcdir}/expected/${test_file_base}_${test_hash}.out; then + diff -w -u \ + ${srcdir}/expected/${test_file_base}_${test_hash}.out \ + ${test_file_base}_${test_hash}.out \ + > ${test_file_base}_${test_hash}.diff + if test $? -ne 0; then + echo OUT: "${full_cmd}" + cat ${test_file_base}_${test_hash}.diff + export has_errors="yes" + fi + else + export has_errors="yes" + fi + + sed -ibak -E \ + -e "s;${builddir};{builddir};g" \ + -e "s;${test_dir};{test_dir};g" \ + -e "s;${top_srcdir};{top_srcdir};g" \ + -e 's;"errorId":".+";;g' \ + ${test_file_base}_${test_hash}.err + printf '\033[0;31mBEGIN\033[0m %s\n' "${test_file_base}_${test_hash}.err" + cat "${test_file_base}_${test_hash}.err" + printf '\033[0;31mEND\033[0m %s\n' "${test_file_base}_${test_hash}.err" + if test -f ${srcdir}/expected/${test_file_base}_${test_hash}.err; then + diff -w -u ${srcdir}/expected/${test_file_base}_${test_hash}.err \ + ${test_file_base}_${test_hash}.err \ + > ${test_file_base}_${test_hash}.err.diff + if test $? -ne 0; then + echo ERR: "${full_cmd}" + cat ${test_file_base}_${test_hash}.err.diff + export has_errors="yes" + fi + else + export has_errors="yes" + fi +} + # # Check the output generated by a run_test() call. # @@ -100,7 +163,7 @@ check_output() { ${test_file_base}_${test_num}.tmp diff -w -u - ${test_file_base}_${test_num}.tmp > ${test_file_base}_${test_num}.diff if test $? -ne 0; then - echo $LAST_TEST + echo "${LAST_TEST[@]}" echo $1 cat ${test_file_base}_${test_num}.diff exit 1 @@ -110,7 +173,7 @@ check_output() { check_output_ws() { diff -u - ${test_file_base}_${test_num}.tmp > ${test_file_base}_${test_num}.diff if test $? -ne 0; then - echo $LAST_TEST + echo "${LAST_TEST[@]}" echo $1 cat ${test_file_base}_${test_num}.diff exit 1 @@ -133,7 +196,7 @@ check_error_output() { diff -w -u - ${test_file_base}_${test_num}.err \ > ${test_file_base}_${test_num}.err.diff if test $? -ne 0; then - echo $LAST_TEST + echo "${LAST_TEST[@]}" echo $1 cat ${test_file_base}_${test_num}.err.diff exit 1 @@ -191,3 +254,11 @@ else shift . ${test_file} fi + +cleanup() { + if test "${has_errors}"x = "yes"x; then + exit 1 + fi +} + +trap "cleanup" EXIT diff --git a/aminclude_static.am b/aminclude_static.am index 587f7b11..e6e5c362 100644 --- a/aminclude_static.am +++ b/aminclude_static.am @@ -1,6 +1,6 @@ # aminclude_static.am generated automatically by Autoconf -# from AX_AM_MACROS_STATIC on Thu Mar 31 15:28:54 PDT 2022 +# from AX_AM_MACROS_STATIC on Fri Apr 22 09:39:38 PDT 2022 # Code coverage diff --git a/cleanup_expected.sh b/cleanup_expected.sh new file mode 100755 index 00000000..5be90bef --- /dev/null +++ b/cleanup_expected.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +srcdir="$1" +builddir="$2" + +for fname in "${srcdir}"/expected/*.out; do + stem=$(basename "$fname" | sed -e 's/.out$//') + + if ! test -f "${builddir}/$stem.cmd"; then + echo "removing $fname" + guilt rm "$fname" + echo "removing ${srcdir}/expected/${stem}.err" + guilt rm "${srcdir}/expected/${stem}.err" + fi +done diff --git a/docs/schemas/config-v1.schema.json b/docs/schemas/config-v1.schema.json index 024bf419..74b500bb 100644 --- a/docs/schemas/config-v1.schema.json +++ b/docs/schemas/config-v1.schema.json @@ -347,6 +347,11 @@ "description": "Styling for 6th-level headers", "title": "/ui/theme-defs//styles/h6", "$ref": "#/definitions/style" + }, + "list-glyph": { + "description": "Styling for glyphs that prefix a list item", + "title": "/ui/theme-defs//styles/list-glyph", + "$ref": "#/definitions/style" } }, "additionalProperties": false diff --git a/docs/schemas/format-v1.schema.json b/docs/schemas/format-v1.schema.json index 766fe163..e78b48bb 100644 --- a/docs/schemas/format-v1.schema.json +++ b/docs/schemas/format-v1.schema.json @@ -314,6 +314,11 @@ "items": { "type": "object", "properties": { + "description": { + "title": "//sample/description", + "description": "A description of this sample.", + "type": "string" + }, "line": { "title": "//sample/line", "description": "A sample log line that should match a pattern in this format.", diff --git a/docs/source/cli.rst b/docs/source/cli.rst index 1edd973f..8700200c 100644 --- a/docs/source/cli.rst +++ b/docs/source/cli.rst @@ -1,16 +1,24 @@ - .. _cli: Command Line Interface ====================== +There are two command-line interfaces provided by **lnav**, one for viewing +files and one for managing **lnav**'s configuration. The file viewing mode is +the default and is all that most people will need. The management mode can +be useful for those that are developing log file formats and is activated by +passing the :option:`-m` option as the first argument. + +File Viewing Mode +----------------- + The following options can be used when starting **lnav**. There are not many flags because the majority of the functionality is accessed using the :option:`-c` option to execute :ref:`commands` or :ref:`SQL queries`. Options -------- +^^^^^^^ .. option:: -h @@ -85,6 +93,32 @@ Options Do not print the log messages after executing all of the commands. +Management Mode (v0.10.2+) +-------------------------- + +Options +^^^^^^^ + +.. option:: -m + + Switch to management mode. This must be the first option passed on the + command-line. + +Subcommands +^^^^^^^^^^^ + +.. option:: regex101 import [] + + Convert a regex101.com entry into a skeleton log format file. + +.. option:: format regex push + + Push a log format regular expression to regex101.com . + +.. option:: format regex pull + + Pull changes to a regex that was previously pushed to regex101.com . + Environment Variables --------------------- diff --git a/docs/source/formats.rst b/docs/source/formats.rst index acc277a5..e00dfb44 100644 --- a/docs/source/formats.rst +++ b/docs/source/formats.rst @@ -55,6 +55,35 @@ own formats or if you need to modify existing ones. Format directories can also contain '.sql' and '.lnav' script files that can be used automate log file analysis. +Creating a Format Using Regex101.com (v0.10.2+) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For plain-text log files, the easiest way to create a log format definition is +to create the regular expression that recognizes log messages using +https://regex101.com . Simply copy a log line into the test string input box +on the site and then start editing the regular expression. When building the +regular expression, you'll want to use named captures for the structured parts +of the log message. Any raw message text should be matched by a captured named +"body". Once you have a regex that matches the whole log message, you can use +**lnav**'s "management CLI" to create a skeleton format file. The skeleton +will be populated with the regular expression from the site and the test +string, along with any unit tests, will be added to the "samples" list. The +"regex101 import" management command is used to create the skeleton and has +the following form: + +.. prompt:: bash + + lnav -m regex101 import [] + +If the import was successful, the path to the new format file should be +printed out. The skeleton will most likely need some changes to make it +fully functional. For example, the :code:`kind` properties for captured values +default to :code:`string`, but you'll want to change them to the appropriate +type. + +Format File Reference +^^^^^^^^^^^^^^^^^^^^^ + An **lnav** format file must contain a single JSON object, preferably with a :code:`$schema` property that refers to the `format-v1.schema `_, @@ -309,8 +338,8 @@ Example format: } } -Modifying an Existing Format ----------------------------- +Patching an Existing Format +--------------------------- When loading log formats from files, **lnav** will overlay any new data over previously loaded data. This feature allows you to override existing value or diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7fa0a961..33a8dcf4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -68,8 +68,10 @@ set(TIME_FORMATS "%m/%d/%Y %l:%M:%S %p %Z" "%m/%e/%Y %I:%M:%S %p" "%m/%e/%Y %l:%M:%S %p" + "%m/%d/%Y %H:%M:%S" "%d/%b/%y %H:%M:%S" "%m%d %H:%M:%S" + "%Y%m%d.%H%M%S" "%H:%M:%S" "%M:%S" "%m/%d %H:%M:%S" @@ -253,6 +255,7 @@ add_library( command_executor.cc curl_looper.cc db_sub_source.cc + dump_internals.cc elem_to_json.cc environ_vtab.cc extension-functions.cc @@ -275,6 +278,8 @@ add_library( input_dispatcher.cc json-extension-functions.cc listview_curses.cc + lnav.indexing.cc + lnav.management_cli.cc lnav_commands.cc lnav_config.cc lnav_util.cc @@ -301,6 +306,8 @@ add_library( readline_highlighters.cc readline_possibilities.cc regexp_vtab.cc + regex101.client.cc + regex101.import.cc relative_time.cc session_data.cc sequence_matcher.cc @@ -347,6 +354,7 @@ add_library( column_namer.hh curl_looper.hh doc_status_source.hh + dump_internals.hh elem_to_json.hh field_overlay_source.hh file_collection.hh @@ -364,6 +372,8 @@ add_library( hotkeys.hh input_dispatcher.hh k_merge_tree.h + lnav.indexing.hh + lnav.management_cli.hh lnav_config.hh lnav_config_fwd.hh log_actions.hh @@ -390,6 +400,8 @@ add_library( readline_callbacks.hh readline_context.hh readline_possibilities.hh + regex101.client.hh + regex101.import.hh regexp_vtab.hh relative_time.hh styling.hh @@ -422,6 +434,7 @@ add_library( url_loader.hh view_helpers.hh view_helpers.examples.hh + view_helpers.hist.hh views_vtab.hh vis_line.hh vtab_module.hh @@ -439,7 +452,24 @@ add_library( ghc/fs_std_fwd.hpp ghc/fs_std_impl.hpp ww898/cp_utf8.hpp - log_level_re.cc) + log_level_re.cc + + third-party/CLI/StringTools.hpp + third-party/CLI/App.hpp + third-party/CLI/Macros.hpp + third-party/CLI/Option.hpp + third-party/CLI/Config.hpp + third-party/CLI/CLI.hpp + third-party/CLI/Formatter.hpp + third-party/CLI/Error.hpp + third-party/CLI/Version.hpp + third-party/CLI/Timer.hpp + third-party/CLI/FormatterFwd.hpp + third-party/CLI/Validators.hpp + third-party/CLI/Split.hpp + third-party/CLI/TypeTools.hpp + third-party/CLI/ConfigFwd.hpp +) set(lnav_SRCS lnav.cc) diff --git a/src/Makefile.am b/src/Makefile.am index e7618970..24281252 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -170,6 +170,7 @@ noinst_HEADERS = \ data_parser.hh \ db_sub_source.hh \ doc_status_source.hh \ + dump_internals.hh \ elem_to_json.hh \ environ_vtab.hh \ field_overlay_source.hh \ @@ -196,6 +197,8 @@ noinst_HEADERS = \ line_buffer.hh \ listview_curses.hh \ lnav.hh \ + lnav.indexing.hh \ + lnav.management_cli.hh \ lnav_commands.hh \ lnav_config.hh \ lnav_config_fwd.hh \ @@ -233,6 +236,8 @@ noinst_HEADERS = \ readline_curses.hh \ readline_highlighters.hh \ readline_possibilities.hh \ + regex101.client.hh \ + regex101.import.hh \ regexp_vtab.hh \ relative_time.hh \ ring_span.hh \ @@ -271,6 +276,7 @@ noinst_HEADERS = \ view_curses.hh \ view_helpers.hh \ view_helpers.examples.hh \ + view_helpers.hist.hh \ views_vtab.hh \ vis_line.hh \ vt52_curses.hh \ @@ -295,6 +301,21 @@ nodist_libdiag_a_SOURCES = \ THIRD_PARTY_SRCS = \ third-party/backward-cpp/backward.hpp \ + third-party/CLI/StringTools.hpp \ + third-party/CLI/App.hpp \ + third-party/CLI/Macros.hpp \ + third-party/CLI/Option.hpp \ + third-party/CLI/Config.hpp \ + third-party/CLI/CLI.hpp \ + third-party/CLI/Formatter.hpp \ + third-party/CLI/Error.hpp \ + third-party/CLI/Version.hpp \ + third-party/CLI/Timer.hpp \ + third-party/CLI/FormatterFwd.hpp \ + third-party/CLI/Validators.hpp \ + third-party/CLI/Split.hpp \ + third-party/CLI/TypeTools.hpp \ + third-party/CLI/ConfigFwd.hpp \ third-party/doctest-root/doctest/doctest.h \ third-party/sqlite/ext/dbdump.c \ third-party/sqlite/ext/series.c @@ -312,6 +333,7 @@ libdiag_a_SOURCES = \ data_scanner.cc \ data_scanner_re.cc \ db_sub_source.cc \ + dump_internals.cc \ elem_to_json.cc \ environ_vtab.cc \ extension-functions.cc \ @@ -359,6 +381,8 @@ libdiag_a_SOURCES = \ readline_curses.cc \ readline_highlighters.cc \ readline_possibilities.cc \ + regex101.client.cc \ + regex101.import.cc \ regexp_vtab.cc \ relative_time.cc \ session_data.cc \ @@ -398,9 +422,18 @@ PLUGIN_SRCS = \ lnav.$(OBJEXT): help-txt.h init-sql.h -lnav_SOURCES = lnav.cc $(PLUGIN_SRCS) +lnav_SOURCES = \ + lnav.cc \ + lnav.indexing.cc \ + lnav.management_cli.cc \ + $(PLUGIN_SRCS) -lnav_test_SOURCES = lnav.cc test_override.c $(PLUGIN_SRCS) +lnav_test_SOURCES = \ + lnav.cc \ + lnav.indexing.cc \ + lnav.management_cli.cc \ + test_override.c \ + $(PLUGIN_SRCS) ptimec$(BUILD_EXEEXT): ptimec.c $(AM_V_CC) $(CC_FOR_BUILD) $(CPPFLAGS_FOR_BUILD) $(LDFLAGS_FOR_BUILD) -g3 -o $@ $? @@ -419,6 +452,9 @@ DISTCLEANFILES = \ $(LNAV_BUILT_FILES) \ $(RE2C_FILES) +distclean-local: + $(RM_V)rm -rf *.dSYM + uncrusty: (cd $(srcdir) && uncrustify -c ../lnav.cfg --replace $(SOURCES) \ $(HEADERS)) diff --git a/src/base/CMakeLists.txt b/src/base/CMakeLists.txt index c1fa9a15..7cf7bf11 100644 --- a/src/base/CMakeLists.txt +++ b/src/base/CMakeLists.txt @@ -40,7 +40,9 @@ add_library( intern_string.hh is_utf8.hh isc.hh + itertools.hh lnav.console.hh + log_level_enum.hh lrucache.hpp math_util.hh network.tcp.hh @@ -56,6 +58,7 @@ target_link_libraries(base cppfmt pcre::libpcre ncurses::libcurses pthread) add_executable( test_base + fs_util.tests.cc humanize.file_size.tests.cc humanize.network.tests.cc humanize.time.tests.cc diff --git a/src/base/Makefile.am b/src/base/Makefile.am index aad3eabd..1f5509a5 100644 --- a/src/base/Makefile.am +++ b/src/base/Makefile.am @@ -38,9 +38,11 @@ noinst_HEADERS = \ intern_string.hh \ is_utf8.hh \ isc.hh \ + itertools.hh \ lnav_log.hh \ lnav.console.hh \ lnav.gzip.hh \ + log_level_enum.hh \ lrucache.hpp \ math_util.hh \ network.tcp.hh \ @@ -78,6 +80,7 @@ check_PROGRAMS = \ test_base test_base_SOURCES = \ + fs_util.tests.cc \ humanize.file_size.tests.cc \ humanize.network.tests.cc \ humanize.time.tests.cc \ diff --git a/src/base/attr_line.cc b/src/base/attr_line.cc index 0fc0bd92..ec548fb1 100644 --- a/src/base/attr_line.cc +++ b/src/base/attr_line.cc @@ -190,7 +190,7 @@ attr_line_t::subline(size_t start, size_t len) const lr.intersection(sa.sa_range).shift(lr.lr_start, -lr.lr_start), std::make_pair(sa.sa_type, sa.sa_value)); - line_range& last_lr = retval.al_attrs.back().sa_range; + const auto& last_lr = retval.al_attrs.back().sa_range; ensure(last_lr.lr_end <= (int) retval.al_string.length()); } @@ -300,6 +300,18 @@ attr_line_t::erase(size_t pos, size_t len) return *this; } +attr_line_t& +attr_line_t::pad_to(size_t size) +{ + const auto curr_len = this->length(); + + if (curr_len < size) { + this->append((size - curr_len), ' '); + } + + return *this; +} + line_range line_range::intersection(const line_range& other) const { diff --git a/src/base/attr_line.hh b/src/base/attr_line.hh index 404a8c38..42b5c38b 100644 --- a/src/base/attr_line.hh +++ b/src/base/attr_line.hh @@ -526,6 +526,23 @@ public: return *this; } + template + attr_line_t& join(const C& container, + const string_attr_pair& sap, + const char* fill) + { + bool init = true; + for (const auto& elem : container) { + if (!init) { + this->append(fill); + } + this->append(std::make_pair(elem, sap)); + init = false; + } + + return *this; + } + attr_line_t& insert(size_t index, const attr_line_t& al, text_wrap_settings* tws = nullptr); @@ -560,6 +577,26 @@ public: return *this; } + template + attr_line_t& add_header(Args... args) + { + if (!this->blank()) { + this->insert(0, args...); + } + return *this; + } + + template + attr_line_t& with_default(Args... args) + { + if (this->blank()) { + this->clear(); + this->append(args...); + } + + return *this; + } + attr_line_t& erase(size_t pos, size_t len = std::string::npos); attr_line_t& rtrim(); @@ -574,6 +611,8 @@ public: attr_line_t& right_justify(unsigned long width); + attr_line_t& pad_to(size_t size); + ssize_t length() const { size_t retval = this->al_string.length(); diff --git a/src/base/file_range.hh b/src/base/file_range.hh index 33548f77..9d285113 100644 --- a/src/base/file_range.hh +++ b/src/base/file_range.hh @@ -62,15 +62,17 @@ public: struct source_location { source_location() - : sl_source(intern_string::lookup("unknown")), sl_line_number(-1) + : sl_source(intern_string::lookup("unknown")), sl_line_number(0) { } - source_location(intern_string_t source, int line) - : sl_source(source), sl_line_number(line){}; + explicit source_location(intern_string_t source, int32_t line = 0) + : sl_source(source), sl_line_number(line) + { + } intern_string_t sl_source; - int sl_line_number; + int32_t sl_line_number; }; #endif diff --git a/src/base/fs_util.cc b/src/base/fs_util.cc index ff1dcfc1..dd2e6ec9 100644 --- a/src/base/fs_util.cc +++ b/src/base/fs_util.cc @@ -31,6 +31,7 @@ #include "config.h" #include "fmt/format.h" +#include "itertools.hh" #include "opt_util.hh" namespace lnav { @@ -87,25 +88,56 @@ read_file(const ghc::filesystem::path& path) } } -std::string -build_path(const std::vector& paths) +Result +write_file(const ghc::filesystem::path& path, const string_fragment& content) { - std::string retval; + auto tmp_pattern = path; + tmp_pattern += ".XXXXXX"; - for (const auto& path : paths) { - if (path.empty()) { - continue; - } - if (!retval.empty()) { - retval += ":"; - } - retval += path.string(); + auto tmp_pair = TRY(open_temp_file(tmp_pattern)); + auto bytes_written + = write(tmp_pair.second.get(), content.data(), content.length()); + if (bytes_written < 0) { + return Err( + fmt::format(FMT_STRING("unable to write to temporary file {}: {}"), + tmp_pair.first.string(), + strerror(errno))); + } + if (bytes_written != content.length()) { + return Err(fmt::format(FMT_STRING("short write to file {}: {} < {}"), + tmp_pair.first.string(), + bytes_written, + content.length())); } - auto env_path = getenv_opt("PATH"); - if (env_path) { - retval += ":" + std::string(*env_path); + std::error_code ec; + ghc::filesystem::rename(tmp_pair.first, path, ec); + if (ec) { + return Err( + fmt::format(FMT_STRING("unable to move temporary file {}: {}"), + tmp_pair.first.string(), + ec.message())); } - return retval; + + return Ok(); +} + +std::string +build_path(const std::vector& paths) +{ + return paths + | lnav::itertools::map( + static_cast( + &ghc::filesystem::path::string)) + | lnav::itertools::append(getenv_opt("PATH").value_or("")) + | lnav::itertools::filter_out(&std::string::empty) + | lnav::itertools::fold( + [](const auto& elem, auto& accum) { + if (!accum.empty()) { + accum.push_back(':'); + } + return accum.append(elem); + }, + std::string()); } Result diff --git a/src/base/fs_util.hh b/src/base/fs_util.hh index 61243998..6b0dffd8 100644 --- a/src/base/fs_util.hh +++ b/src/base/fs_util.hh @@ -35,6 +35,7 @@ #include "auto_fd.hh" #include "ghc/filesystem.hpp" +#include "intern_string.hh" #include "result.h" namespace lnav { @@ -69,6 +70,9 @@ Result, std::string> open_temp_file( Result read_file(const ghc::filesystem::path& path); +Result write_file(const ghc::filesystem::path& path, + const string_fragment& content); + std::string build_path(const std::vector& paths); } // namespace filesystem diff --git a/src/base/fs_util.tests.cc b/src/base/fs_util.tests.cc new file mode 100644 index 00000000..9ed33776 --- /dev/null +++ b/src/base/fs_util.tests.cc @@ -0,0 +1,56 @@ +/** + * Copyright (c) 2022, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "base/fs_util.hh" + +#include "config.h" +#include "doctest/doctest.h" + +TEST_CASE("fs_util::build_path") +{ + auto* old_path = getenv("PATH"); + unsetenv("PATH"); + + CHECK("" == lnav::filesystem::build_path({})); + + CHECK("/bin:/usr/bin" + == lnav::filesystem::build_path({"", "/bin", "/usr/bin", ""})); + setenv("PATH", "/usr/local/bin", 1); + CHECK("/bin:/usr/bin:/usr/local/bin" + == lnav::filesystem::build_path({"", "/bin", "/usr/bin", ""})); + setenv("PATH", "/usr/local/bin:/opt/bin", 1); + CHECK("/usr/local/bin:/opt/bin" == lnav::filesystem::build_path({})); + CHECK("/bin:/usr/bin:/usr/local/bin:/opt/bin" + == lnav::filesystem::build_path({"", "/bin", "/usr/bin", ""})); + if (old_path != nullptr) { + setenv("PATH", old_path, 1); + } +} diff --git a/src/base/func_util.hh b/src/base/func_util.hh index ee910bf7..ec78ada0 100644 --- a/src/base/func_util.hh +++ b/src/base/func_util.hh @@ -64,4 +64,30 @@ struct noop_func { } }; +namespace lnav { +namespace func { + +template>{}, int> = 0> +constexpr decltype(auto) +invoke(Fn&& f, Args&&... args) noexcept( + noexcept(std::mem_fn(f)(std::forward(args)...))) +{ + return std::mem_fn(f)(std::forward(args)...); +} + +template>{}, int> = 0> +constexpr decltype(auto) +invoke(Fn&& f, Args&&... args) noexcept( + noexcept(std::forward(f)(std::forward(args)...))) +{ + return std::forward(f)(std::forward(args)...); +} + +} // namespace func +} // namespace lnav + #endif diff --git a/src/base/intern_string.cc b/src/base/intern_string.cc index 5dc7e845..f11f6fe9 100644 --- a/src/base/intern_string.cc +++ b/src/base/intern_string.cc @@ -185,3 +185,20 @@ string_fragment::consume_n(int amount) const this->sf_end, }; } + +std::vector +string_fragment::split_lines() const +{ + std::vector retval; + int start = this->sf_begin; + + for (auto index = start; index < this->sf_end; index++) { + if ((*this)[index] == '\n') { + retval.emplace_back(this->sf_string, start, index + 1); + start = index + 1; + } + } + retval.emplace_back(this->sf_string, start, this->sf_end); + + return retval; +} diff --git a/src/base/intern_string.hh b/src/base/intern_string.hh index c17cc0e8..71effc93 100644 --- a/src/base/intern_string.hh +++ b/src/base/intern_string.hh @@ -33,6 +33,7 @@ #define intern_string_hh #include +#include #include #include @@ -239,13 +240,12 @@ struct string_fragment { }); } + std::vector split_lines() const; + struct tag1 { const char t_value; - bool operator()(char ch) const - { - return this->t_value == ch; - } + bool operator()(char ch) const { return this->t_value == ch; } }; struct quoted_string_body { @@ -367,20 +367,11 @@ public: { } - const intern_string* unwrap() const - { - return this->ist_interned_string; - } + const intern_string* unwrap() const { return this->ist_interned_string; } - void clear() - { - this->ist_interned_string = nullptr; - }; + void clear() { this->ist_interned_string = nullptr; }; - bool empty() const - { - return this->ist_interned_string == nullptr; - } + bool empty() const { return this->ist_interned_string == nullptr; } const char* get() const { @@ -390,15 +381,11 @@ public: return this->ist_interned_string->get(); } - iterator begin() const - { - return this->get(); - } + const char* c_str() const { return this->get(); } - iterator end() const - { - return this->get() + this->size(); - } + iterator begin() const { return this->get(); } + + iterator end() const { return this->get() + this->size(); } size_t size() const { @@ -456,6 +443,11 @@ public: return strcmp(this->get(), rhs) != 0; } + static bool case_lt(const intern_string_t& lhs, const intern_string_t& rhs) + { + return strnatcasecmp(lhs.size(), lhs.get(), rhs.size(), rhs.get()) < 0; + } + private: const intern_string* ist_interned_string; }; diff --git a/src/base/intern_string.tests.cc b/src/base/intern_string.tests.cc index 93b299e5..82e6c5b0 100644 --- a/src/base/intern_string.tests.cc +++ b/src/base/intern_string.tests.cc @@ -35,6 +35,29 @@ #include "config.h" #include "doctest/doctest.h" +TEST_CASE("split_lines") +{ + std::string in1 = "Hello, World!"; + std::string in2 = "Hello, World!\nGoodbye, World!"; + + { + auto sf = string_fragment(in1); + auto split = sf.split_lines(); + + CHECK(1 == split.size()); + CHECK(in1 == split[0].to_string()); + } + + { + auto sf = string_fragment(in2); + auto split = sf.split_lines(); + + CHECK(2 == split.size()); + CHECK("Hello, World!\n" == split[0].to_string()); + CHECK("Goodbye, World!" == split[1].to_string()); + } +} + TEST_CASE("consume") { auto is_eq = string_fragment::tag1{'='}; diff --git a/src/base/itertools.hh b/src/base/itertools.hh new file mode 100644 index 00000000..2d0f038a --- /dev/null +++ b/src/base/itertools.hh @@ -0,0 +1,400 @@ + +#ifndef lnav_itertools_hh +#define lnav_itertools_hh + +#include +#include +#include + +#include "func_util.hh" +#include "optional.hpp" + +namespace lnav { +namespace itertools { + +struct empty {}; + +struct not_empty {}; + +struct full { + size_t f_max_size; +}; + +namespace details { + +template +struct unwrap_or { + T uo_value; +}; + +template +struct find_if { + P fi_predicate; +}; + +template +struct find { + T f_value; +}; + +template +struct filter_in { + F f_func; +}; + +template +struct filter_out { + F f_func; +}; + +template +struct sort_by { + C sb_cmp; +}; + +struct sorted {}; + +template +struct mapper { + F m_func; +}; + +template +struct folder { + R f_func; + T f_init; +}; + +template +struct prepend { + T p_value; +}; + +template +struct append { + T p_value; +}; + +} // namespace details + +template +inline details::unwrap_or +unwrap_or(T value) +{ + return details::unwrap_or{ + value, + }; +} + +template +inline details::find_if

+find_if(P predicate) +{ + return details::find_if

{ + predicate, + }; +} + +template +inline details::find +find(T value) +{ + return details::find{ + value, + }; +} + +template +inline details::filter_in +filter_in(F func) +{ + return details::filter_in{ + func, + }; +} + +template +inline details::filter_out +filter_out(F func) +{ + return details::filter_out{ + func, + }; +} + +template +inline details::prepend +prepend(T value) +{ + return details::prepend{ + std::move(value), + }; +} + +template +inline details::append +append(T value) +{ + return details::append{ + std::move(value), + }; +} + +template +inline details::sort_by +sort_with(C cmp) +{ + return details::sort_by{cmp}; +} + +template +inline auto +sort_by(T C::*m) +{ + return sort_with( + [m](const C& lhs, const C& rhs) { return lhs.*m < rhs.*m; }); +} + +template +inline details::mapper +map(F func) +{ + return details::mapper{func}; +} + +template +inline details::folder +fold(R func, T init) +{ + return details::folder{func, init}; +} + +inline details::sorted +sorted() +{ + return details::sorted{}; +} + +template +T +chain(const T& value1, const Args&... args) +{ + T retval; + + for (const auto& arg : {value1, args...}) { + for (const auto& elem : arg) { + retval.emplace_back(elem); + } + } + + return retval; +} + +} // namespace itertools +} // namespace lnav + +template +nonstd::optional +operator|(const C& in, const lnav::itertools::details::find_if

& finder) +{ + for (const auto& elem : in) { + if (lnav::func::invoke(finder.fi_predicate, elem)) { + return nonstd::make_optional(elem); + } + } + + return nonstd::nullopt; +} + +template +nonstd::optional +operator|(const C& in, const lnav::itertools::details::find& finder) +{ + size_t retval = 0; + for (const auto& elem : in) { + if (elem == finder.f_value) { + return nonstd::make_optional(retval); + } + retval += 1; + } + + return nonstd::nullopt; +} + +template +C +operator|(const C& in, const lnav::itertools::details::filter_in& filterer) +{ + C retval; + + for (const auto& elem : in) { + if (lnav::func::invoke(filterer.f_func, elem)) { + retval.emplace_back(elem); + } + } + + return retval; +} + +template +C +operator|(const C& in, const lnav::itertools::details::filter_out& filterer) +{ + C retval; + + for (const auto& elem : in) { + if (!lnav::func::invoke(filterer.f_func, elem)) { + retval.emplace_back(elem); + } + } + + return retval; +} + +template +C +operator|(C in, const lnav::itertools::details::prepend& prepender) +{ + in.emplace(in.begin(), prepender.p_value); + + return in; +} + +template +C +operator|(C in, const lnav::itertools::details::append& appender) +{ + in.emplace_back(appender.p_value); + + return in; +} + +template +T +operator|(const C& in, const lnav::itertools::details::folder& folder) +{ + auto accum = folder.f_init; + + for (const auto& elem : in) { + accum = folder.f_func(elem, accum); + } + + return accum; +} + +template +T +operator|(T in, const lnav::itertools::details::sort_by& sorter) +{ + std::sort(in.begin(), in.end(), sorter.sb_cmp); + + return in; +} + +template +T +operator|(T in, const lnav::itertools::details::sorted& sorter) +{ + std::sort(in.begin(), in.end()); + + return in; +} + +template +auto +operator|(const T& in, const lnav::itertools::details::mapper& mapper) + -> std::vector +{ + using return_type + = std::vector; + return_type retval; + + retval.reserve(in.size()); + std::transform( + in.begin(), in.end(), std::back_inserter(retval), mapper.m_func); + + return retval; +} + +template +auto +operator|(const std::vector>& in, + const lnav::itertools::details::mapper& mapper) + -> std::vector> +{ + using return_type = std::vector>; + return_type retval; + + retval.reserve(in.size()); + std::transform( + in.begin(), + in.end(), + std::back_inserter(retval), + [&mapper](const auto& elem) { return ((*elem).*mapper.m_func)(); }); + + return retval; +} + +template +auto +operator|(const std::vector>& in, + const lnav::itertools::details::mapper& mapper) + -> std::vector>> +{ + using return_type = std::vector< + typename std::remove_reference_t>>; + return_type retval; + + retval.reserve(in.size()); + for (const auto& elem : in) { + retval.template emplace_back(((*elem).*mapper.m_func)); + } + + return retval; +} + +template +auto +operator|(nonstd::optional in, + const lnav::itertools::details::mapper& mapper) + -> nonstd::optional>> +{ + if (!in) { + return nonstd::nullopt; + } + + return nonstd::make_optional((in.value()).*mapper.m_func); +} + +template +T +operator|(nonstd::optional in, + const lnav::itertools::details::unwrap_or& unwrapper) +{ + return in.value_or(unwrapper.uo_value); +} + +template +auto +operator|(const T& in, const lnav::itertools::details::mapper& mapper) + -> std::vector> +{ + using return_type = std::vector>; + return_type retval; + + retval.reserve(in.size()); + for (const auto& elem : in) { + retval.template emplace_back((elem.*mapper.m_func)()); + } + + return retval; +} + +#endif diff --git a/src/base/lnav.console.cc b/src/base/lnav.console.cc index 6d4d0ffc..608e29d6 100644 --- a/src/base/lnav.console.cc +++ b/src/base/lnav.console.cc @@ -33,11 +33,25 @@ #include "config.h" #include "fmt/color.h" +#include "itertools.hh" +#include "log_level_enum.hh" #include "view_curses.hh" +using namespace lnav::roles::literals; + namespace lnav { namespace console { +user_message +user_message::raw(const attr_line_t& al) +{ + user_message retval; + + retval.um_level = level::raw; + retval.um_message.append(al); + return retval; +} + user_message user_message::error(const attr_line_t& al) { @@ -81,10 +95,17 @@ user_message::warning(const attr_line_t& al) attr_line_t user_message::to_attr_line(std::set flags) const { + auto indent = 1; attr_line_t retval; + if (this->um_level == level::warning) { + indent = 3; + } + if (flags.count(render_flags::prefix)) { switch (this->um_level) { + case level::raw: + break; case level::ok: retval.append(lnav::roles::ok("\u2714 ")); break; @@ -105,13 +126,18 @@ user_message::to_attr_line(std::set flags) const if (!this->um_reason.empty()) { bool first_line = true; for (const auto& line : this->um_reason.split_lines()) { + auto role = this->um_level == level::error ? role_t::VCR_ERROR + : role_t::VCR_WARNING; attr_line_t prefix; if (first_line) { - prefix.append(lnav::roles::error(" reason")).append(": "); + prefix.append(indent, ' ') + .append("reason", VC_ROLE.value(role)) + .append(": "); first_line = false; } else { - prefix.append(lnav::roles::error(" | ")); + prefix.append(" | ", VC_ROLE.value(role)) + .append(indent, ' '); } retval.append(prefix).append(line).append("\n"); } @@ -120,35 +146,34 @@ user_message::to_attr_line(std::set flags) const for (const auto& snip : this->um_snippets) { attr_line_t header; - header.append(lnav::roles::comment(" --> ")) - .append(lnav::roles::file(snip.s_source)); - if (snip.s_line > 0) { - header.append(":").append(FMT_STRING("{}"), snip.s_line); - if (snip.s_column > 0) { - header.append(":").append(FMT_STRING("{}"), snip.s_column); - } + header.append(" --> "_comment) + .append(lnav::roles::file(snip.s_location.sl_source.get())); + if (snip.s_location.sl_line_number > 0) { + header.append(":").append(FMT_STRING("{}"), + snip.s_location.sl_line_number); } retval.append(header).append("\n"); if (!snip.s_content.blank()) { for (const auto& line : snip.s_content.split_lines()) { - retval.append(lnav::roles::comment(" | ")) - .append(line) - .append("\n"); + retval.append(" | "_comment).append(line).append("\n"); } } } } if (!this->um_notes.empty()) { - bool first_line = true; for (const auto& note : this->um_notes) { + bool first_line = true; for (const auto& line : note.split_lines()) { attr_line_t prefix; if (first_line) { - prefix.append(lnav::roles::comment(" = note")).append(": "); + prefix.append(" ="_comment) + .append(indent, ' ') + .append("note"_comment) + .append(": "); first_line = false; } else { - prefix.append(" "); + prefix.append(" ").append(indent, ' '); } retval.append(prefix).append(line).append("\n"); @@ -161,7 +186,10 @@ user_message::to_attr_line(std::set flags) const attr_line_t prefix; if (first_line) { - prefix.append(lnav::roles::comment(" = help")).append(": "); + prefix.append(" ="_comment) + .append(indent, ' ') + .append("help"_comment) + .append(": "); first_line = false; } else { prefix.append(" "); @@ -174,85 +202,152 @@ user_message::to_attr_line(std::set flags) const return retval; } +fmt::terminal_color +curses_color_to_terminal_color(int curses_color) +{ + switch (curses_color) { + case COLOR_BLACK: + return fmt::terminal_color::black; + case COLOR_CYAN: + return fmt::terminal_color::cyan; + case COLOR_WHITE: + return fmt::terminal_color::white; + case COLOR_MAGENTA: + return fmt::terminal_color::magenta; + case COLOR_BLUE: + return fmt::terminal_color::blue; + case COLOR_YELLOW: + return fmt::terminal_color::yellow; + case COLOR_GREEN: + return fmt::terminal_color::green; + case COLOR_RED: + return fmt::terminal_color::red; + } + + ensure(false); +} + void println(FILE* file, const attr_line_t& al) { const auto& str = al.get_string(); - if (!isatty(fileno(file))) { + if (getenv("NO_COLOR") != nullptr + || (!isatty(fileno(file)) && getenv("YES_COLOR") == nullptr)) + { fmt::print(file, "{}\n", str); return; } - string_attrs_t style_attrs; + std::set points = {0, (int) al.length()}; - for (const auto& sa : al.get_attrs()) { - if (sa.sa_type != &VC_ROLE) { + for (const auto& attr : al.get_attrs()) { + if (!attr.sa_range.is_valid()) { continue; } - - style_attrs.emplace_back(sa); + points.insert(attr.sa_range.lr_start); + if (attr.sa_range.lr_end > 0) { + points.insert(attr.sa_range.lr_end); + } } - std::sort(style_attrs.begin(), style_attrs.end(), [](auto lhs, auto rhs) { - return lhs.sa_range < rhs.sa_range; - }); - - auto start = size_t{0}; - for (const auto& attr : style_attrs) { - fmt::print( - file, "{}", str.substr(start, attr.sa_range.lr_start - start)); - if (attr.sa_type == &VC_ROLE) { - auto saw = string_attr_wrapper(&attr); - auto role = saw.get(); - auto line_style = fmt::text_style(); - - switch (role) { - case role_t::VCR_ERROR: - line_style = fmt::fg(fmt::terminal_color::red); - break; - case role_t::VCR_WARNING: - line_style = fmt::fg(fmt::terminal_color::yellow); - break; - case role_t::VCR_COMMENT: - line_style = fmt::fg(fmt::terminal_color::cyan); - break; - case role_t::VCR_OK: - line_style = fmt::emphasis::bold - | fmt::fg(fmt::terminal_color::red); - break; - case role_t::VCR_STATUS: - line_style = fmt::emphasis::bold - | fmt::fg(fmt::terminal_color::magenta); - break; - case role_t::VCR_VARIABLE: - line_style = fmt::emphasis::underline; - break; - case role_t::VCR_SYMBOL: - case role_t::VCR_NUMBER: - case role_t::VCR_FILE: - line_style = fmt::emphasis::bold; - break; - case role_t::VCR_H1: - case role_t::VCR_H2: - case role_t::VCR_H3: - case role_t::VCR_H4: - case role_t::VCR_H5: - case role_t::VCR_H6: - line_style = fmt::emphasis::underline; - break; - default: - break; + nonstd::optional last_point; + for (const auto& point : points) { + if (last_point) { + auto line_style = fmt::text_style{}; + auto fg_style = fmt::text_style{}; + auto start = last_point.value(); + + for (const auto& attr : al.get_attrs()) { + if (!attr.sa_range.contains(start) + && !attr.sa_range.contains(point - 1)) { + continue; + } + + if (attr.sa_type == &VC_BACKGROUND) { + auto saw = string_attr_wrapper(&attr); + auto color = saw.get(); + + if (color >= 0) { + line_style + |= fmt::bg(curses_color_to_terminal_color(color)); + } + } else if (attr.sa_type == &VC_FOREGROUND) { + auto saw = string_attr_wrapper(&attr); + auto color = saw.get(); + + if (color >= 0) { + fg_style + = fmt::fg(curses_color_to_terminal_color(color)); + } + } else if (attr.sa_type == &VC_ROLE) { + auto saw = string_attr_wrapper(&attr); + auto role = saw.get(); + + switch (role) { + case role_t::VCR_ERROR: + line_style |= fmt::fg(fmt::terminal_color::red); + break; + case role_t::VCR_WARNING: + line_style |= fmt::fg(fmt::terminal_color::yellow); + break; + case role_t::VCR_COMMENT: + line_style |= fmt::fg(fmt::terminal_color::cyan); + break; + case role_t::VCR_OK: + line_style |= fmt::emphasis::bold + | fmt::fg(fmt::terminal_color::green); + break; + case role_t::VCR_STATUS: + line_style |= fmt::emphasis::bold + | fmt::fg(fmt::terminal_color::magenta); + break; + case role_t::VCR_KEYWORD: + line_style |= fmt::emphasis::bold + | fmt::fg(fmt::terminal_color::blue); + break; + case role_t::VCR_VARIABLE: + line_style |= fmt::emphasis::underline; + break; + case role_t::VCR_SYMBOL: + case role_t::VCR_NUMBER: + case role_t::VCR_FILE: + line_style |= fmt::emphasis::bold; + break; + case role_t::VCR_H1: + line_style |= fmt::emphasis::bold + | fmt::fg(fmt::terminal_color::magenta); + break; + case role_t::VCR_H2: + line_style |= fmt::emphasis::bold; + break; + case role_t::VCR_H3: + case role_t::VCR_H4: + case role_t::VCR_H5: + case role_t::VCR_H6: + line_style |= fmt::emphasis::underline; + break; + case role_t::VCR_LIST_GLYPH: + line_style |= fmt::fg(fmt::terminal_color::yellow); + break; + default: + break; + } + } } - fmt::print( - file, - line_style, - "{}", - str.substr(attr.sa_range.lr_start, attr.sa_range.length())); + + if (!line_style.has_foreground() && fg_style.has_foreground()) { + line_style |= fg_style; + } + + fmt::print(file, + line_style, + FMT_STRING("{}"), + str.substr(start, point - start)); } - start = attr.sa_range.lr_end; + last_point = point; } - fmt::print(file, "{}\n", str.substr(start)); + fmt::print(file, "\n"); } void diff --git a/src/base/lnav.console.hh b/src/base/lnav.console.hh index b03f8600..0b024204 100644 --- a/src/base/lnav.console.hh +++ b/src/base/lnav.console.hh @@ -34,6 +34,7 @@ #include #include "base/attr_line.hh" +#include "base/file_range.hh" namespace lnav { namespace console { @@ -41,41 +42,45 @@ namespace console { void println(FILE* file, const attr_line_t& al); struct snippet { - static snippet from(std::string src, const attr_line_t& content) + static snippet from(intern_string_t src, const attr_line_t& content) { snippet retval; - retval.s_source = std::move(src); + retval.s_location.sl_source = src; retval.s_content = content; return retval; } - snippet& with_line(int32_t line) + static snippet from(source_location loc, const attr_line_t& content) { - this->s_line = line; - return *this; + snippet retval; + + retval.s_location = loc; + retval.s_content = content; + return retval; } - snippet& with_column(int32_t column) + snippet& with_line(int32_t line) { - this->s_column = column; + this->s_location.sl_line_number = line; return *this; } - std::string s_source; - int32_t s_line{0}; - int32_t s_column{0}; + source_location s_location; attr_line_t s_content; }; struct user_message { enum class level { + raw, ok, info, warning, error, }; + static user_message raw(const attr_line_t& al); + static user_message error(const attr_line_t& al); static user_message warning(const attr_line_t& al); @@ -91,6 +96,11 @@ struct user_message { return *this; } + user_message& with_reason(const user_message& um) + { + return this->with_reason(um.to_attr_line({})); + } + user_message& with_errno_reason() { this->um_reason = strerror(errno); @@ -113,15 +123,21 @@ struct user_message { user_message& with_note(const attr_line_t& al) { - this->um_notes.emplace_back(al); + if (!al.blank()) { + this->um_notes.emplace_back(al); + } return *this; } user_message& with_help(const attr_line_t& al) { - this->um_help = al; - this->um_help.rtrim(); + if (al.blank()) { + this->um_help.clear(); + } else { + this->um_help = al; + this->um_help.rtrim(); + } return *this; } diff --git a/src/base/log_level_enum.hh b/src/base/log_level_enum.hh new file mode 100644 index 00000000..983fd5ce --- /dev/null +++ b/src/base/log_level_enum.hh @@ -0,0 +1,65 @@ +/** + * Copyright (c) 2020, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef lnav_log_level_enum_hh +#define lnav_log_level_enum_hh + +/** + * The logging level identifiers for a line(s). + */ +enum log_level_t : int { + LEVEL_UNKNOWN, + LEVEL_TRACE, + LEVEL_DEBUG5, + LEVEL_DEBUG4, + LEVEL_DEBUG3, + LEVEL_DEBUG2, + LEVEL_DEBUG, + LEVEL_INFO, + LEVEL_STATS, + LEVEL_NOTICE, + LEVEL_WARNING, + LEVEL_ERROR, + LEVEL_CRITICAL, + LEVEL_FATAL, + LEVEL_INVALID, + + LEVEL__MAX, + + LEVEL_IGNORE = 0x10, /*< Ignore */ + LEVEL_TIME_SKEW = 0x20, /*< Received after timestamp. */ + LEVEL_MARK = 0x40, /*< Bookmarked line. */ + LEVEL_CONTINUED = 0x80, /*< Continuation of multiline entry. */ + + /** Mask of flags for the level field. */ + LEVEL__FLAGS + = (LEVEL_IGNORE | LEVEL_TIME_SKEW | LEVEL_MARK | LEVEL_CONTINUED) +}; + +#endif diff --git a/src/base/string_attr_type.cc b/src/base/string_attr_type.cc index 956003f1..933e2b19 100644 --- a/src/base/string_attr_type.cc +++ b/src/base/string_attr_type.cc @@ -38,6 +38,7 @@ string_attr_type SA_FORMAT("format"); string_attr_type SA_REMOVED("removed"); string_attr_type SA_INVALID("invalid"); string_attr_type SA_ERROR("error"); +string_attr_type SA_LEVEL("level"); string_attr_type VC_ROLE("role"); string_attr_type VC_ROLE_FG("role-fg"); diff --git a/src/base/string_attr_type.hh b/src/base/string_attr_type.hh index 88b0b35b..30dbbcb0 100644 --- a/src/base/string_attr_type.hh +++ b/src/base/string_attr_type.hh @@ -109,6 +109,8 @@ enum class role_t : int32_t { VCR_H5, VCR_H6, + VCR_LIST_GLYPH, + VCR__MAX }; @@ -162,6 +164,7 @@ extern string_attr_type SA_FORMAT; extern string_attr_type SA_REMOVED; extern string_attr_type SA_INVALID; extern string_attr_type SA_ERROR; +extern string_attr_type SA_LEVEL; extern string_attr_type VC_ROLE; extern string_attr_type VC_ROLE_FG; @@ -310,6 +313,13 @@ inline std::pair operator"" _symbol( VC_ROLE.template value(role_t::VCR_SYMBOL)); } +inline std::pair operator"" _keyword( + const char* str, std::size_t len) +{ + return std::make_pair(std::string(str, len), + VC_ROLE.template value(role_t::VCR_KEYWORD)); +} + inline std::pair operator"" _variable( const char* str, std::size_t len) { @@ -317,6 +327,13 @@ inline std::pair operator"" _variable( VC_ROLE.template value(role_t::VCR_VARIABLE)); } +inline std::pair operator"" _comment( + const char* str, std::size_t len) +{ + return std::make_pair(std::string(str, len), + VC_ROLE.template value(role_t::VCR_COMMENT)); +} + inline std::pair operator"" _h1(const char* str, std::size_t len) { @@ -338,6 +355,13 @@ inline std::pair operator"" _h3(const char* str, VC_ROLE.template value(role_t::VCR_H3)); } +inline std::pair operator"" _list_glyph( + const char* str, std::size_t len) +{ + return std::make_pair(std::string(str, len), + VC_ROLE.template value(role_t::VCR_LIST_GLYPH)); +} + } // namespace literals } // namespace roles diff --git a/src/base/string_util.cc b/src/base/string_util.cc index 9f6b5019..f6a1a69e 100644 --- a/src/base/string_util.cc +++ b/src/base/string_util.cc @@ -153,7 +153,7 @@ truncate_to(std::string& str, size_t max_char_len) } bool -is_url(const char* fn) +is_url(const std::string& fn) { static const auto url_re = std::regex("^(file|https?|ftps?|scp|sftp):.*"); diff --git a/src/base/string_util.hh b/src/base/string_util.hh index fe5efc7b..8b682004 100644 --- a/src/base/string_util.hh +++ b/src/base/string_util.hh @@ -105,6 +105,17 @@ trim(const std::string& str) return str.substr(start, end - start); } +inline std::string +rtrim(const std::string& str) +{ + std::string::size_type end; + + for (end = str.size(); end > 0 && isspace(str[end - 1]); end--) + ; + + return str.substr(0, end); +} + inline std::string tolower(const char* str) { @@ -186,7 +197,7 @@ utf8_string_length(const std::string& str) return utf8_string_length(str.c_str(), str.length()); } -bool is_url(const char* fn); +bool is_url(const std::string& fn); bool is_blank(const std::string& str); @@ -201,4 +212,14 @@ std::string center_str(const std::string& subject, size_t width); template size_t strtonum(T& num_out, const char* data, size_t len); +inline std::string +on_blank(const std::string& str, const std::string& def) +{ + if (is_blank(str)) { + return def; + } + + return str; +} + #endif diff --git a/src/bookmarks.cc b/src/bookmarks.cc index 19dcff5f..69eb2889 100644 --- a/src/bookmarks.cc +++ b/src/bookmarks.cc @@ -31,6 +31,7 @@ #include "bookmarks.hh" +#include "base/itertools.hh" #include "config.h" std::unordered_set bookmark_metadata::KNOWN_TAGS; @@ -38,10 +39,8 @@ std::unordered_set bookmark_metadata::KNOWN_TAGS; void bookmark_metadata::add_tag(const std::string& tag) { - if (std::find(this->bm_tags.begin(), this->bm_tags.end(), tag) - == this->bm_tags.end()) - { - this->bm_tags.push_back(tag); + if (!(this->bm_tags | lnav::itertools::find(tag))) { + this->bm_tags.emplace_back(tag); } } @@ -72,14 +71,18 @@ bookmark_metadata::clear() this->bm_tags.clear(); } -bookmark_type_t* +nonstd::optional bookmark_type_t::find_type(const std::string& name) { - auto iter = std::find_if(type_begin(), type_end(), mark_eq(name)); - bookmark_type_t* retval = nullptr; + return get_all_types() + | lnav::itertools::find_if( + [&name](const auto& elem) { return elem->bt_name == name; }); +} - if (iter != type_end()) { - retval = (*iter); - } - return retval; +std::vector& +bookmark_type_t::get_all_types() +{ + static std::vector all_types; + + return all_types; } diff --git a/src/bookmarks.hh b/src/bookmarks.hh index d72bdfc5..8cb8365e 100644 --- a/src/bookmarks.hh +++ b/src/bookmarks.hh @@ -141,47 +141,23 @@ class bookmark_type_t { public: using type_iterator = std::vector::iterator; - static type_iterator type_begin() - { - return get_all_types().begin(); - } + static type_iterator type_begin() { return get_all_types().begin(); } - static type_iterator type_end() - { - return get_all_types().end(); - } + static type_iterator type_end() { return get_all_types().end(); } - static bookmark_type_t* find_type(const std::string& name); + static nonstd::optional find_type( + const std::string& name); - static std::vector& get_all_types() - { - static std::vector all_types; - - return all_types; - } + static std::vector& get_all_types(); explicit bookmark_type_t(std::string name) : bt_name(std::move(name)) { get_all_types().push_back(this); } - const std::string& get_name() const - { - return this->bt_name; - } + const std::string& get_name() const { return this->bt_name; } private: - struct mark_eq { - explicit mark_eq(const std::string& name) : me_name(name){}; - - bool operator()(bookmark_type_t* bt) - { - return bt->bt_name == this->me_name; - }; - - const std::string& me_name; - }; - const std::string bt_name; }; diff --git a/src/bottom_status_source.hh b/src/bottom_status_source.hh index d25fd066..38e39ed2 100644 --- a/src/bottom_status_source.hh +++ b/src/bottom_status_source.hh @@ -53,20 +53,17 @@ public: bottom_status_source(); - status_field& get_field(field_t id) - { - return this->bss_fields[id]; - }; + status_field& get_field(field_t id) { return this->bss_fields[id]; } void set_prompt(const std::string& prompt) { this->bss_prompt.set_value(prompt); - }; + } void grep_error(const std::string& msg) override { this->bss_error.set_value(msg); - }; + } size_t statusview_fields() override; diff --git a/src/byte_array.hh b/src/byte_array.hh index 3233353d..5fb97ff5 100644 --- a/src/byte_array.hh +++ b/src/byte_array.hh @@ -44,32 +44,29 @@ struct byte_array { static constexpr size_t BYTE_COUNT = COUNT * sizeof(T); static constexpr size_t STRING_SIZE = BYTE_COUNT * 2 + 1; - byte_array(){}; + byte_array() {} byte_array(const byte_array& other) { memcpy(this->ba_data, other.ba_data, BYTE_COUNT); - }; + } bool operator<(const byte_array& other) const { return memcmp(this->ba_data, other.ba_data, BYTE_COUNT) < 0; - }; + } bool operator!=(const byte_array& other) const { return memcmp(this->ba_data, other.ba_data, BYTE_COUNT) != 0; - }; + } bool operator==(const byte_array& other) const { return memcmp(this->ba_data, other.ba_data, BYTE_COUNT) == 0; - }; + } - void clear() - { - memset(this->ba_data, 0, BYTE_COUNT); - }; + void clear() { memset(this->ba_data, 0, BYTE_COUNT); } template void to_string(OutputIt out) const @@ -88,17 +85,14 @@ struct byte_array { return retval; } - const unsigned char* in() const - { - return this->ba_data; - }; + const unsigned char* in() const { return this->ba_data; } T* out(int offset = 0) { T* ptr = (T*) this->ba_data; return &ptr[offset]; - }; + } unsigned char ba_data[BYTE_COUNT]; }; diff --git a/src/column_namer.cc b/src/column_namer.cc index 6b40c999..e7c12fa8 100644 --- a/src/column_namer.cc +++ b/src/column_namer.cc @@ -33,6 +33,7 @@ #include "column_namer.hh" +#include "base/itertools.hh" #include "base/lnav_log.hh" #include "base/string_util.hh" #include "config.h" @@ -47,17 +48,11 @@ column_namer::existing_name(const std::string& in_name) const return true; } - if (std::find(this->cn_builtin_names.begin(), - this->cn_builtin_names.end(), - in_name) - != this->cn_builtin_names.end()) - { + if (this->cn_builtin_names | lnav::itertools::find(in_name)) { return true; } - if (std::find(this->cn_names.begin(), this->cn_names.end(), in_name) - != this->cn_names.end()) - { + if (this->cn_names | lnav::itertools::find(in_name)) { return true; } diff --git a/src/command_executor.cc b/src/command_executor.cc index 86e9441e..74af1c62 100644 --- a/src/command_executor.cc +++ b/src/command_executor.cc @@ -40,6 +40,7 @@ #include "db_sub_source.hh" #include "help_text_formatter.hh" #include "lnav.hh" +#include "lnav.indexing.hh" #include "lnav_config.hh" #include "lnav_util.hh" #include "log_format_loader.hh" @@ -175,8 +176,7 @@ execute_sql(exec_context& ec, const std::string& sql, std::string& alt_msg) auto source = ec.ec_source.top(); sql_progress_guard progress_guard(sql_progress, sql_progress_finished, - source.s_source, - source.s_line, + source.s_location, source.s_content); gettimeofday(&start_tv, nullptr); retcode = sqlite3_prepare_v2( @@ -315,8 +315,17 @@ execute_sql(exec_context& ec, const std::string& sql, std::string& alt_msg) log_error("sqlite3_step error code: %d", retcode); errmsg = sqlite3_errmsg(lnav_data.ld_db); if (startswith(errmsg, "lnav-error:")) { - return Err(lnav::from_json( - &errmsg[11])); + auto from_res + = lnav::from_json( + &errmsg[11]); + + if (from_res.isOk()) { + return Err(from_res.unwrap()); + } + + return ec.make_error( + "internal error: {}", + from_res.unwrapErr()[0].um_message.get_string()); } return ec.make_error("{}", errmsg); } @@ -344,7 +353,7 @@ execute_sql(exec_context& ec, const std::string& sql, std::string& alt_msg) } if (lnav_data.ld_rl_view != nullptr) { - lnav_data.ld_rl_view->set_value(""); + lnav_data.ld_rl_view->clear_value(); } } @@ -607,7 +616,8 @@ execute_from_file(exec_context& ec, const std::string& cmdline) { std::string retval, alt_msg; - auto _sg = ec.enter_source(path.string(), line_number, cmdline); + auto _sg = ec.enter_source( + intern_string::lookup(path.string()), line_number, cmdline); switch (mode) { case ':': @@ -691,12 +701,15 @@ execute_init_commands( log_info("Executing initial commands"); for (auto& cmd : lnav_data.ld_commands) { + static const auto COMMAND_OPTION_SRC + = intern_string::lookup("command-option"); + std::string alt_msg; wait_for_children(); { - auto _sg = ec.enter_source("command-option", option_index++, cmd); + auto _sg = ec.enter_source(COMMAND_OPTION_SRC, option_index++, cmd); switch (cmd.at(0)) { case ':': msgs.emplace_back(execute_command(ec, cmd.substr(1)), @@ -906,10 +919,12 @@ exec_context::exec_context(std::vector* line_values, ec_accumulator(std::make_unique()), ec_sql_callback(sql_callback), ec_pipe_callback(pipe_callback) { + static const auto COMMAND_SRC = intern_string::lookup("command"); + this->ec_local_vars.push(std::map()); this->ec_path_stack.emplace_back("."); this->ec_source.emplace( - lnav::console::snippet::from("command", "").with_line(1)); + lnav::console::snippet::from(COMMAND_SRC, "").with_line(1)); this->ec_output_stack.emplace_back("screen", nonstd::nullopt); } diff --git a/src/command_executor.hh b/src/command_executor.hh index b3f767d2..b9cb735a 100644 --- a/src/command_executor.hh +++ b/src/command_executor.hh @@ -138,7 +138,7 @@ struct exec_context { exec_context& sg_context; }; - source_guard enter_source(const std::string& path, + source_guard enter_source(intern_string_t path, int line_number, const std::string& content) { diff --git a/src/curl_looper.cc b/src/curl_looper.cc index 63f68e73..78b57061 100644 --- a/src/curl_looper.cc +++ b/src/curl_looper.cc @@ -91,6 +91,17 @@ curl_request::debug_cb( return 0; } +size_t +curl_request::string_cb(void* data, size_t size, size_t nmemb, void* userp) +{ + auto realsize = size * nmemb; + auto& vec = *static_cast(userp); + + vec.append((char*) data, ((char*) data) + realsize); + + return realsize; +} + void curl_looper::loop_body() { @@ -245,4 +256,18 @@ curl_looper::compute_timeout(mstime_t current_time) const return retval; } +void +curl_looper::process_all() +{ + this->check_for_new_requests(); + + this->requeue_requests(LONG_MAX); + + while (!this->cl_handle_to_request.empty()) { + this->perform_io(); + + this->check_for_finished_requests(); + } +} + #endif diff --git a/src/curl_looper.hh b/src/curl_looper.hh index f751b325..2b350ad2 100644 --- a/src/curl_looper.hh +++ b/src/curl_looper.hh @@ -39,6 +39,8 @@ #include #include "base/isc.hh" +#include "base/lnav.console.hh" +#include "base/result.h" #include "config.h" #if !defined(HAVE_LIBCURL) @@ -73,8 +75,7 @@ public: class curl_request { public: curl_request(std::string name) - : cr_name(std::move(name)), cr_open(true), cr_handle(curl_easy_cleanup), - cr_completions(0) + : cr_name(std::move(name)), cr_handle(curl_easy_cleanup) { this->cr_handle.reset(curl_easy_init()); curl_easy_setopt(this->cr_handle, CURLOPT_NOSIGNAL, 1); @@ -91,34 +92,39 @@ public: # endif CURLSSH_AUTH_PASSWORD); } - }; + } virtual ~curl_request() = default; const std::string& get_name() const { return this->cr_name; - }; + } virtual void close() { this->cr_open = false; - }; + } bool is_open() const { return this->cr_open; - }; + } CURL* get_handle() const { return this->cr_handle; - }; + } + + operator CURL*() const + { + return this->cr_handle; + } int get_completions() const { return this->cr_completions; - }; + } virtual long complete(CURLcode result) { @@ -136,17 +142,42 @@ public: "%s: download_speed=%f", this->cr_name.c_str(), download_speed); return -1; - }; + } + + Result perform() + { + std::string response; + + curl_easy_setopt(this->get_handle(), CURLOPT_WRITEFUNCTION, string_cb); + curl_easy_setopt(this->get_handle(), CURLOPT_WRITEDATA, &response); + + auto rc = curl_easy_perform(this->get_handle()); + if (rc == CURLE_OK) { + return Ok(response); + } + + return Err(rc); + } + + long get_response_code() const + { + long retval; + + curl_easy_getinfo(this->get_handle(), CURLINFO_RESPONSE_CODE, &retval); + return retval; + } protected: static int debug_cb( CURL* handle, curl_infotype type, char* data, size_t size, void* userp); + static size_t string_cb(void* data, size_t size, size_t nmemb, void* userp); + const std::string cr_name; - bool cr_open; + bool cr_open{true}; auto_mem cr_handle; char cr_error_buffer[CURL_ERROR_SIZE]; - int cr_completions; + int cr_completions{0}; }; class curl_looper : public isc::service { @@ -154,20 +185,9 @@ public: curl_looper() : cl_curl_multi(curl_multi_cleanup) { this->cl_curl_multi.reset(curl_multi_init()); - }; + } - void process_all() - { - this->check_for_new_requests(); - - this->requeue_requests(LONG_MAX); - - while (!this->cl_handle_to_request.empty()) { - this->perform_io(); - - this->check_for_finished_requests(); - } - }; + void process_all(); void add_request(const std::shared_ptr& cr) { @@ -175,12 +195,12 @@ public: this->cl_all_requests.emplace_back(cr); this->cl_new_requests.emplace_back(cr); - }; + } void close_request(const std::string& name) { this->cl_close_requests.emplace_back(name); - }; + } protected: void loop_body() override; diff --git a/src/db_sub_source.cc b/src/db_sub_source.cc index d993c084..be8d67b6 100644 --- a/src/db_sub_source.cc +++ b/src/db_sub_source.cc @@ -32,6 +32,7 @@ #include "db_sub_source.hh" #include "base/date_time_scanner.hh" +#include "base/itertools.hh" #include "base/time_util.hh" #include "config.h" #include "yajlpp/json_ptr.hh" @@ -261,17 +262,10 @@ db_label_source::clear() this->dls_cell_width.clear(); } -long +nonstd::optional db_label_source::column_name_to_index(const std::string& name) const { - std::vector::const_iterator iter; - - iter = std::find(this->dls_headers.begin(), this->dls_headers.end(), name); - if (iter == this->dls_headers.end()) { - return -1; - } - - return std::distance(this->dls_headers.begin(), iter); + return this->dls_headers | lnav::itertools::find(name); } nonstd::optional diff --git a/src/db_sub_source.hh b/src/db_sub_source.hh index b477c7a0..6b90cbc1 100644 --- a/src/db_sub_source.hh +++ b/src/db_sub_source.hh @@ -48,20 +48,14 @@ public: this->clear(); } - bool has_log_time_column() const - { - return !this->dls_time_column.empty(); - }; + bool has_log_time_column() const { return !this->dls_time_column.empty(); } - size_t text_line_count() - { - return this->dls_rows.size(); - }; + size_t text_line_count() { return this->dls_rows.size(); } size_t text_size_for_line(textview_curses& tc, int line, line_flags_t flags) { return this->text_line_width(tc); - }; + } size_t text_line_width(textview_curses& curses) { @@ -71,7 +65,7 @@ public: retval += dls_header.hm_column_size + 1; } return retval; - }; + } void text_value_for_line(textview_curses& tc, int row, @@ -86,7 +80,8 @@ public: void clear(); - long column_name_to_index(const std::string& name) const; + nonstd::optional column_name_to_index( + const std::string& name) const; nonstd::optional row_for_time(struct timeval time_bucket); @@ -97,26 +92,22 @@ public: } return this->dls_time_column[row]; - }; + } struct header_meta { - explicit header_meta(std::string name) - : hm_name(std::move(name)), hm_column_type(SQLITE3_TEXT), - hm_graphable(false), hm_log_time(false), hm_column_size(0) - { - } + explicit header_meta(std::string name) : hm_name(std::move(name)) {} bool operator==(const std::string& name) const { return this->hm_name == name; - }; + } std::string hm_name; - int hm_column_type; + int hm_column_type{SQLITE3_TEXT}; unsigned int hm_sub_type{0}; - bool hm_graphable; - bool hm_log_time; - size_t hm_column_size; + bool hm_graphable{false}; + bool hm_log_time{false}; + size_t hm_column_size{0}; }; stacked_bar_chart dls_chart; diff --git a/src/doc_status_source.hh b/src/doc_status_source.hh index 79cb4c32..0624fa40 100644 --- a/src/doc_status_source.hh +++ b/src/doc_status_source.hh @@ -55,17 +55,14 @@ public: role_t::VCR_STATUS_STITCH_NORMAL_TO_TITLE); this->tss_fields[TSF_DESCRIPTION].set_share(1); this->tss_fields[TSF_DESCRIPTION].set_role(role_t::VCR_STATUS); - }; + } - size_t statusview_fields() override - { - return TSF__MAX; - }; + size_t statusview_fields() override { return TSF__MAX; } status_field& statusview_value_for_field(int field) override { return this->tss_fields[field]; - }; + } void set_title(const std::string& title) { diff --git a/src/dump_internals.cc b/src/dump_internals.cc new file mode 100644 index 00000000..9a282477 --- /dev/null +++ b/src/dump_internals.cc @@ -0,0 +1,83 @@ +/** + * Copyright (c) 2022, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "dump_internals.hh" + +#include "lnav.hh" +#include "lnav_config.hh" +#include "log_format_loader.hh" +#include "sql_help.hh" +#include "view_helpers.examples.hh" +#include "yajlpp/yajlpp.hh" + +namespace lnav { + +void +dump_internals(const char* internals_dir) +{ + dump_schema_to( + lnav_config_handlers, internals_dir, "config-v1.schema.json"); + dump_schema_to(root_format_handler, internals_dir, "format-v1.schema.json"); + + execute_examples(); + + auto cmd_ref_path = ghc::filesystem::path(internals_dir) / "cmd-ref.rst"; + auto cmd_file = std::unique_ptr( + fopen(cmd_ref_path.c_str(), "w+"), fclose); + + if (cmd_file != nullptr) { + std::set unique_cmds; + + for (auto& cmd : lnav_commands) { + if (unique_cmds.find(cmd.second) != unique_cmds.end()) { + continue; + } + unique_cmds.insert(cmd.second); + format_help_text_for_rst( + cmd.second->c_help, eval_example, cmd_file.get()); + } + } + + auto sql_ref_path = ghc::filesystem::path(internals_dir) / "sql-ref.rst"; + auto sql_file = std::unique_ptr( + fopen(sql_ref_path.c_str(), "w+"), fclose); + std::set unique_sql_help; + + if (sql_file != nullptr) { + for (auto& sql : sqlite_function_help) { + if (unique_sql_help.find(sql.second) != unique_sql_help.end()) { + continue; + } + unique_sql_help.insert(sql.second); + format_help_text_for_rst(*sql.second, eval_example, sql_file.get()); + } + } +} + +} // namespace lnav diff --git a/src/dump_internals.hh b/src/dump_internals.hh new file mode 100644 index 00000000..52a4815e --- /dev/null +++ b/src/dump_internals.hh @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2022, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef dump_internals_hh +#define dump_internals_hh + +#include + +namespace lnav { + +void dump_internals(const char* dir); + +} // namespace lnav + +#endif diff --git a/src/elem_to_json.cc b/src/elem_to_json.cc index 10ed6fa6..70b9eee6 100644 --- a/src/elem_to_json.cc +++ b/src/elem_to_json.cc @@ -31,6 +31,7 @@ #include "elem_to_json.hh" +#include "base/itertools.hh" #include "config.h" #include "yajlpp/yajlpp.hh" @@ -150,7 +151,7 @@ map_elements_to_json(yajl_gen gen, if (key_str.empty()) { continue; } - if (find(names.begin(), names.end(), key_str) != names.end()) { + if (names | lnav::itertools::find(key_str)) { unique_names = false; break; } else { diff --git a/src/field_overlay_source.cc b/src/field_overlay_source.cc index c2bc3cd9..88b1b965 100644 --- a/src/field_overlay_source.cc +++ b/src/field_overlay_source.cc @@ -579,22 +579,19 @@ field_overlay_source::build_meta_line(const listview_curses& lv, size_t filename_width = this->fos_lss.get_filename_offset(); if (!line_meta.bm_comment.empty()) { + const auto* lead = line_meta.bm_tags.empty() ? " \u2514 " + : " \u251c "; attr_line_t al; - al.with_string(" + ") - .with_attr(string_attr( - line_range(1, 2), - VC_GRAPHIC.value( - line_meta.bm_tags.empty() ? ACS_LLCORNER : ACS_LTEE))) - .append(line_meta.bm_comment); + al.with_string(lead).append( + lnav::roles::comment(line_meta.bm_comment)); al.insert(0, filename_width, ' '); dst.emplace_back(al); } if (!line_meta.bm_tags.empty()) { attr_line_t al; - al.with_string(" +").with_attr(string_attr( - line_range(1, 2), VC_GRAPHIC.value(ACS_LLCORNER))); + al.with_string(" \u2514"); for (const auto& str : line_meta.bm_tags) { al.append(1, ' ').append( str, VC_STYLE.value(vc.attrs_for_ident(str))); @@ -614,3 +611,53 @@ field_overlay_source::build_meta_line(const listview_curses& lv, } } } + +void +field_overlay_source::add_key_line_attrs(int key_size, bool last_line) +{ + string_attrs_t& sa = this->fos_lines.back().get_attrs(); + struct line_range lr(1, 2); + int64_t graphic = (int64_t) (last_line ? ACS_LLCORNER : ACS_LTEE); + sa.emplace_back(lr, VC_GRAPHIC.value(graphic)); + + lr.lr_start = 3 + key_size + 3; + lr.lr_end = -1; + sa.emplace_back(lr, VC_STYLE.value(A_BOLD)); +} + +bool +field_overlay_source::list_value_for_overlay(const listview_curses& lv, + int y, + int bottom, + vis_line_t row, + attr_line_t& value_out) +{ + if (y == 0) { + this->build_field_lines(lv); + this->build_summary_lines(lv); + return false; + } + + if (1 <= y && y <= (int) this->fos_lines.size()) { + value_out = this->fos_lines[y - 1]; + return true; + } + + if (!this->fos_summary_lines.empty() && y == (bottom - 1)) { + value_out = this->fos_summary_lines[0]; + return true; + } + + if (!this->fos_meta_lines.empty()) { + value_out = this->fos_meta_lines.front(); + this->fos_meta_lines.erase(this->fos_meta_lines.begin()); + + return true; + } + + if (row < lv.get_inner_height()) { + this->build_meta_line(lv, this->fos_meta_lines, row); + } + + return false; +} diff --git a/src/field_overlay_source.hh b/src/field_overlay_source.hh index 8e7b23e9..13a7fd53 100644 --- a/src/field_overlay_source.hh +++ b/src/field_overlay_source.hh @@ -42,57 +42,17 @@ class field_overlay_source : public list_overlay_source { public: explicit field_overlay_source(logfile_sub_source& lss, textfile_sub_source& tss) - : fos_lss(lss), fos_tss(tss), fos_log_helper(lss){ - - }; - - void add_key_line_attrs(int key_size, bool last_line = false) + : fos_lss(lss), fos_tss(tss), fos_log_helper(lss) { - string_attrs_t& sa = this->fos_lines.back().get_attrs(); - struct line_range lr(1, 2); - int64_t graphic = (int64_t) (last_line ? ACS_LLCORNER : ACS_LTEE); - sa.emplace_back(lr, VC_GRAPHIC.value(graphic)); + } - lr.lr_start = 3 + key_size + 3; - lr.lr_end = -1; - sa.emplace_back(lr, VC_STYLE.value(A_BOLD)); - }; + void add_key_line_attrs(int key_size, bool last_line = false); bool list_value_for_overlay(const listview_curses& lv, int y, int bottom, vis_line_t row, - attr_line_t& value_out) override - { - if (y == 0) { - this->build_field_lines(lv); - this->build_summary_lines(lv); - return false; - } - - if (1 <= y && y <= (int) this->fos_lines.size()) { - value_out = this->fos_lines[y - 1]; - return true; - } - - if (!this->fos_summary_lines.empty() && y == (bottom - 1)) { - value_out = this->fos_summary_lines[0]; - return true; - } - - if (!this->fos_meta_lines.empty()) { - value_out = this->fos_meta_lines.front(); - this->fos_meta_lines.erase(this->fos_meta_lines.begin()); - - return true; - } - - if (row < lv.get_inner_height()) { - this->build_meta_line(lv, this->fos_meta_lines, row); - } - - return false; - }; + attr_line_t& value_out) override; void build_field_lines(const listview_curses& lv); void build_summary_lines(const listview_curses& lv); diff --git a/src/file_collection.cc b/src/file_collection.cc index f8eefc92..b0228b5c 100644 --- a/src/file_collection.cc +++ b/src/file_collection.cc @@ -37,6 +37,7 @@ #include "base/humanize.network.hh" #include "base/isc.hh" +#include "base/itertools.hh" #include "base/opt_util.hh" #include "base/string_util.hh" #include "config.h" @@ -280,21 +281,19 @@ file_collection::watch_logfile(const std::string& filename, return lnav::futures::make_ready_future(std::move(retval)); } - auto stat_iter = find_if(this->fc_new_stats.begin(), - this->fc_new_stats.end(), - [&st](auto& elem) { - return st.st_ino == elem.st_ino - && st.st_dev == elem.st_dev; - }); - if (stat_iter != this->fc_new_stats.end()) { + if (this->fc_new_stats | lnav::itertools::find_if([&st](const auto& elem) { + return st.st_ino == elem.st_ino && st.st_dev == elem.st_dev; + })) + { // this file is probably a link that we have already scanned in this // pass. return lnav::futures::make_ready_future(std::move(retval)); } this->fc_new_stats.emplace_back(st); - auto file_iter - = find_if(this->fc_files.begin(), this->fc_files.end(), same_file(st)); + + auto file_iter = std::find_if( + this->fc_files.begin(), this->fc_files.end(), same_file(st)); if (file_iter == this->fc_files.end()) { if (this->fc_other_files.find(filename) != this->fc_other_files.end()) { @@ -313,7 +312,8 @@ file_collection::watch_logfile(const std::string& filename, return retval; } - auto ff = detect_file_format(filename); + auto ff = loo2.loo_temp_file ? file_format_t::UNKNOWN + : detect_file_format(filename); loo2.loo_file_format = ff; switch (ff) { @@ -619,7 +619,7 @@ file_collection::rescan_files(bool required) this->expand_filename(fq, path, pair.second, false); } - } else { + } else if (pair.second.loo_fd.get() != -1) { fq.push_back(watch_logfile(pair.first, pair.second, required)); } diff --git a/src/files_sub_source.cc b/src/files_sub_source.cc index a0941e7c..53ffd571 100644 --- a/src/files_sub_source.cc +++ b/src/files_sub_source.cc @@ -113,7 +113,7 @@ files_sub_source::list_input_handle_key(listview_curses& lv, int ch) } lv.reload_data(); - lnav_data.ld_mode = LNM_PAGING; + lnav_data.ld_mode = ln_mode_t::PAGING; }); return true; @@ -205,7 +205,7 @@ files_sub_source::list_input_handle_key(listview_curses& lv, int ch) void files_sub_source::list_input_handle_scroll_out(listview_curses& lv) { - lnav_data.ld_mode = LNM_PAGING; + lnav_data.ld_mode = ln_mode_t::PAGING; lnav_data.ld_filter_view.reload_data(); } @@ -300,7 +300,7 @@ files_sub_source::text_attrs_for_line(textview_curses& tc, string_attrs_t& value_out) { bool selected - = lnav_data.ld_mode == LNM_FILES && line == tc.get_selection(); + = lnav_data.ld_mode == ln_mode_t::FILES && line == tc.get_selection(); const auto& fc = lnav_data.ld_active_files; auto& vcolors = view_colors::singleton(); const auto dim = tc.get_dimensions(); diff --git a/src/filter_status_source.cc b/src/filter_status_source.cc index 00c77151..08108936 100644 --- a/src/filter_status_source.cc +++ b/src/filter_status_source.cc @@ -91,12 +91,12 @@ size_t filter_status_source::statusview_fields() { switch (lnav_data.ld_mode) { - case LNM_SEARCH_FILTERS: - case LNM_SEARCH_FILES: + case ln_mode_t::SEARCH_FILTERS: + case ln_mode_t::SEARCH_FILES: this->tss_fields[TSF_HELP].set_value(""); break; - case LNM_FILTER: - case LNM_FILES: + case ln_mode_t::FILTER: + case ln_mode_t::FILES: this->tss_fields[TSF_HELP].set_value(EXIT_MSG); break; default: @@ -104,12 +104,12 @@ filter_status_source::statusview_fields() break; } - if (lnav_data.ld_mode == LNM_FILES || lnav_data.ld_mode == LNM_SEARCH_FILES) + if (lnav_data.ld_mode == ln_mode_t::FILES + || lnav_data.ld_mode == ln_mode_t::SEARCH_FILES) { this->tss_fields[TSF_FILES_TITLE].set_value( " " ANSI_ROLE("F") "iles ", role_t::VCR_STATUS_TITLE_HOTKEY); - this->tss_fields[TSF_FILES_TITLE].set_role( - role_t::VCR_STATUS_TITLE); + this->tss_fields[TSF_FILES_TITLE].set_role(role_t::VCR_STATUS_TITLE); this->tss_fields[TSF_FILES_RIGHT_STITCH].set_stitch_value( role_t::VCR_STATUS_STITCH_TITLE_TO_NORMAL, role_t::VCR_STATUS_STITCH_NORMAL_TO_TITLE); @@ -245,7 +245,7 @@ filter_help_status_source::statusview_fields() return; } - if (lnav_data.ld_mode == LNM_FILTER) { + if (lnav_data.ld_mode == ln_mode_t::FILTER) { auto& editor = lnav_data.ld_filter_source; auto& lv = lnav_data.ld_filter_view; auto& fs = tss->get_filters(); @@ -285,7 +285,7 @@ filter_help_status_source::statusview_fields() tss->tss_apply_filters ? "Disable Filtering" : "Enable Filtering"); } - } else if (lnav_data.ld_mode == LNM_FILES + } else if (lnav_data.ld_mode == ln_mode_t::FILES && lnav_data.ld_session_loaded) { auto& lv = lnav_data.ld_files_view; auto sel = files_model::from_selection(lv.get_selection()); diff --git a/src/filter_sub_source.cc b/src/filter_sub_source.cc index 431a3f4d..0e8a8581 100644 --- a/src/filter_sub_source.cc +++ b/src/filter_sub_source.cc @@ -41,13 +41,10 @@ filter_sub_source::filter_sub_source() { this->fss_regex_context.set_highlighter(readline_regex_highlighter) .set_append_character(0); - this->fss_editor.add_context( - lnav::enums::to_underlying(filter_lang_t::REGEX), - this->fss_regex_context); + this->fss_editor.add_context(filter_lang_t::REGEX, this->fss_regex_context); this->fss_sql_context.set_highlighter(readline_sqlite_highlighter) .set_append_character(0); - this->fss_editor.add_context(lnav::enums::to_underlying(filter_lang_t::SQL), - this->fss_sql_context); + this->fss_editor.add_context(filter_lang_t::SQL, this->fss_sql_context); this->fss_editor.set_change_action( bind_mem(&filter_sub_source::rl_change, this)); this->fss_editor.set_perform_action( @@ -80,8 +77,8 @@ filter_sub_source::list_input_handle_key(listview_curses& lv, int ch) switch (ch) { case 'f': { - auto top_view = *lnav_data.ld_view_stack.top(); - auto tss = top_view->get_sub_source(); + auto* top_view = *lnav_data.ld_view_stack.top(); + auto* tss = top_view->get_sub_source(); tss->toggle_apply_filters(); break; @@ -159,19 +156,18 @@ filter_sub_source::list_input_handle_key(listview_curses& lv, int ch) this->fss_editing = true; - add_view_text_possibilities( - &this->fss_editor, - lnav::enums::to_underlying(filter_lang_t::REGEX), - "*", - top_view); + add_view_text_possibilities(&this->fss_editor, + filter_lang_t::REGEX, + "*", + top_view, + text_quoting::regex); this->fss_editor.set_window(lv.get_window()); this->fss_editor.set_visible(true); this->fss_editor.set_y(lv.get_y() + (int) (lv.get_selection() - lv.get_top())); this->fss_editor.set_left(25); this->fss_editor.set_width(this->tss_view->get_width() - 8 - 1); - this->fss_editor.focus( - lnav::enums::to_underlying(filter_lang_t::REGEX), "", ""); + this->fss_editor.focus(filter_lang_t::REGEX, "", ""); this->fss_filter_state = true; ef->disable(); return true; @@ -196,19 +192,18 @@ filter_sub_source::list_input_handle_key(listview_curses& lv, int ch) this->fss_editing = true; - add_view_text_possibilities( - &this->fss_editor, - lnav::enums::to_underlying(filter_lang_t::REGEX), - "*", - top_view); + add_view_text_possibilities(&this->fss_editor, + filter_lang_t::REGEX, + "*", + top_view, + text_quoting::regex); this->fss_editor.set_window(lv.get_window()); this->fss_editor.set_visible(true); this->fss_editor.set_y(lv.get_y() + (int) (lv.get_selection() - lv.get_top())); this->fss_editor.set_left(25); this->fss_editor.set_width(this->tss_view->get_width() - 8 - 1); - this->fss_editor.focus( - lnav::enums::to_underlying(filter_lang_t::REGEX), "", ""); + this->fss_editor.focus(filter_lang_t::REGEX, "", ""); this->fss_filter_state = true; ef->disable(); return true; @@ -227,23 +222,20 @@ filter_sub_source::list_input_handle_key(listview_curses& lv, int ch) this->fss_editing = true; + auto tq = tf->get_lang() == filter_lang_t::SQL + ? text_quoting::sql + : text_quoting::regex; add_view_text_possibilities( - &this->fss_editor, - lnav::enums::to_underlying(filter_lang_t::REGEX), - "*", - top_view); + &this->fss_editor, tf->get_lang(), "*", top_view, tq); add_filter_expr_possibilities( - &this->fss_editor, - lnav::enums::to_underlying(filter_lang_t::SQL), - "*"); + &this->fss_editor, filter_lang_t::SQL, "*"); this->fss_editor.set_window(lv.get_window()); this->fss_editor.set_visible(true); this->fss_editor.set_y(lv.get_y() + (int) (lv.get_selection() - lv.get_top())); this->fss_editor.set_left(25); this->fss_editor.set_width(this->tss_view->get_width() - 8 - 1); - this->fss_editor.focus(lnav::enums::to_underlying(tf->get_lang()), - ""); + this->fss_editor.focus(tf->get_lang(), ""); this->fss_editor.rewrite_line(0, tf->get_id().c_str()); this->fss_filter_state = tf->is_enabled(); tf->disable(); @@ -349,7 +341,7 @@ filter_sub_source::text_attrs_for_line(textview_curses& tc, filter_stack& fs = tss->get_filters(); auto tf = *(fs.begin() + line); bool selected - = lnav_data.ld_mode == LNM_FILTER && line == tc.get_selection(); + = lnav_data.ld_mode == ln_mode_t::FILTER && line == tc.get_selection(); if (selected) { value_out.emplace_back(line_range{0, 1}, @@ -509,7 +501,7 @@ filter_sub_source::rl_perform(readline_curses* rc) filter_stack& fs = tss->get_filters(); auto iter = fs.begin() + this->tss_view->get_selection(); auto tf = *iter; - auto new_value = rc->get_value(); + auto new_value = rc->get_value().get_string(); if (new_value.empty()) { this->rl_abort(rc); @@ -663,6 +655,6 @@ filter_sub_source::rl_display_next(readline_curses* rc) void filter_sub_source::list_input_handle_scroll_out(listview_curses& lv) { - lnav_data.ld_mode = LNM_PAGING; + lnav_data.ld_mode = ln_mode_t::PAGING; lnav_data.ld_filter_view.reload_data(); } diff --git a/src/formats/block_log.json b/src/formats/block_log.json index 176a54c4..aaba6d82 100644 --- a/src/formats/block_log.json +++ b/src/formats/block_log.json @@ -5,15 +5,15 @@ "description": "A generic format for logs, like cron, that have a date at the start of a block.", "regex": { "std": { - "pattern": "^(?\\S{3,8} \\w{3}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\w+ \\d{4})(?.*)$" + "pattern": "^(?\\S{3,8} \\w{3}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\w+ \\d{4})\\s*(?.*)$" }, "sq-brackets": { - "pattern": "^\\[(?\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?)Z?\\](?.*)$" + "pattern": "^\\[(?\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?)Z?\\]\\s*(?.*)$" } }, "sample": [ { - "line": "Sat Apr 27 03:33:07 PDT 2013" + "line": "Sat Apr 27 03:33:07 PDT 2013\nHello, World" }, { "line": "[2021-05-21T21:58:57.022497Z]" diff --git a/src/fts_fuzzy_match.cc b/src/fts_fuzzy_match.cc index d976bccc..8ae7098b 100644 --- a/src/fts_fuzzy_match.cc +++ b/src/fts_fuzzy_match.cc @@ -4,8 +4,12 @@ // license: you are granted a perpetual, irrevocable license to copy, modify, // publish, and distribute this file as you see fit. +#include // memcpy + #include "fts_fuzzy_match.hh" +#include // ::tolower, ::toupper + #include "config.h" namespace fts { diff --git a/src/fts_fuzzy_match.hh b/src/fts_fuzzy_match.hh index dbee59b2..1d3b0132 100644 --- a/src/fts_fuzzy_match.hh +++ b/src/fts_fuzzy_match.hh @@ -35,10 +35,6 @@ #define FTS_FUZZY_MATCH_H #include // uint8_t -#include -#include // memcpy - -#include // ::tolower, ::toupper // Public interface namespace fts { diff --git a/src/grep_proc.hh b/src/grep_proc.hh index 9a77a908..2aad29be 100644 --- a/src/grep_proc.hh +++ b/src/grep_proc.hh @@ -79,12 +79,9 @@ public: return highest; } return start; - }; + } - virtual void grep_next_line(LineType& line) - { - line = line + LineType(1); - }; + virtual void grep_next_line(LineType& line) { line = line + LineType(1); } grep_proc* gps_proc; }; @@ -97,7 +94,7 @@ public: virtual ~grep_proc_control() = default; /** @param msg The error encountered while attempting the grep. */ - virtual void grep_error(const std::string& msg){}; + virtual void grep_error(const std::string& msg) {} }; /** @@ -114,10 +111,10 @@ public: LineType stop){}; /** Called periodically between grep_begin and grep_end. */ - virtual void grep_end_batch(grep_proc& gp){}; + virtual void grep_end_batch(grep_proc& gp) {} /** Called at the end of a grep run. */ - virtual void grep_end(grep_proc& gp){}; + virtual void grep_end(grep_proc& gp) {} /** * Called when a match is found on 'line' and between [start, end). @@ -183,24 +180,15 @@ public: virtual ~grep_proc(); /** @param gpd The sink to send resuls to. */ - void set_sink(grep_proc_sink* gpd) - { - this->gp_sink = gpd; - }; + void set_sink(grep_proc_sink* gpd) { this->gp_sink = gpd; } grep_proc& invalidate(); /** @param gpd The sink to send results to. */ - void set_control(grep_proc_control* gpc) - { - this->gp_control = gpc; - }; + void set_control(grep_proc_control* gpc) { this->gp_control = gpc; } /** @return The sink to send results to. */ - grep_proc_sink* get_sink() - { - return this->gp_sink; - }; + grep_proc_sink* get_sink() { return this->gp_sink; } /** * Queue a request to search the input between the given line numbers. @@ -221,7 +209,7 @@ public: } return *this; - }; + } /** * Start the search requests that have been queued up with queue_request. @@ -237,7 +225,7 @@ public: if (this->gp_err_pipe != -1) { pollfds.push_back((struct pollfd){this->gp_err_pipe, POLLIN, 0}); } - }; + } /** * Check the fd_set to see if there is any new data to be processed. @@ -259,7 +247,7 @@ public: } return true; - }; + } protected: /** @@ -277,15 +265,9 @@ protected: virtual void child_init(){}; - virtual void child_batch() - { - fflush(stdout); - }; + virtual void child_batch() { fflush(stdout); } - virtual void child_term() - { - fflush(stdout); - }; + virtual void child_term() { fflush(stdout); } virtual void handle_match( int line, std::string& line_value, int off, int* matches, int count); @@ -319,4 +301,5 @@ protected: grep_proc_sink* gp_sink{nullptr}; /*< The sink delegate. */ grep_proc_control* gp_control{nullptr}; /*< The control delegate. */ }; + #endif diff --git a/src/help_text.hh b/src/help_text.hh index 2c480f7e..81dc8fb9 100644 --- a/src/help_text.hh +++ b/src/help_text.hh @@ -100,56 +100,56 @@ struct help_text { this->ht_context = help_context_t::HC_COMMAND; this->ht_name = &name[1]; } - }; + } help_text& command() noexcept { this->ht_context = help_context_t::HC_COMMAND; return *this; - }; + } help_text& sql_function() noexcept { this->ht_context = help_context_t::HC_SQL_FUNCTION; return *this; - }; + } help_text& sql_agg_function() noexcept { this->ht_context = help_context_t::HC_SQL_FUNCTION; this->ht_function_type = help_function_type_t::HFT_AGGREGATE; return *this; - }; + } help_text& sql_table_valued_function() noexcept { this->ht_context = help_context_t::HC_SQL_TABLE_VALUED_FUNCTION; return *this; - }; + } help_text& sql_command() noexcept { this->ht_context = help_context_t::HC_SQL_COMMAND; return *this; - }; + } help_text& sql_keyword() noexcept { this->ht_context = help_context_t::HC_SQL_KEYWORD; return *this; - }; + } help_text& sql_infix() noexcept { this->ht_context = help_context_t::HC_SQL_INFIX; return *this; - }; + } help_text& with_summary(const char* summary) noexcept { this->ht_summary = summary; return *this; - }; + } help_text& with_flag_name(const char* flag) noexcept { @@ -181,19 +181,19 @@ struct help_text { { this->ht_nargs = help_nargs_t::HN_OPTIONAL; return *this; - }; + } help_text& zero_or_more() noexcept { this->ht_nargs = help_nargs_t::HN_ZERO_OR_MORE; return *this; - }; + } help_text& one_or_more() noexcept { this->ht_nargs = help_nargs_t::HN_ONE_OR_MORE; return *this; - }; + } help_text& with_format(help_parameter_format_t format) noexcept { diff --git a/src/highlighter.hh b/src/highlighter.hh index 9be4ac4e..6ba0328f 100644 --- a/src/highlighter.hh +++ b/src/highlighter.hh @@ -46,7 +46,7 @@ struct highlighter { { pcre_refcount(this->h_code, 1); this->study(); - }; + } highlighter(const highlighter& other); @@ -59,7 +59,7 @@ struct highlighter { this->h_code = nullptr; } free(this->h_code_extra); - }; + } void study(); @@ -75,14 +75,14 @@ struct highlighter { this->h_role = role; return *this; - }; + } highlighter& with_attrs(int attrs) { this->h_attrs = attrs; return *this; - }; + } highlighter& with_text_format(text_format_t tf) { @@ -96,7 +96,7 @@ struct highlighter { this->h_format_name = name; return *this; - }; + } highlighter& with_color(const styling::color_unit& fg, const styling::color_unit& bg) @@ -105,7 +105,7 @@ struct highlighter { this->h_bg = bg; return *this; - }; + } highlighter& with_nestable(bool val) { @@ -118,7 +118,7 @@ struct highlighter { ensure(this->h_attrs != -1); return this->h_attrs; - }; + } void annotate(attr_line_t& al, int start) const; diff --git a/src/hist_source.hh b/src/hist_source.hh index 28dedef8..adc48112 100644 --- a/src/hist_source.hh +++ b/src/hist_source.hh @@ -73,31 +73,25 @@ struct stacked_bar_chart_base { template class stacked_bar_chart : public stacked_bar_chart_base { public: - stacked_bar_chart() - : sbc_do_stacking(true), sbc_left(0), sbc_right(0), - sbc_show_state(show_all()){ - - }; - stacked_bar_chart& with_stacking_enabled(bool enabled) { this->sbc_do_stacking = enabled; return *this; - }; + } stacked_bar_chart& with_attrs_for_ident(const T& ident, int attrs) { struct chart_ident& ci = this->find_ident(ident); ci.ci_attrs = attrs; return *this; - }; + } stacked_bar_chart& with_margins(unsigned long left, unsigned long right) { this->sbc_left = left; this->sbc_right = right; return *this; - }; + } show_state show_next_ident(direction dir) { @@ -152,7 +146,7 @@ public: }); return this->sbc_show_state; - }; + } void get_ident_to_show(T& ident_out) const { @@ -162,7 +156,7 @@ public: [&](const show_one& one) { ident_out = this->sbc_idents[one.so_index].ci_ident; }); - }; + } void chart_attrs_for_value(const listview_curses& lc, int& left, @@ -230,20 +224,20 @@ public: value_out.emplace_back( lr, VC_STYLE.value(ci.ci_attrs | A_REVERSE)); } - }; + } void clear() { this->sbc_idents.clear(); this->sbc_ident_lookup.clear(); this->sbc_show_state = show_all(); - }; + } void add_value(const T& ident, double amount = 1.0) { struct chart_ident& ci = this->find_ident(ident); ci.ci_stats.update(amount); - }; + } struct bucket_stats_t { bucket_stats_t() @@ -281,11 +275,11 @@ public: const chart_ident& ci = this->find_ident(ident); return ci.ci_stats; - }; + } protected: struct chart_ident { - chart_ident(const T& ident) : ci_ident(ident){}; + chart_ident(const T& ident) : ci_ident(ident) {} T ci_ident; int ci_attrs{0}; @@ -303,13 +297,13 @@ protected: return this->sbc_idents.back(); } return this->sbc_idents[iter->second]; - }; + } - bool sbc_do_stacking; - unsigned long sbc_left, sbc_right; + bool sbc_do_stacking{true}; + unsigned long sbc_left{0}, sbc_right{0}; std::vector sbc_idents; std::map sbc_ident_lookup; - show_state sbc_show_state; + show_state sbc_show_state{show_all()}; }; class hist_source2 diff --git a/src/hotkeys.cc b/src/hotkeys.cc index 492cc5f3..3a93700d 100644 --- a/src/hotkeys.cc +++ b/src/hotkeys.cc @@ -31,6 +31,7 @@ #include "base/ansi_scrubber.hh" #include "base/injector.hh" +#include "base/itertools.hh" #include "base/math_util.hh" #include "base/opt_util.hh" #include "bookmarks.hh" @@ -160,8 +161,17 @@ handle_keyseq(const char* keyseq) log_debug("executing key sequence %s: %s", keyseq, kc.kc_cmd.c_str()); auto result = execute_any(ec, kc.kc_cmd); - lnav_data.ld_rl_view->set_value( - result.map(ok_prefix).orElse(err_to_ok).unwrap()); + if (result.isOk()) { + lnav_data.ld_rl_view->set_value(result.unwrap()); + } else { + auto um = result.unwrapErr(); + + um.um_snippets.clear(); + um.um_reason.clear(); + um.um_notes.clear(); + um.um_help.clear(); + lnav_data.ld_rl_view->set_attr_value(um.to_attr_line()); + } if (!kc.kc_alt_msg.empty()) { shlex lexer(kc.kc_alt_msg); @@ -731,13 +741,13 @@ handle_paging_key(int ch) db_label_source& dls = lnav_data.ld_db_row_source; if (toggle_view(db_tc)) { - long log_line_index = dls.column_name_to_index("log_line"); + auto log_line_index = dls.column_name_to_index("log_line"); - if (log_line_index == -1) { + if (!log_line_index) { log_line_index = dls.column_name_to_index("min(log_line)"); } - if (log_line_index != -1) { + if (log_line_index) { fmt::memory_buffer linestr; int line_number = (int) tc->get_top(); unsigned int row; @@ -747,7 +757,7 @@ handle_paging_key(int ch) line_number); linestr.push_back('\0'); for (row = 0; row < dls.dls_rows.size(); row++) { - if (strcmp(dls.dls_rows[row][log_line_index], + if (strcmp(dls.dls_rows[row][log_line_index.value()], linestr.data()) == 0) { vis_line_t db_line(row); @@ -761,16 +771,16 @@ handle_paging_key(int ch) } else if (db_tc->get_inner_height() > 0) { int db_row = db_tc->get_top(); tc = &lnav_data.ld_views[LNV_LOG]; - long log_line_index = dls.column_name_to_index("log_line"); + auto log_line_index = dls.column_name_to_index("log_line"); - if (log_line_index == -1) { + if (!log_line_index) { log_line_index = dls.column_name_to_index("min(log_line)"); } - if (log_line_index != -1) { + if (log_line_index) { unsigned int line_number; - if (sscanf(dls.dls_rows[db_row][log_line_index], + if (sscanf(dls.dls_rows[db_row][log_line_index.value()], "%d", &line_number) && line_number < tc->listview_rows(*tc)) @@ -839,11 +849,10 @@ handle_paging_key(int ch) if (line_attr_opt) { const auto& fc = lnav_data.ld_active_files; auto lf = line_attr_opt.value().get(); - auto iter - = find(fc.fc_files.begin(), fc.fc_files.end(), lf); - if (iter != fc.fc_files.end()) { - auto index = distance(fc.fc_files.begin(), iter); - auto index_vl = vis_line_t(index); + auto index_opt + = fc.fc_files | lnav::itertools::find(lf); + if (index_opt) { + auto index_vl = vis_line_t(index_opt.value()); lnav_data.ld_files_view.set_top(index_vl); lnav_data.ld_files_view.set_selection(index_vl); diff --git a/src/json-extension-functions.cc b/src/json-extension-functions.cc index b1c93201..963ca6ad 100644 --- a/src/json-extension-functions.cc +++ b/src/json-extension-functions.cc @@ -185,7 +185,7 @@ gen_handle_null(void* ctx) sql_json_op* sjo = (sql_json_op*) ctx; yajl_gen gen = (yajl_gen) sjo->jo_ptr_data; - if (sjo->jo_ptr.jp_state == json_ptr::MS_DONE) { + if (sjo->jo_ptr.jp_state == json_ptr::match_state_t::DONE) { sjo->sjo_type = SQLITE_NULL; } else { sjo->jo_ptr_error_code = yajl_gen_null(gen); @@ -200,7 +200,7 @@ gen_handle_boolean(void* ctx, int boolVal) sql_json_op* sjo = (sql_json_op*) ctx; yajl_gen gen = (yajl_gen) sjo->jo_ptr_data; - if (sjo->jo_ptr.jp_state == json_ptr::MS_DONE) { + if (sjo->jo_ptr.jp_state == json_ptr::match_state_t::DONE) { sjo->sjo_type = SQLITE_INTEGER; sjo->sjo_int = boolVal; } else { @@ -216,7 +216,7 @@ gen_handle_string(void* ctx, const unsigned char* stringVal, size_t len) sql_json_op* sjo = (sql_json_op*) ctx; yajl_gen gen = (yajl_gen) sjo->jo_ptr_data; - if (sjo->jo_ptr.jp_state == json_ptr::MS_DONE) { + if (sjo->jo_ptr.jp_state == json_ptr::match_state_t::DONE) { sjo->sjo_type = SQLITE3_TEXT; sjo->sjo_str = std::string((char*) stringVal, len); } else { @@ -232,7 +232,7 @@ gen_handle_number(void* ctx, const char* numval, size_t numlen) sql_json_op* sjo = (sql_json_op*) ctx; yajl_gen gen = (yajl_gen) sjo->jo_ptr_data; - if (sjo->jo_ptr.jp_state == json_ptr::MS_DONE) { + if (sjo->jo_ptr.jp_state == json_ptr::match_state_t::DONE) { if (strtonum(sjo->sjo_int, numval, numlen) == numlen) { sjo->sjo_type = SQLITE_INTEGER; } else { @@ -298,7 +298,8 @@ sql_jget(sqlite3_context* context, int argc, sqlite3_value** argv) return; } case yajl_status_client_canceled: - if (jo.jo_ptr.jp_state == json_ptr::MS_ERR_INVALID_ESCAPE) { + if (jo.jo_ptr.jp_state + == json_ptr::match_state_t::ERR_INVALID_ESCAPE) { sqlite3_result_error( context, jo.jo_ptr.error_msg().c_str(), -1); } else { @@ -320,7 +321,8 @@ sql_jget(sqlite3_context* context, int argc, sqlite3_value** argv) return; } case yajl_status_client_canceled: - if (jo.jo_ptr.jp_state == json_ptr::MS_ERR_INVALID_ESCAPE) { + if (jo.jo_ptr.jp_state + == json_ptr::match_state_t::ERR_INVALID_ESCAPE) { sqlite3_result_error( context, jo.jo_ptr.error_msg().c_str(), -1); } else { diff --git a/src/line_buffer.cc b/src/line_buffer.cc index 35aebe22..f9ba61ff 100644 --- a/src/line_buffer.cc +++ b/src/line_buffer.cc @@ -800,3 +800,40 @@ line_buffer::get_available() { return {this->lb_file_offset, this->lb_buffer_size}; } + +line_buffer::gz_indexed::indexDict::indexDict(const z_stream& s, + const file_size_t size) +{ + assert((s.data_type & GZ_END_OF_BLOCK_MASK)); + assert(!(s.data_type & GZ_END_OF_FILE_MASK)); + assert(size >= s.avail_out + GZ_WINSIZE); + this->bits = s.data_type & GZ_BORROW_BITS_MASK; + this->in = s.total_in; + this->out = s.total_out; + auto last_byte_in = s.next_in[-1]; + this->in_bits = last_byte_in >> (8 - this->bits); + // Copy the last 32k uncompressed data (sliding window) to our + // index + memcpy(this->index, s.next_out - GZ_WINSIZE, GZ_WINSIZE); +} + +int +line_buffer::gz_indexed::indexDict::apply(z_streamp s) +{ + s->zalloc = Z_NULL; + s->zfree = Z_NULL; + s->opaque = Z_NULL; + s->avail_in = 0; + s->next_in = Z_NULL; + auto ret = inflateInit2(s, GZ_RAW_MODE); + if (ret != Z_OK) { + return ret; + } + if (this->bits) { + inflatePrime(s, this->bits, this->in_bits); + } + s->total_in = this->in; + s->total_out = this->out; + inflateSetDictionary(s, this->index, GZ_WINSIZE); + return ret; +} diff --git a/src/line_buffer.hh b/src/line_buffer.hh index 4591b11a..38774c3e 100644 --- a/src/line_buffer.hh +++ b/src/line_buffer.hh @@ -99,7 +99,7 @@ public: return this->gz_fd != -1; } - uLong get_source_offset() + uLong get_source_offset() const { return !!*this ? this->strm.total_in + this->strm.avail_in : 0; } @@ -123,40 +123,9 @@ public: unsigned char bits = 0; unsigned char in_bits = 0; Bytef index[GZ_WINSIZE]; - indexDict(z_stream const& s, const file_size_t size) - { - assert((s.data_type & GZ_END_OF_BLOCK_MASK)); - assert(!(s.data_type & GZ_END_OF_FILE_MASK)); - assert(size >= s.avail_out + GZ_WINSIZE); - this->bits = s.data_type & GZ_BORROW_BITS_MASK; - this->in = s.total_in; - this->out = s.total_out; - auto last_byte_in = s.next_in[-1]; - this->in_bits = last_byte_in >> (8 - this->bits); - // Copy the last 32k uncompressed data (sliding window) to our - // index - memcpy(this->index, s.next_out - GZ_WINSIZE, GZ_WINSIZE); - } - - int apply(z_streamp s) - { - s->zalloc = Z_NULL; - s->zfree = Z_NULL; - s->opaque = Z_NULL; - s->avail_in = 0; - s->next_in = Z_NULL; - auto ret = inflateInit2(s, GZ_RAW_MODE); - if (ret != Z_OK) { - return ret; - } - if (this->bits) { - inflatePrime(s, this->bits, this->in_bits); - } - s->total_in = this->in; - s->total_out = this->out; - inflateSetDictionary(s, this->index, GZ_WINSIZE); - return ret; - } + indexDict(z_stream const& s, const file_size_t size); + + int apply(z_streamp s); }; private: @@ -181,12 +150,12 @@ public: int get_fd() const { return this->lb_fd; - }; + } time_t get_file_time() const { return this->lb_file_time; - }; + } /** * @return The size of the file or the amount of data pulled from a pipe. @@ -194,22 +163,22 @@ public: file_ssize_t get_file_size() const { return this->lb_file_size; - }; + } bool is_pipe() const { return !this->lb_seekable; - }; + } bool is_pipe_closed() const { return !this->lb_seekable && (this->lb_file_size != -1); - }; + } bool is_compressed() const { return this->lb_gz_file || this->lb_bz_file; - }; + } file_off_t get_read_offset(file_off_t off) const { @@ -218,7 +187,7 @@ public: } else { return off; } - }; + } bool is_data_available(file_off_t off, file_off_t stat_size) const { @@ -226,7 +195,7 @@ public: return (this->lb_file_size == -1 || off < this->lb_file_size); } return off < stat_size; - }; + } /** * Attempt to load the next line into the buffer. @@ -244,7 +213,7 @@ public: void clear() { this->lb_buffer_size = 0; - }; + } /** Release any resources held by this object. */ void reset() @@ -255,16 +224,16 @@ public: this->lb_file_size = (ssize_t) -1; this->lb_buffer_size = 0; this->lb_last_line_offset = -1; - }; + } /** Check the invariants for this object. */ - bool invariant() + bool invariant() const { require(this->lb_buffer != nullptr); require(this->lb_buffer_size <= this->lb_buffer_max); return true; - }; + } private: /** @@ -275,7 +244,7 @@ private: { return this->lb_file_offset <= off && off < (this->lb_file_offset + this->lb_buffer_size); - }; + } void resize_buffer(size_t new_max); @@ -325,7 +294,7 @@ private: avail_out = this->lb_buffer_size - buffer_offset; return retval; - }; + } shared_buffer lb_share_manager; diff --git a/src/listview_curses.cc b/src/listview_curses.cc index 6d4a0e76..97498c50 100644 --- a/src/listview_curses.cc +++ b/src/listview_curses.cc @@ -397,14 +397,15 @@ listview_curses::handle_mouse(mouse_event& me) this->lv_mouse_time = me.me_time; if (me.me_button != mouse_button_t::BUTTON_LEFT || inner_height == 0 - || (this->lv_mouse_mode != LV_MODE_DRAG && me.me_x < (int) (width - 2))) + || (this->lv_mouse_mode != lv_mode_t::DRAG + && me.me_x < (int) (width - 2))) { return false; } if (me.me_state == mouse_button_state_t::BUTTON_STATE_RELEASED) { this->lv_mouse_y = -1; - this->lv_mouse_mode = LV_MODE_NONE; + this->lv_mouse_mode = lv_mode_t::NONE; return true; } @@ -416,35 +417,35 @@ listview_curses::handle_mouse(mouse_event& me) scroll_top = (this->get_y() + (int) (top_pct * (double) height)); scroll_bottom = (this->get_y() + (int) (bot_pct * (double) height)); - if (this->lv_mouse_mode == LV_MODE_NONE) { + if (this->lv_mouse_mode == lv_mode_t::NONE) { if ((scroll_top - 1) <= me.me_y && me.me_y <= (scroll_bottom + 1)) { - this->lv_mouse_mode = LV_MODE_DRAG; + this->lv_mouse_mode = lv_mode_t::DRAG; this->lv_mouse_y = me.me_y - scroll_top; } else if (me.me_y < scroll_top) { - this->lv_mouse_mode = LV_MODE_UP; + this->lv_mouse_mode = lv_mode_t::UP; } else { - this->lv_mouse_mode = LV_MODE_DOWN; + this->lv_mouse_mode = lv_mode_t::DOWN; } } switch (this->lv_mouse_mode) { - case LV_MODE_NONE: + case lv_mode_t::NONE: require(0); break; - case LV_MODE_UP: + case lv_mode_t::UP: if (me.me_y < scroll_top) { shift_amount = -1 * height; } break; - case LV_MODE_DOWN: + case lv_mode_t::DOWN: if (me.me_y > scroll_bottom) { shift_amount = height; } break; - case LV_MODE_DRAG: + case lv_mode_t::DRAG: pct = (double) inner_height / (double) height; new_top = me.me_y - this->get_y() - this->lv_mouse_y; new_top = (int) floor(((double) new_top * pct) + 0.5); @@ -490,3 +491,57 @@ listview_curses::set_top(vis_line_t top, bool suppress_flash) this->set_needs_update(); } } + +vis_line_t +listview_curses::get_bottom() const +{ + auto retval = this->lv_top; + auto avail = this->rows_available(retval, RD_DOWN); + + if (avail > 0) { + retval += vis_line_t(avail - 1); + } + + return retval; +} + +vis_line_t +listview_curses::rows_available(vis_line_t line, + listview_curses::row_direction_t dir) const +{ + unsigned long width; + vis_line_t height; + vis_line_t retval(0); + + this->get_dimensions(height, width); + if (this->lv_word_wrap) { + size_t row_count = this->lv_source->listview_rows(*this); + + width -= 1; + while ((height > 0) && (line >= 0) && ((size_t) line < row_count)) { + size_t len = this->lv_source->listview_size_for_row(*this, line); + + do { + len -= std::min((size_t) width, len); + --height; + } while (len > 0); + line += vis_line_t(dir); + if (height >= 0) { + ++retval; + } + } + } else { + switch (dir) { + case RD_UP: + retval = std::min(height, line + 1_vl); + break; + case RD_DOWN: + retval = std::min( + height, + vis_line_t(this->lv_source->listview_rows(*this) - line)); + break; + } + } + + return retval; +} diff --git a/src/listview_curses.hh b/src/listview_curses.hh index 07bdb51f..be9c3a3a 100644 --- a/src/listview_curses.hh +++ b/src/listview_curses.hh @@ -55,10 +55,7 @@ public: /** @return The number of rows in the list. */ virtual size_t listview_rows(const listview_curses& lv) = 0; - virtual size_t listview_width(const listview_curses& lv) - { - return INT_MAX; - }; + virtual size_t listview_width(const listview_curses& lv) { return INT_MAX; } /** * Get the string value for a row in the list. @@ -78,7 +75,7 @@ public: virtual std::string listview_source_name(const listview_curses& lv) { return ""; - }; + } }; class list_gutter_source { @@ -94,7 +91,7 @@ public: role_t& bar_role_out) { ch_out = ACS_VLINE; - }; + } }; class list_overlay_source { @@ -115,7 +112,7 @@ public: virtual bool list_input_handle_key(listview_curses& lv, int ch) = 0; - virtual void list_input_handle_scroll_out(listview_curses& lv){}; + virtual void list_input_handle_scroll_out(listview_curses& lv) {} }; /** @@ -129,15 +126,9 @@ public: listview_curses(); - void set_title(const std::string& title) - { - this->lv_title = title; - }; + void set_title(const std::string& title) { this->lv_title = title; } - const std::string& get_title() const - { - return this->lv_title; - }; + const std::string& get_title() const { return this->lv_title; } /** @param src The data source delegate. */ void set_data_source(list_data_source* src) @@ -145,18 +136,15 @@ public: this->lv_source = src; this->invoke_scroll(); this->reload_data(); - }; + } /** @return The data source delegate. */ - list_data_source* get_data_source() const - { - return this->lv_source; - }; + list_data_source* get_data_source() const { return this->lv_source; } void set_gutter_source(list_gutter_source* src) { this->lv_gutter_source = src; - }; + } /** @param src The data source delegate. */ listview_curses& set_overlay_source(list_overlay_source* src) @@ -165,38 +153,30 @@ public: this->reload_data(); return *this; - }; + } /** @return The overlay source delegate. */ list_overlay_source* get_overlay_source() { return this->lv_overlay_source; - }; + } listview_curses& add_input_delegate(list_input_delegate& lid) { this->lv_input_delegates.push_back(&lid); return *this; - }; + } /** * @param va The action to invoke when the view is scrolled. * @todo Allow multiple observers. */ - void set_scroll_action(action va) - { - this->lv_scroll = std::move(va); - }; + void set_scroll_action(action va) { this->lv_scroll = std::move(va); } - void set_show_scrollbar(bool ss) - { - this->lv_show_scrollbar = ss; - }; - bool get_show_scrollbar() const - { - return this->lv_show_scrollbar; - }; + void set_show_scrollbar(bool ss) { this->lv_show_scrollbar = ss; } + + bool get_show_scrollbar() const { return this->lv_show_scrollbar; } void set_show_bottom_border(bool val) { @@ -204,21 +184,12 @@ public: this->lv_show_bottom_border = val; this->set_needs_update(); } - }; - bool get_show_bottom_border() const - { - return this->lv_show_bottom_border; - }; + } + bool get_show_bottom_border() const { return this->lv_show_bottom_border; } - void set_selectable(bool sel) - { - this->lv_selectable = sel; - }; + void set_selectable(bool sel) { this->lv_selectable = sel; } - bool is_selectable() const - { - return this->lv_selectable; - }; + bool is_selectable() const { return this->lv_selectable; } void set_selection(vis_line_t sel) { @@ -266,58 +237,16 @@ public: this->set_needs_update(); return *this; - }; + } - bool get_word_wrap() const - { - return this->lv_word_wrap; - }; + bool get_word_wrap() const { return this->lv_word_wrap; } enum row_direction_t { RD_UP = -1, RD_DOWN = 1, }; - vis_line_t rows_available(vis_line_t line, row_direction_t dir) const - { - unsigned long width; - vis_line_t height; - vis_line_t retval(0); - - this->get_dimensions(height, width); - if (this->lv_word_wrap) { - size_t row_count = this->lv_source->listview_rows(*this); - - width -= 1; - while ((height > 0) && (line >= 0) && ((size_t) line < row_count)) { - size_t len - = this->lv_source->listview_size_for_row(*this, line); - - do { - len -= std::min((size_t) width, len); - --height; - } while (len > 0); - line += vis_line_t(dir); - if (height >= 0) { - ++retval; - } - } - } else { - switch (dir) { - case RD_UP: - retval = std::min(height, line + 1_vl); - break; - case RD_DOWN: - retval = std::min( - height, - vis_line_t(this->lv_source->listview_rows(*this) - - line)); - break; - } - } - - return retval; - }; + vis_line_t rows_available(vis_line_t line, row_direction_t dir) const; template auto map_top_row(F func) -> @@ -334,16 +263,10 @@ public: } /** @param win The curses window this view is attached to. */ - void set_window(WINDOW* win) - { - this->lv_window = win; - }; + void set_window(WINDOW* win) { this->lv_window = win; } /** @return The curses window this view is attached to. */ - WINDOW* get_window() const - { - return this->lv_window; - }; + WINDOW* get_window() const { return this->lv_window; } void set_y(unsigned int y) { @@ -351,11 +274,9 @@ public: this->lv_y = y; this->set_needs_update(); } - }; - unsigned int get_y() const - { - return this->lv_y; - }; + } + + unsigned int get_y() const { return this->lv_y; } void set_x(unsigned int x) { @@ -363,11 +284,9 @@ public: this->lv_x = x; this->set_needs_update(); } - }; - unsigned int get_x() const - { - return this->lv_x; - }; + } + + unsigned int get_x() const { return this->lv_x; } /** * Set the line number to be displayed at the top of the view. If the @@ -380,10 +299,7 @@ public: void set_top(vis_line_t top, bool suppress_flash = false); /** @return The line number that is displayed at the top. */ - vis_line_t get_top() const - { - return this->lv_top; - }; + vis_line_t get_top() const { return this->lv_top; } nonstd::optional get_top_opt() const { @@ -395,19 +311,7 @@ public: } /** @return The line number that is displayed at the bottom. */ - vis_line_t get_bottom() const - { - auto retval = this->lv_top; - auto avail = this->rows_available(retval, RD_DOWN); - - if (avail > 0) { - retval += vis_line_t(avail - 1); - } else { - retval = -1_vl; - } - - return retval; - }; + vis_line_t get_bottom() const; vis_line_t get_top_for_last_row() { @@ -424,13 +328,13 @@ public: } return retval; - }; + } /** @return True if the given line is visible. */ bool is_line_visible(vis_line_t line) const { return this->get_top() <= line && line <= this->get_bottom(); - }; + } /** * Shift the value of top by the given value. @@ -451,7 +355,7 @@ public: } return this->lv_top; - }; + } /** * Set the column number to be displayed at the left of the view. If the @@ -480,13 +384,10 @@ public: this->lv_left = left; this->invoke_scroll(); this->set_needs_update(); - }; + } /** @return The column number that is displayed at the left. */ - unsigned int get_left() const - { - return this->lv_left; - }; + unsigned int get_left() const { return this->lv_left; } /** * Shift the value of left by the given value. @@ -505,7 +406,7 @@ public: } return this->lv_left; - }; + } /** * Set the height of the view. A value greater than one is considered to @@ -520,13 +421,10 @@ public: this->lv_height = height; this->set_needs_update(); } - }; + } /** @return The absolute or relative height of the window. */ - vis_line_t get_height() const - { - return this->lv_height; - }; + vis_line_t get_height() const { return this->lv_height; } /** @return The number of rows of data in this view's source data. */ vis_line_t get_inner_height() const @@ -534,19 +432,16 @@ public: return vis_line_t(this->lv_source == nullptr ? 0 : this->lv_source->listview_rows(*this)); - }; + } size_t get_inner_width() const { return this->lv_source == nullptr ? 0 : this->lv_source->listview_width(*this); - }; + } - void set_overlay_needs_update() - { - this->lv_overlay_needs_update = true; - }; + void set_overlay_needs_update() { this->lv_overlay_needs_update = true; } /** * Get the actual dimensions of the view. @@ -579,7 +474,7 @@ public: } else { width_out = 0; } - }; + } std::pair get_dimensions() const { @@ -619,13 +514,10 @@ public: log_debug(" lv_title=%s", this->lv_title.c_str()); log_debug(" lv_y=%u", this->lv_y); log_debug(" lv_top=%d", (int) this->lv_top); - }; - - virtual void invoke_scroll() - { - this->lv_scroll(this); } + virtual void invoke_scroll() { this->lv_scroll(this); } + protected: void delegate_scroll_out() { @@ -634,11 +526,11 @@ protected: } } - enum lv_mode_t { - LV_MODE_NONE, - LV_MODE_DOWN, - LV_MODE_UP, - LV_MODE_DRAG + enum class lv_mode_t { + NONE, + DOWN, + UP, + DRAG }; static list_gutter_source DEFAULT_GUTTER_SOURCE; @@ -667,7 +559,7 @@ protected: int lv_scroll_accel{1}; int lv_scroll_velo; int lv_mouse_y{-1}; - lv_mode_t lv_mouse_mode{LV_MODE_NONE}; + lv_mode_t lv_mouse_mode{lv_mode_t::NONE}; vis_line_t lv_tail_space{1}; }; #endif diff --git a/src/lnav.cc b/src/lnav.cc index 4f02dfb0..d48ec995 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -39,7 +39,6 @@ #include #include #include -#include #include #include #include @@ -65,6 +64,7 @@ #include #include #include +#include #include #include @@ -79,7 +79,6 @@ #include "base/func_util.hh" #include "base/future_util.hh" #include "base/humanize.hh" -#include "base/humanize.network.hh" #include "base/humanize.time.hh" #include "base/injector.bind.hh" #include "base/isc.hh" @@ -90,7 +89,8 @@ #include "bookmarks.hh" #include "bottom_status_source.hh" #include "bound_tags.hh" -#include "column_namer.hh" +#include "CLI/CLI.hpp" +#include "dump_internals.hh" #include "environ_vtab.hh" #include "fstat_vtab.hh" #include "grep_proc.hh" @@ -99,6 +99,8 @@ #include "init-sql.h" #include "listview_curses.hh" #include "lnav.hh" +#include "lnav.indexing.hh" +#include "lnav.management_cli.hh" #include "lnav_commands.hh" #include "lnav_config.hh" #include "lnav_util.hh" @@ -122,10 +124,10 @@ #include "term_extra.hh" #include "termios_guard.hh" #include "textfile_highlighters.hh" -#include "textfile_sub_source.hh" #include "textview_curses.hh" #include "top_status_source.hh" #include "view_helpers.examples.hh" +#include "view_helpers.hist.hh" #include "views_vtab.hh" #include "vt52_curses.hh" #include "xpath_vtab.hh" @@ -159,8 +161,7 @@ using namespace std::literals::chrono_literals; using namespace lnav::roles::literals; -static bool initial_build = false; -static std::multimap DEFAULT_FILES; +static std::vector DEFAULT_FILES; static auto intern_lifetime = intern_string::get_table_lifetime(); struct lnav_data_t lnav_data; @@ -195,17 +196,6 @@ const char* lnav_zoom_strings[] = { nullptr, }; -static const char* view_titles[LNV__MAX] = { - "LOG", - "TEXT", - "HELP", - "HIST", - "DB", - "SCHEMA", - "PRETTY", - "SPECTRO", -}; - static std::vector DEFAULT_DB_KEY_NAMES = { "match_index", "capture_index", @@ -285,21 +275,6 @@ force_linking(services::main_t anno) } } // namespace injector -void -add_recent_netlocs_possibilities() -{ - readline_curses* rc = lnav_data.ld_rl_view; - - rc->clear_possibilities(LNM_COMMAND, "recent-netlocs"); - std::set netlocs; - - isc::to().send_and_wait( - [&netlocs](auto& tlooper) { netlocs = tlooper.active_netlocs(); }); - netlocs.insert(session_data.sd_recent_netlocs.begin(), - session_data.sd_recent_netlocs.end()); - rc->add_possibility(LNM_COMMAND, "recent-netlocs", netlocs); -} - bool setup_logline_table(exec_context& ec) { @@ -319,9 +294,12 @@ setup_logline_table(exec_context& ec) = (lnav_data.ld_rl_view != nullptr && ec.ec_local_vars.size() == 1); if (update_possibilities) { - lnav_data.ld_rl_view->clear_possibilities(LNM_SQL, "*"); - add_view_text_possibilities( - lnav_data.ld_rl_view, LNM_SQL, "*", &log_view); + lnav_data.ld_rl_view->clear_possibilities(ln_mode_t::SQL, "*"); + add_view_text_possibilities(lnav_data.ld_rl_view, + ln_mode_t::SQL, + "*", + &log_view, + text_quoting::sql); } if (log_view.get_inner_height()) { @@ -349,7 +327,7 @@ setup_logline_table(exec_context& ec) { for (size_t lpc = 0; lpc < pair_iter->second.size(); lpc++) { lnav_data.ld_rl_view->add_possibility( - LNM_SQL, + ln_mode_t::SQL, "*", ldh.format_json_getter(pair_iter->first, lpc)); } @@ -364,13 +342,16 @@ setup_logline_table(exec_context& ec) db_key_names = DEFAULT_DB_KEY_NAMES; if (update_possibilities) { - add_env_possibilities(LNM_SQL); + add_env_possibilities(ln_mode_t::SQL); + lnav_data.ld_rl_view->add_possibility(ln_mode_t::SQL, + "*", + std::begin(sql_keywords), + std::end(sql_keywords)); lnav_data.ld_rl_view->add_possibility( - LNM_SQL, "*", std::begin(sql_keywords), std::end(sql_keywords)); - lnav_data.ld_rl_view->add_possibility(LNM_SQL, "*", sql_function_names); + ln_mode_t::SQL, "*", sql_function_names); lnav_data.ld_rl_view->add_possibility( - LNM_SQL, "*", hidden_table_columns); + ln_mode_t::SQL, "*", hidden_table_columns); for (int lpc = 0; sqlite_registration_funcs[lpc]; lpc++) { struct FuncDef* basic_funcs; @@ -381,7 +362,7 @@ setup_logline_table(exec_context& ec) const FuncDef& func_def = basic_funcs[lpc2]; lnav_data.ld_rl_view->add_possibility( - LNM_SQL, + ln_mode_t::SQL, "*", std::string(func_def.zName) + (func_def.nArg ? "(" : "()")); } @@ -389,7 +370,7 @@ setup_logline_table(exec_context& ec) const FuncDefAgg& func_def = agg_funcs[lpc2]; lnav_data.ld_rl_view->add_possibility( - LNM_SQL, + ln_mode_t::SQL, "*", std::string(func_def.zName) + (func_def.nArg ? "(" : "()")); } @@ -402,7 +383,8 @@ setup_logline_table(exec_context& ec) std::string poss = pair.first + (pair.second->ht_parameters.empty() ? "()" : ("(")); - lnav_data.ld_rl_view->add_possibility(LNM_SQL, "*", poss); + lnav_data.ld_rl_view->add_possibility( + ln_mode_t::SQL, "*", poss); break; } default: @@ -422,353 +404,19 @@ setup_logline_table(exec_context& ec) return retval; } -/** - * Observer for loading progress that updates the bottom status bar. - */ -class loading_observer : public logfile_observer { -public: - loading_observer() : lo_last_offset(0){}; - - indexing_result logfile_indexing(const std::shared_ptr& lf, - file_off_t off, - file_size_t total) override - { - static sig_atomic_t index_counter = 0; - - if (lnav_data.ld_flags & (LNF_HEADLESS | LNF_CHECK_CONFIG)) { - return indexing_result::CONTINUE; - } - - /* XXX require(off <= total); */ - if (off > (off_t) total) { - off = total; - } - - if ((((size_t) off == total) && (this->lo_last_offset != off)) - || ui_periodic_timer::singleton().time_to_update(index_counter)) - { - lnav_data.ld_bottom_source.update_loading(off, total); - do_update(lf); - this->lo_last_offset = off; - } - - if (!lnav_data.ld_looping) { - return indexing_result::BREAK; - } - return indexing_result::CONTINUE; - }; - - static void do_update(const std::shared_ptr& lf) - { - if (isendwin()) { - return; - } - lnav_data.ld_top_source.update_time(); - for (auto& sc : lnav_data.ld_status) { - sc.do_update(); - } - if (lf && lnav_data.ld_mode == LNM_FILES && !initial_build) { - auto& fc = lnav_data.ld_active_files; - auto iter = std::find(fc.fc_files.begin(), fc.fc_files.end(), lf); - - if (iter != fc.fc_files.end()) { - auto index = std::distance(fc.fc_files.begin(), iter); - lnav_data.ld_files_view.set_selection( - vis_line_t(fc.fc_other_files.size() + index)); - lnav_data.ld_files_view.reload_data(); - lnav_data.ld_files_view.do_update(); - } - } - refresh(); - }; - - off_t lo_last_offset; -}; - -class hist_index_delegate : public index_delegate { -public: - hist_index_delegate(hist_source2& hs, textview_curses& tc) - : hid_source(hs), hid_view(tc){ - - }; - - void index_start(logfile_sub_source& lss) override - { - this->hid_source.clear(); - }; - - void index_line(logfile_sub_source& lss, - logfile* lf, - logfile::iterator ll) override - { - if (ll->is_continued() || ll->get_time() == 0) { - return; - } - - hist_source2::hist_type_t ht; - - switch (ll->get_msg_level()) { - case LEVEL_FATAL: - case LEVEL_CRITICAL: - case LEVEL_ERROR: - ht = hist_source2::HT_ERROR; - break; - case LEVEL_WARNING: - ht = hist_source2::HT_WARNING; - break; - default: - ht = hist_source2::HT_NORMAL; - break; - } - - this->hid_source.add_value(ll->get_time(), ht); - if (ll->is_marked() || ll->is_expr_marked()) { - this->hid_source.add_value(ll->get_time(), hist_source2::HT_MARK); - } - }; - - void index_complete(logfile_sub_source& lss) override - { - this->hid_view.reload_data(); - }; - -private: - hist_source2& hid_source; - textview_curses& hid_view; -}; - -void -rebuild_hist() -{ - logfile_sub_source& lss = lnav_data.ld_log_source; - hist_source2& hs = lnav_data.ld_hist_source2; - int zoom = lnav_data.ld_zoom_level; - - hs.set_time_slice(ZOOM_LEVELS[zoom]); - lss.reload_index_delegate(); -} - -class textfile_callback { -public: - textfile_callback() : front_file(nullptr), front_top(-1){}; - - void closed_files(const std::vector>& files) - { - for (const auto& lf : files) { - log_info("closed text files: %s", lf->get_filename().c_str()); - } - lnav_data.ld_active_files.close_files(files); - }; - - void promote_file(const std::shared_ptr& lf) - { - if (lnav_data.ld_log_source.insert_file(lf)) { - this->did_promotion = true; - log_info("promoting text file to log file: %s (%s)", - lf->get_filename().c_str(), - lf->get_content_id().c_str()); - auto format = lf->get_format(); - if (format->lf_is_self_describing) { - auto vt = format->get_vtab_impl(); - - if (vt != nullptr) { - lnav_data.ld_vtab_manager->register_vtab(vt); - } - } - - auto iter = session_data.sd_file_states.find(lf->get_filename()); - if (iter != session_data.sd_file_states.end()) { - log_debug("found state for log file %d", - iter->second.fs_is_visible); - - lnav_data.ld_log_source.find_data(lf) | [&iter](auto ld) { - ld->set_visibility(iter->second.fs_is_visible); - }; - } - } else { - this->closed_files({lf}); - } - }; - - void scanned_file(const std::shared_ptr& lf) - { - if (!lnav_data.ld_files_to_front.empty() - && lnav_data.ld_files_to_front.front().first == lf->get_filename()) - { - this->front_file = lf; - this->front_top = lnav_data.ld_files_to_front.front().second; - - lnav_data.ld_files_to_front.pop_front(); - } - }; - - std::shared_ptr front_file; - int front_top; - bool did_promotion{false}; -}; - -size_t -rebuild_indexes(nonstd::optional deadline) -{ - logfile_sub_source& lss = lnav_data.ld_log_source; - textview_curses& log_view = lnav_data.ld_views[LNV_LOG]; - textview_curses& text_view = lnav_data.ld_views[LNV_TEXT]; - vis_line_t old_bottoms[LNV__MAX]; - bool scroll_downs[LNV__MAX]; - size_t retval = 0; - - for (int lpc = 0; lpc < LNV__MAX; lpc++) { - old_bottoms[lpc] = lnav_data.ld_views[lpc].get_top_for_last_row(); - scroll_downs[lpc] - = (lnav_data.ld_views[lpc].get_top() >= old_bottoms[lpc]) - && !(lnav_data.ld_flags & LNF_HEADLESS); - } - - { - textfile_sub_source* tss = &lnav_data.ld_text_source; - textfile_callback cb; - - if (tss->rescan_files(cb, deadline)) { - text_view.reload_data(); - retval += 1; - } - - if (cb.front_file != nullptr) { - ensure_view(&text_view); - - if (tss->current_file() != cb.front_file) { - tss->to_front(cb.front_file); - old_bottoms[LNV_TEXT] = -1_vl; - } - - if (cb.front_top < 0) { - cb.front_top += text_view.get_inner_height(); - } - if (cb.front_top < text_view.get_inner_height()) { - text_view.set_top(vis_line_t(cb.front_top)); - scroll_downs[LNV_TEXT] = false; - } - } - if (cb.did_promotion && deadline) { - // If there's a new log file, extend the deadline so it can be - // indexed quickly. - deadline = deadline.value() + 500ms; - } - } - - std::vector> closed_files; - for (auto& lf : lnav_data.ld_active_files.fc_files) { - if ((!lf->exists() || lf->is_closed())) { - log_info("closed log file: %s", lf->get_filename().c_str()); - lnav_data.ld_text_source.remove(lf); - lnav_data.ld_log_source.remove_file(lf); - closed_files.emplace_back(lf); - } - } - if (!closed_files.empty()) { - lnav_data.ld_active_files.close_files(closed_files); - } - - auto result = lss.rebuild_index(deadline); - if (result != logfile_sub_source::rebuild_result::rr_no_change) { - size_t new_count = lss.text_line_count(); - bool force - = result == logfile_sub_source::rebuild_result::rr_full_rebuild; - - if ((!scroll_downs[LNV_LOG] - || log_view.get_top() > vis_line_t(new_count)) - && force) - { - scroll_downs[LNV_LOG] = false; - } - - log_view.reload_data(); - - { - std::unordered_map>> - id_to_files; - bool reload = false; - - for (const auto& lf : lnav_data.ld_active_files.fc_files) { - id_to_files[lf->get_content_id()].push_back(lf); - } - - for (auto& pair : id_to_files) { - if (pair.second.size() == 1) { - continue; - } - - pair.second.sort([](const auto& left, const auto& right) { - return right->get_stat().st_size < left->get_stat().st_size; - }); - - auto dupe_name = pair.second.front()->get_unique_path(); - pair.second.pop_front(); - for_each(pair.second.begin(), - pair.second.end(), - [&dupe_name](auto& lf) { - log_info("Hiding duplicate file: %s", - lf->get_filename().c_str()); - lf->mark_as_duplicate(dupe_name); - lnav_data.ld_log_source.find_data(lf) | - [](auto ld) { ld->set_visibility(false); }; - }); - reload = true; - } - - if (reload) { - lss.text_filters_changed(); - } - } - - retval += 1; - } - - for (int lpc = 0; lpc < LNV__MAX; lpc++) { - textview_curses& scroll_view = lnav_data.ld_views[lpc]; - - if (scroll_downs[lpc] - && scroll_view.get_top_for_last_row() > scroll_view.get_top()) - { - scroll_view.set_top(scroll_view.get_top_for_last_row()); - } - } - - lnav_data.ld_view_stack.top() | [](auto tc) { - lnav_data.ld_filter_status_source.update_filtered(tc->get_sub_source()); - lnav_data.ld_scroll_broadcaster(tc); - }; - - return retval; -} - -void -rebuild_indexes_repeatedly() -{ - for (size_t attempt = 0; attempt < 10 && rebuild_indexes() > 0; attempt++) { - log_info("continuing to rebuild indexes..."); - } -} - static bool -append_default_files(lnav_flags_t flag) +append_default_files() { bool retval = true; - - if (lnav_data.ld_flags & flag) { auto cwd = ghc::filesystem::current_path(); - for (auto range = DEFAULT_FILES.equal_range(flag); - range.first != range.second; - range.first++) - { - auto path = range.first->second; - + for (const auto& path : DEFAULT_FILES) { if (access(path.c_str(), R_OK) == 0) { auto_mem abspath; - path = cwd / range.first->second; - if ((abspath = realpath(path.c_str(), nullptr)) == nullptr) { + auto full_path = cwd / path; + if ((abspath = realpath(full_path.c_str(), nullptr)) == nullptr) + { perror("Unable to resolve path"); } else { lnav_data.ld_active_files.fc_file_names[abspath.in()]; @@ -783,7 +431,6 @@ append_default_files(lnav_flags_t flag) retval = false; } } - } return retval; } @@ -826,25 +473,6 @@ handle_rl_key(int ch) } } -void -rl_focus(readline_curses* rc) -{ - auto fos = (field_overlay_source*) lnav_data.ld_views[LNV_LOG] - .get_overlay_source(); - - fos->fos_contexts.emplace("", false, true); -} - -void -rl_blur(readline_curses* rc) -{ - auto fos = (field_overlay_source*) lnav_data.ld_views[LNV_LOG] - .get_overlay_source(); - - fos->fos_contexts.pop(); - lnav_data.ld_preview_generation += 1; -} - readline_context::command_map_t lnav_commands; static attr_line_t @@ -861,7 +489,7 @@ command_arg_help() .append(" ") .append(";"_symbol) .append(" - ") - .append("an SQL statement (e.g. SELECT * FROM syslog_log)\n") + .append("an SQL statement (e.g. ;SELECT * FROM syslog_log)\n") .append(" ") .append("|"_symbol) .append(" - ") @@ -871,81 +499,221 @@ command_arg_help() static void usage() { - const char* usage_msg = R"(usage: %s [options] [logfile1 logfile2 ...] - -A curses-based log file viewer that indexes log messages by type -and time to make it easier to navigate through files quickly. - -Key bindings: - ? View/leave the online help text. - q Quit the program. - -Options: - -h Print this message, then exit. - -H Display the internal help text. - -I path An additional configuration directory. - -i Install the given format files and exit. Pass 'extra' - to install the default set of third-party formats. - -u Update formats installed from git repositories. - -C Check configuration and then exit. - -d path Write debug messages to the given file. - -V Print version information. - - -a Load all of the most recent log file types. - -r Recursively load files from the given directory hierarchies. - -R Load older rotated log files as well. - -t Prepend timestamps to the lines of data being read in - on the standard input. - -w file Write the contents of the standard input to this file. - - -c cmd Execute a command after the files have been loaded. - -f path Execute the commands in the given file. - -n Run without the curses UI. (headless mode) - -N Do not open the default syslog file if no files are given. - -q Do not print the log messages after executing all + attr_line_t ex1_term; + + ex1_term.append(lnav::roles::ok("$")) + .append(" ") + .append(lnav::roles::file("lnav")) + .pad_to(40); + ex1_term.get_attrs().emplace_back(line_range{0, (int) ex1_term.length()}, + VC_BACKGROUND.value(COLOR_BLACK)); + ex1_term.get_attrs().emplace_back(line_range{0, (int) ex1_term.length()}, + VC_FOREGROUND.value(COLOR_WHITE)); + + attr_line_t ex2_term; + + ex2_term.append(lnav::roles::ok("$")) + .append(" ") + .append(lnav::roles::file("lnav")) + .append(" ") + .append(lnav::roles::file("/var/log")) + .pad_to(40); + ex2_term.get_attrs().emplace_back(line_range{0, (int) ex2_term.length()}, + VC_BACKGROUND.value(COLOR_BLACK)); + ex2_term.get_attrs().emplace_back(line_range{0, (int) ex2_term.length()}, + VC_FOREGROUND.value(COLOR_WHITE)); + + attr_line_t ex3_term; + + ex3_term.append(lnav::roles::ok("$")) + .append(" ") + .append(lnav::roles::file("make")) + .append(" 2>&1 | ") + .append(lnav::roles::file("lnav")) + .append(" ") + .append("-t"_symbol) + .pad_to(40); + ex3_term.get_attrs().emplace_back(line_range{0, (int) ex3_term.length()}, + VC_BACKGROUND.value(COLOR_BLACK)); + ex3_term.get_attrs().emplace_back(line_range{0, (int) ex3_term.length()}, + VC_FOREGROUND.value(COLOR_WHITE)); + + attr_line_t usage_al; + + usage_al.append("usage"_h1) + .append(": ") + .append(lnav::roles::file(lnav_data.ld_program_name)) + .append(" [") + .append("options"_variable) + .append("] [") + .append("logfile1"_variable) + .append(" ") + .append("logfile2"_variable) + .append(" ") + .append("..."_variable) + .append("]\n") + .append(R"( +A log file viewer for the terminal that indexes log messages to +make it easier to navigate through files quickly. + +)") + .append("Key Bindings"_h2) + .append("\n") + .append(" ?"_symbol) + .append(" View/leave the online help text.\n") + .append(" q"_symbol) + .append(" Quit the program.\n") + .append("\n") + .append("Options"_h2) + .append("\n") + .append(" ") + .append("-h"_symbol) + .append(" ") + .append("Print this message, then exit.\n") + .append(" ") + .append("-H"_symbol) + .append(" ") + .append("Display the internal help text.\n") + .append("\n") + .append(" ") + .append("-I"_symbol) + .append(" ") + .append("dir"_variable) + .append(" ") + .append("An additional configuration directory.\n") + .append(" ") + .append("-u"_symbol) + .append(" ") + .append("Update formats installed from git repositories.\n") + .append(" ") + .append("-d"_symbol) + .append(" ") + .append("file"_variable) + .append(" ") + .append("Write debug messages to the given file.\n") + .append(" ") + .append("-V"_symbol) + .append(" ") + .append("Print version information.\n") + .append("\n") + .append(" ") + .append("-r"_symbol) + .append(" ") + .append( + "Recursively load files from the given directory hierarchies.\n") + .append(" ") + .append("-R"_symbol) + .append(" ") + .append("Load older rotated log files as well.\n") + .append(" ") + .append("-t"_symbol) + .append(" ") + .append(R"(Prepend timestamps to the lines of data being read in + from the standard input. +)") + .append(" ") + .append("-w"_symbol) + .append(" ") + .append("file"_variable) + .append(" ") + .append("Write the contents of the standard input to this file.\n") + .append("\n") + .append(" ") + .append("-c"_symbol) + .append(" ") + .append("cmd"_variable) + .append(" ") + .append("Execute a command after the files have been loaded.\n") + .append(" ") + .append("-f"_symbol) + .append(" ") + .append("file"_variable) + .append(" ") + .append("Execute the commands in the given file.\n") + .append(" ") + .append("-n"_symbol) + .append(" ") + .append("Run without the curses UI. (headless mode)\n") + .append(" ") + .append("-N"_symbol) + .append(" ") + .append("Do not open the default syslog file if no files are given.\n") + .append(" ") + .append("-q"_symbol) + .append(" ") + .append( + R"(Do not print the log messages after executing all of the commands. - -Optional arguments: - logfileN The log files, directories, or remote paths to view. - If a directory is given, all of the files in the - directory will be loaded. - -Examples: - To load and follow the syslog file: - $ lnav - - To load all of the files in /var/log: - $ lnav /var/log - - To watch the output of make with timestamps prepended: - $ make 2>&1 | lnav -t - -Paths: - Configuration, session, and format files are stored in: - %s %s - - Local copies of remote files and files extracted from - archives are stored in: - %s %s - -Documentation: https://docs.lnav.org -Contact: - %s https://github.com/tstack/lnav/discussions - %s %s -Version: %s -)"; - - fprintf(stderr, - usage_msg, - lnav_data.ld_program_name, - "\U0001F4C2", - lnav::paths::dotlnav().c_str(), - "\U0001F4C2", - lnav::paths::workdir().c_str(), - "\U0001F4AC", - "\U0001F4EB", - PACKAGE_BUGREPORT, - VCS_PACKAGE_STRING); +)") + .append("\n") + .append("Optional arguments"_h2) + .append("\n") + .append(" ") + .append("logfileN"_variable) + .append(R"( The log files, directories, or remote paths to view. + If a directory is given, all of the files in the + directory will be loaded. +)") + .append("\n") + .append("Management-Mode Options"_h2) + .append("\n") + .append(" ") + .append("-i"_symbol) + .append(" ") + .append(R"(Install the given format files and exit. Pass 'extra' + to install the default set of third-party formats. +)") + .append(" ") + .append("-m"_symbol) + .append(" ") + .append(R"(Switch to the management command-line mode. This mode + is used to work with lnav's configuration. +)") + .append("\n") + .append("Examples"_h2) + .append("\n ") + .append("\u2022"_list_glyph) + .append(" To load and follow the syslog file:\n") + .append(" ") + .append(ex1_term) + .append("\n\n ") + .append("\u2022"_list_glyph) + .append(" To load all of the files in ") + .append(lnav::roles::file("/var/log")) + .append(":\n") + .append(" ") + .append(ex2_term) + .append("\n\n ") + .append("\u2022"_list_glyph) + .append(" To watch the output of ") + .append(lnav::roles::file("make")) + .append(" with timestamps prepended:\n") + .append(" ") + .append(ex3_term) + .append("\n\n") + .append("Paths"_h2) + .append("\n ") + .append("\u2022"_list_glyph) + .append(" Configuration, session, and format files are stored in:\n") + .append(" \U0001F4C2 ") + .append(lnav::roles::file(lnav::paths::dotlnav().string())) + .append("\n\n ") + .append("\u2022"_list_glyph) + .append(" Local copies of remote files and files extracted from\n") + .append(" archives are stored in:\n") + .append(" \U0001F4C2 ") + .append(lnav::roles::file(lnav::paths::workdir().string())) + .append("\n\n") + .append("Documentation"_h1) + .append(": https://docs.lnav.org\n") + .append("Contact"_h1) + .append("\n") + .append(" \U0001F4AC https://github.com/tstack/lnav/discussions\n") + .append(FMT_STRING(" \U0001F4EB {}\n"), PACKAGE_BUGREPORT) + .append("Version"_h1) + .append(FMT_STRING(": {}"), VCS_PACKAGE_STRING); + + lnav::console::println(stderr, usage_al); } static void @@ -960,79 +728,6 @@ clear_last_user_mark(listview_curses* lv) } } -bool -update_active_files(file_collection& new_files) -{ - static loading_observer obs; - - if (lnav_data.ld_active_files.fc_invalidate_merge) { - lnav_data.ld_active_files.fc_invalidate_merge = false; - - return true; - } - - for (const auto& lf : new_files.fc_files) { - lf->set_logfile_observer(&obs); - lnav_data.ld_text_source.push_back(lf); - } - for (const auto& other_pair : new_files.fc_other_files) { - switch (other_pair.second.ofd_format) { - case file_format_t::SQLITE_DB: - attach_sqlite_db(lnav_data.ld_db.in(), other_pair.first); - break; - default: - break; - } - } - lnav_data.ld_active_files.merge(new_files); - if (!new_files.fc_files.empty() || !new_files.fc_other_files.empty() - || !new_files.fc_name_to_errors.empty()) - { - lnav_data.ld_active_files.regenerate_unique_file_names(); - } - lnav_data.ld_child_pollers.insert( - lnav_data.ld_child_pollers.begin(), - std::make_move_iterator( - lnav_data.ld_active_files.fc_child_pollers.begin()), - std::make_move_iterator( - lnav_data.ld_active_files.fc_child_pollers.end())); - - return true; -} - -bool -rescan_files(bool req) -{ - auto& mlooper = injector::get(); - bool done = false; - auto delay = 0ms; - - do { - auto fc = lnav_data.ld_active_files.rescan_files(req); - bool all_synced = true; - - update_active_files(fc); - mlooper.get_port().process_for(delay); - if (lnav_data.ld_flags & LNF_HEADLESS) { - for (const auto& pair : lnav_data.ld_active_files.fc_other_files) { - if (pair.second.ofd_format != file_format_t::REMOTE) { - continue; - } - - if (lnav_data.ld_active_files.fc_synced_files.count(pair.first) - == 0) { - all_synced = false; - } - } - if (!all_synced) { - delay = 30ms; - } - } - done = fc.fc_file_names.empty() && all_synced; - } while (!done); - return true; -} - class lnav_behavior : public mouse_behavior { public: void mouse_event(int button, bool release, int x, int y) override @@ -1080,10 +775,10 @@ handle_config_ui_key(int ch) bool retval = false; switch (lnav_data.ld_mode) { - case LNM_FILES: + case ln_mode_t::FILES: retval = lnav_data.ld_files_view.handle_key(ch); break; - case LNM_FILTER: + case ln_mode_t::FILTER: retval = lnav_data.ld_filter_view.handle_key(ch); break; default: @@ -1098,21 +793,23 @@ handle_config_ui_key(int ch) lnav_data.ld_filter_help_status_source.fss_error.clear(); if (ch == 'F') { - new_mode = LNM_FILES; + new_mode = ln_mode_t::FILES; } else if (ch == 'T') { - new_mode = LNM_FILTER; + new_mode = ln_mode_t::FILTER; } else if (ch == '\t' || ch == KEY_BTAB) { - if (lnav_data.ld_mode == LNM_FILES) { - new_mode = LNM_FILTER; + if (lnav_data.ld_mode == ln_mode_t::FILES) { + new_mode = ln_mode_t::FILTER; } else { - new_mode = LNM_FILES; + new_mode = ln_mode_t::FILES; } } else if (ch == 'q') { - new_mode = LNM_PAGING; + new_mode = ln_mode_t::PAGING; } if (new_mode) { - if (new_mode.value() == LNM_FILES || new_mode.value() == LNM_FILTER) { + if (new_mode.value() == ln_mode_t::FILES + || new_mode.value() == ln_mode_t::FILTER) + { lnav_data.ld_last_config_mode = new_mode.value(); } lnav_data.ld_mode = new_mode.value(); @@ -1137,21 +834,21 @@ handle_key(int ch) break; default: { switch (lnav_data.ld_mode) { - case LNM_PAGING: + case ln_mode_t::PAGING: return handle_paging_key(ch); - case LNM_FILTER: - case LNM_FILES: + case ln_mode_t::FILTER: + case ln_mode_t::FILES: return handle_config_ui_key(ch); - case LNM_COMMAND: - case LNM_SEARCH: - case LNM_SEARCH_FILTERS: - case LNM_SEARCH_FILES: - case LNM_CAPTURE: - case LNM_SQL: - case LNM_EXEC: - case LNM_USER: + case ln_mode_t::COMMAND: + case ln_mode_t::SEARCH: + case ln_mode_t::SEARCH_FILTERS: + case ln_mode_t::SEARCH_FILES: + case ln_mode_t::CAPTURE: + case ln_mode_t::SQL: + case ln_mode_t::EXEC: + case ln_mode_t::USER: handle_rl_key(ch); break; @@ -1168,7 +865,7 @@ handle_key(int ch) static input_dispatcher::escape_match_t match_escape_seq(const char* keyseq) { - if (lnav_data.ld_mode != LNM_PAGING) { + if (lnav_data.ld_mode != ln_mode_t::PAGING) { return input_dispatcher::escape_match_t::NONE; } @@ -1194,89 +891,6 @@ match_escape_seq(const char* keyseq) return input_dispatcher::escape_match_t::NONE; } -void -update_hits(textview_curses* tc) -{ - if (isendwin()) { - return; - } - - auto top_tc = lnav_data.ld_view_stack.top(); - - if (top_tc && tc == *top_tc) { - lnav_data.ld_bottom_source.update_hits(tc); - - if (lnav_data.ld_mode == LNM_SEARCH) { - const auto MAX_MATCH_COUNT = 10_vl; - const auto PREVIEW_SIZE = MAX_MATCH_COUNT + 1_vl; - - int preview_count = 0; - - vis_bookmarks& bm = tc->get_bookmarks(); - const auto& bv = bm[&textview_curses::BM_SEARCH]; - auto vl = tc->get_top(); - unsigned long width; - vis_line_t height; - attr_line_t all_matches; - char linebuf[64]; - int last_line = tc->get_inner_height(); - int max_line_width; - - snprintf(linebuf, sizeof(linebuf), "%d", last_line); - max_line_width = strlen(linebuf); - - tc->get_dimensions(height, width); - vl += height; - if (vl > PREVIEW_SIZE) { - vl -= PREVIEW_SIZE; - } - - auto prev_vl = bv.prev(tc->get_top()); - - if (prev_vl != -1_vl) { - attr_line_t al; - - tc->textview_value_for_row(prev_vl, al); - if (preview_count > 0) { - all_matches.append("\n"); - } - snprintf(linebuf, - sizeof(linebuf), - "L%*d: ", - max_line_width, - (int) prev_vl); - all_matches.append(linebuf).append(al); - preview_count += 1; - } - - while ((vl = bv.next(vl)) != -1_vl - && preview_count < MAX_MATCH_COUNT) { - attr_line_t al; - - tc->textview_value_for_row(vl, al); - if (preview_count > 0) { - all_matches.append("\n"); - } - snprintf(linebuf, - sizeof(linebuf), - "L%*d: ", - max_line_width, - (int) vl); - all_matches.append(linebuf).append(al); - preview_count += 1; - } - - if (preview_count > 0) { - lnav_data.ld_preview_status_source.get_description().set_value( - "Matching lines for search"); - lnav_data.ld_preview_source.replace_with(all_matches) - .set_text_format(text_format_t::TF_UNKNOWN); - lnav_data.ld_preview_view.set_needs_update(); - } - } - } -} - static void gather_pipers() { @@ -1312,10 +926,10 @@ wait_for_pipers() if (lnav_data.ld_pipers.empty() && lnav_data.ld_child_pollers.empty()) { log_debug("all pipers finished"); break; - } else { - usleep(10000); - rebuild_indexes(); } + usleep(10000); + rebuild_indexes(); + log_debug("%d pipers and %d children still active", lnav_data.ld_pipers.size(), lnav_data.ld_child_pollers.size()); @@ -1326,8 +940,15 @@ static void looper() { try { - auto sql_cmd_map = injector::get(); + auto_fd errpipe[2]; + auto_fd::pipe(errpipe); + + dup2(errpipe[1], STDERR_FILENO); + errpipe[1].reset(); + log_pipe_err(errpipe[0]); + + auto* sql_cmd_map = injector::get(); auto& ec = lnav_data.ld_exec_context; readline_context command_context("cmd", &lnav_commands); @@ -1360,20 +981,20 @@ looper() lnav_data.ld_log_source.lss_sorting_observer = [](auto& lss, auto off, auto size) { lnav_data.ld_bottom_source.update_loading(off, size); - loading_observer::do_update(nullptr); + do_observer_update(nullptr); }; auto& sb = lnav_data.ld_scroll_broadcaster; auto& vsb = lnav_data.ld_view_stack_broadcaster; - rlc.add_context(LNM_COMMAND, command_context); - rlc.add_context(LNM_SEARCH, search_context); - rlc.add_context(LNM_SEARCH_FILTERS, search_filters_context); - rlc.add_context(LNM_SEARCH_FILES, search_files_context); - rlc.add_context(LNM_CAPTURE, index_context); - rlc.add_context(LNM_SQL, sql_context); - rlc.add_context(LNM_EXEC, exec_context); - rlc.add_context(LNM_USER, user_context); + rlc.add_context(ln_mode_t::COMMAND, command_context); + rlc.add_context(ln_mode_t::SEARCH, search_context); + rlc.add_context(ln_mode_t::SEARCH_FILTERS, search_filters_context); + rlc.add_context(ln_mode_t::SEARCH_FILES, search_files_context); + rlc.add_context(ln_mode_t::CAPTURE, index_context); + rlc.add_context(ln_mode_t::SQL, sql_context); + rlc.add_context(ln_mode_t::EXEC, exec_context); + rlc.add_context(ln_mode_t::USER, user_context); rlc.start(); lnav_data.ld_filter_source.fss_editor.start(); @@ -1381,13 +1002,13 @@ looper() lnav_data.ld_rl_view = &rlc; lnav_data.ld_rl_view->add_possibility( - LNM_COMMAND, "viewname", lnav_view_strings); + ln_mode_t::COMMAND, "viewname", lnav_view_strings); lnav_data.ld_rl_view->add_possibility( - LNM_COMMAND, "zoomlevel", lnav_zoom_strings); + ln_mode_t::COMMAND, "zoomlevel", lnav_zoom_strings); lnav_data.ld_rl_view->add_possibility( - LNM_COMMAND, "levelname", level_names); + ln_mode_t::COMMAND, "levelname", level_names); (void) signal(SIGINT, sigint); (void) signal(SIGTERM, sigint); @@ -1397,13 +1018,6 @@ looper() screen_curses sc; lnav_behavior lb; - auto_fd errpipe[2]; - auto_fd::pipe(errpipe); - - dup2(errpipe[1], STDERR_FILENO); - errpipe[1].reset(); - log_pipe_err(errpipe[0]); - ui_periodic_timer::singleton(); auto mouse_i = injector::get(); @@ -1609,7 +1223,7 @@ looper() static sig_atomic_t index_counter; - lnav_data.ld_mode = LNM_FILES; + lnav_data.ld_mode = ln_mode_t::FILES; timer.start_fade(index_counter, 1); @@ -1751,21 +1365,22 @@ looper() lnav_data.ld_filter_source.fss_match_view.set_needs_update(); } switch (lnav_data.ld_mode) { - case LNM_FILTER: - case LNM_SEARCH_FILTERS: + case ln_mode_t::FILTER: + case ln_mode_t::SEARCH_FILTERS: lnav_data.ld_filter_view.set_needs_update(); lnav_data.ld_filter_view.do_update(); break; - case LNM_SEARCH_FILES: - case LNM_FILES: + case ln_mode_t::SEARCH_FILES: + case ln_mode_t::FILES: lnav_data.ld_files_view.set_needs_update(); lnav_data.ld_files_view.do_update(); break; default: break; } - if (lnav_data.ld_mode != LNM_FILTER - && lnav_data.ld_mode != LNM_FILES) { + if (lnav_data.ld_mode != ln_mode_t::FILTER + && lnav_data.ld_mode != ln_mode_t::FILES) + { rlc.do_update(); } refresh(); @@ -1773,15 +1388,15 @@ looper() if (lnav_data.ld_session_loaded) { // Only take input from the user after everything has loaded. pollfds.push_back((struct pollfd){STDIN_FILENO, POLLIN, 0}); - if (initial_build) { + if (lnav_data.ld_initial_build) { switch (lnav_data.ld_mode) { - case LNM_COMMAND: - case LNM_SEARCH: - case LNM_SEARCH_FILTERS: - case LNM_SEARCH_FILES: - case LNM_SQL: - case LNM_EXEC: - case LNM_USER: + case ln_mode_t::COMMAND: + case ln_mode_t::SEARCH: + case ln_mode_t::SEARCH_FILTERS: + case ln_mode_t::SEARCH_FILES: + case ln_mode_t::SQL: + case ln_mode_t::EXEC: + case ln_mode_t::USER: if (rlc.consume_ready_for_input()) { // log_debug("waiting for readline input") view_curses::awaiting_user_input(); @@ -1864,19 +1479,19 @@ looper() next_status_update_time = ui_clock::now(); switch (lnav_data.ld_mode) { - case LNM_PAGING: - case LNM_FILTER: - case LNM_FILES: + case ln_mode_t::PAGING: + case ln_mode_t::FILTER: + case ln_mode_t::FILES: next_rescan_time = next_status_update_time + 1s; break; - case LNM_COMMAND: - case LNM_SEARCH: - case LNM_SEARCH_FILTERS: - case LNM_SEARCH_FILES: - case LNM_CAPTURE: - case LNM_SQL: - case LNM_EXEC: - case LNM_USER: + case ln_mode_t::COMMAND: + case ln_mode_t::SEARCH: + case ln_mode_t::SEARCH_FILTERS: + case ln_mode_t::SEARCH_FILES: + case ln_mode_t::CAPTURE: + case ln_mode_t::SQL: + case ln_mode_t::EXEC: + case ln_mode_t::USER: next_rescan_time = next_status_update_time + 1min; break; } @@ -1900,9 +1515,9 @@ looper() if (lnav_data.ld_mode != old_mode) { switch (lnav_data.ld_mode) { - case LNM_PAGING: - case LNM_FILTER: - case LNM_FILES: + case ln_mode_t::PAGING: + case ln_mode_t::FILTER: + case ln_mode_t::FILES: next_rescan_time = next_status_update_time + 1s; next_rebuild_time = next_rescan_time; break; @@ -1924,16 +1539,17 @@ looper() } if (initial_rescan_completed && session_stage < 2 - && (!initial_build || timer.fade_diff(index_counter) == 0)) + && (!lnav_data.ld_initial_build + || timer.fade_diff(index_counter) == 0)) { - if (lnav_data.ld_mode == LNM_PAGING) { + if (lnav_data.ld_mode == ln_mode_t::PAGING) { timer.start_fade(index_counter, 1); } else { timer.start_fade(index_counter, 3); } // log_debug("initial build rebuild"); changes += rebuild_indexes(loop_deadline); - if (!initial_build + if (!lnav_data.ld_initial_build && lnav_data.ld_log_source.text_line_count() == 0 && lnav_data.ld_text_source.text_line_count() > 0) { @@ -1953,7 +1569,7 @@ looper() lnav_data.ld_views[LNV_LOG].set_top( tc_log->get_top_for_last_row()); } - if (!initial_build + if (!lnav_data.ld_initial_build && lnav_data.ld_log_source.text_line_count() == 0 && !lnav_data.ld_active_files.fc_other_files.empty() && std::any_of( @@ -1967,21 +1583,24 @@ looper() ensure_view(&lnav_data.ld_views[LNV_SCHEMA]); } - if (!initial_build && lnav_data.ld_flags & LNF_HELP) { + if (!lnav_data.ld_initial_build && lnav_data.ld_show_help_view) + { toggle_view(&lnav_data.ld_views[LNV_HELP]); - initial_build = true; + lnav_data.ld_initial_build = true; } - if (!initial_build && lnav_data.ld_flags & LNF_NO_DEFAULT) { - initial_build = true; + if (!lnav_data.ld_initial_build + && lnav_data.ld_active_files.fc_file_names.empty()) + { + lnav_data.ld_initial_build = true; } if (lnav_data.ld_log_source.text_line_count() > 0 || lnav_data.ld_text_source.text_line_count() > 0 || !lnav_data.ld_active_files.fc_other_files.empty()) { - initial_build = true; + lnav_data.ld_initial_build = true; } - if (initial_build) { + if (lnav_data.ld_initial_build) { static bool ran_cleanup = false; std::vector, @@ -2021,11 +1640,11 @@ looper() vis_line_t(vs.vs_top)); } } - if (lnav_data.ld_mode == LNM_FILES) { + if (lnav_data.ld_mode == ln_mode_t::FILES) { if (lnav_data.ld_active_files.fc_name_to_errors.empty()) { log_debug("switching to paging!"); - lnav_data.ld_mode = LNM_PAGING; + lnav_data.ld_mode = ln_mode_t::PAGING; } else { lnav_data.ld_files_view.set_selection(0_vl); } @@ -2143,43 +1762,67 @@ wait_for_children() } while (true); } -textview_curses* -get_textview_for_mode(ln_mode_t mode) +static int +print_user_msgs(std::vector error_list) { - switch (mode) { - case LNM_SEARCH_FILTERS: - case LNM_FILTER: - return &lnav_data.ld_filter_view; - case LNM_SEARCH_FILES: - case LNM_FILES: - return &lnav_data.ld_files_view; - default: - return *lnav_data.ld_view_stack.top(); - } -} + int retval = EXIT_SUCCESS; -static void -print_errors(std::vector error_list) -{ for (auto& iter : error_list) { - lnav::console::print(stderr, iter); + FILE* out_file; + + switch (iter.um_level) { + case lnav::console::user_message::level::raw: + case lnav::console::user_message::level::ok: + out_file = stdout; + break; + default: + out_file = stderr; + break; + } + + lnav::console::print(out_file, iter); + if (iter.um_level == lnav::console::user_message::level::error) { + retval = EXIT_FAILURE; + } } + + return retval; } +struct mode_flags_t { + bool mf_check_configs{false}; + bool mf_install{false}; + bool mf_update_formats{false}; + bool mf_no_default{false}; +}; + +enum class verbosity_t : int { + quiet, + standard, + verbose, +}; + +struct stdin_options_t { + ghc::filesystem::path so_out; + bool so_timestamp{false}; + auto_fd so_out_fd; +}; + int main(int argc, char* argv[]) { std::vector config_errors; std::vector loader_errors; exec_context& ec = lnav_data.ld_exec_context; - int lpc, c, retval = EXIT_SUCCESS; + int retval = EXIT_SUCCESS; std::shared_ptr stdin_reader; - ghc::filesystem::path stdin_out; - auto_fd stdin_out_fd; + stdin_options_t stdin_opts; bool exec_stdin = false, load_stdin = false, stdin_captured = false; + mode_flags_t mode_flags; const char* LANG = getenv("LANG"); ghc::filesystem::path stdin_tmp_path; + verbosity_t verbosity = verbosity_t::standard; if (LANG == nullptr || strcmp(LANG, "C") == 0) { setenv("LANG", "en_US.utf-8", 1); @@ -2221,156 +1864,169 @@ main(int argc, char* argv[]) curl_global_init(CURL_GLOBAL_DEFAULT); #endif - lnav_data.ld_debug_log_name = "/dev/null"; + static const std::string DEFAULT_DEBUG_LOG = "/dev/null"; + + lnav_data.ld_debug_log_name = DEFAULT_DEBUG_LOG; lnav_data.ld_config_paths.emplace_back("/etc/lnav"); lnav_data.ld_config_paths.emplace_back(SYSCONFDIR "/lnav"); lnav_data.ld_config_paths.emplace_back(lnav::paths::dotlnav()); - while ((c = getopt(argc, argv, "hHarRCc:I:iuf:d:nNqtw:vVW")) != -1) { - switch (c) { - case 'h': - usage(); - exit(retval); - break; - case 'H': - lnav_data.ld_flags |= LNF_HELP; - break; + std::vector file_args; + std::vector arg_errors; - case 'C': - lnav_data.ld_flags |= LNF_CHECK_CONFIG; - break; + CLI::App app{"The Logfile Navigator"}; - case 'c': - switch (optarg[0]) { - case ':': - case '/': - case ';': - break; - case '|': - if (strcmp("|-", optarg) == 0 - || strcmp("|/dev/stdin", optarg) == 0) { - exec_stdin = true; - } - break; - default: - lnav::console::print( - stderr, - lnav::console::user_message::error( - attr_line_t("invalid value for ") - .append_quoted("-c"_symbol) - .append(" option")) - .with_snippet(lnav::console::snippet::from( - "arg", - attr_line_t(" -c ") - .append(optarg) - .append("\n") - .append(4, ' ') - .append(lnav::roles::error( - "^ command type prefix " - "is missing")))) - .with_help(command_arg_help())); - exit(EXIT_FAILURE); - break; - } - lnav_data.ld_commands.emplace_back(optarg); - break; + app.add_option("-d", + lnav_data.ld_debug_log_name, + "Write debug messages to the given file.") + ->type_name("FILE"); + app.add_flag("-q{0},-v{2}", verbosity, "Control the verbosity"); + app.set_version_flag("-V,--version"); + app.footer(fmt::format(FMT_STRING("Version: {}"), VCS_PACKAGE_STRING)); - case 'f': - // XXX Not the best way to check for stdin. - if (strcmp("-", optarg) == 0 - || strcmp("/dev/stdin", optarg) == 0) { - exec_stdin = true; - } - lnav_data.ld_commands.emplace_back( - fmt::format(FMT_STRING("|{}"), optarg)); - break; + std::shared_ptr mmode_ops; - case 'I': - if (access(optarg, X_OK) != 0) { - lnav::console::print( - stderr, + if (argc < 2 || strcmp(argv[1], "-m") != 0) { + app.add_flag("-H", lnav_data.ld_show_help_view, "show help"); + app.add_option("-I", lnav_data.ld_config_paths, "include paths") + ->check(CLI::ExistingDirectory) + ->check([&arg_errors](std::string inc_path) -> std::string { + if (access(inc_path.c_str(), X_OK) != 0) { + arg_errors.emplace_back( lnav::console::user_message::error( attr_line_t("invalid configuration directory: ") - .append(lnav::roles::file(optarg))) + .append(lnav::roles::file(inc_path))) .with_errno_reason()); - exit(EXIT_FAILURE); + return "unreadable"; } - lnav_data.ld_config_paths.emplace_back(optarg); - break; - - case 'i': - lnav_data.ld_flags |= LNF_INSTALL; - break; - - case 'u': - lnav_data.ld_flags |= LNF_UPDATE_FORMATS; - break; - - case 'd': - lnav_data.ld_debug_log_name = optarg; - lnav_log_level = lnav_log_level_t::TRACE; - break; - - case 'a': - lnav_data.ld_flags |= LNF__ALL; - break; - - case 'n': - lnav_data.ld_flags |= LNF_HEADLESS; - break; - - case 'N': - lnav_data.ld_flags |= LNF_NO_DEFAULT; - break; - case 'q': - lnav_data.ld_flags |= LNF_QUIET; - break; - - case 'R': - lnav_data.ld_active_files.fc_rotated = true; - break; - - case 'r': - lnav_data.ld_active_files.fc_recursive = true; - break; - - case 't': - lnav_data.ld_flags |= LNF_TIMESTAMP; - break; - - case 'w': - stdin_out = optarg; - break; - - case 'W': { - char b; - if (isatty(STDIN_FILENO) && read(STDIN_FILENO, &b, 1) == -1) { - perror("Read key from STDIN"); - } - break; + return std::string(); + }) + ->allow_extra_args(false); + app.add_flag("-C", mode_flags.mf_check_configs, "check"); + auto* install_flag + = app.add_flag("-i", mode_flags.mf_install, "install"); + app.add_flag("-u", mode_flags.mf_update_formats, "update"); + auto* write_flag = app.add_option("-w", stdin_opts.so_out, "write"); + auto* ts_flag + = app.add_flag("-t", stdin_opts.so_timestamp, "timestamp"); + auto* no_default_flag + = app.add_flag("-N", mode_flags.mf_no_default, "no def"); + auto* rotated_flag = app.add_flag( + "-R", lnav_data.ld_active_files.fc_rotated, "rotated"); + auto* recurse_flag = app.add_flag( + "-r", lnav_data.ld_active_files.fc_recursive, "recurse"); + auto* headless_flag = app.add_flag( + "-n", + [](size_t count) { lnav_data.ld_flags |= LNF_HEADLESS; }, + "headless"); + auto* file_opt = app.add_option("file", file_args, "files"); + + auto wait_cb = [](size_t count) { + char b; + if (isatty(STDIN_FILENO) && read(STDIN_FILENO, &b, 1) == -1) { + perror("Read key from STDIN"); } + }; + app.add_flag("-W", wait_cb); - case 'v': - lnav_data.ld_flags |= LNF_VERBOSE; - break; + auto cmd_appender + = [](std::string cmd) { lnav_data.ld_commands.emplace_back(cmd); }; + auto cmd_validator = [&arg_errors](std::string cmd) -> std::string { + static const auto ARG_SRC = intern_string::lookup("arg"); - case 'V': - fmt::print("{}\n", VCS_PACKAGE_STRING); - exit(0); - break; + if (cmd.empty()) { + return "empty commands are not allowed"; + } - default: - retval = EXIT_FAILURE; - break; + switch (cmd[0]) { + case ':': + case '/': + case ';': + case '|': + break; + default: + arg_errors.emplace_back( + lnav::console::user_message::error( + attr_line_t("invalid value for ") + .append_quoted("-c"_symbol) + .append(" option")) + .with_snippet(lnav::console::snippet::from( + ARG_SRC, + attr_line_t(" -c ") + .append(cmd) + .append("\n") + .append(4, ' ') + .append(lnav::roles::error( + "^ command type prefix " + "is missing")))) + .with_help(command_arg_help())); + return "invalid prefix"; + } + return std::string(); + }; + auto* cmd_opt = app.add_option("-c") + ->check(cmd_validator) + ->each(cmd_appender) + ->allow_extra_args(false) + ->trigger_on_parse(true); + + auto file_appender = [](std::string file_path) { + lnav_data.ld_commands.emplace_back( + fmt::format(FMT_STRING("|{}"), file_path)); + }; + auto* exec_file_opt = app.add_option("-f") + ->trigger_on_parse(true) + ->allow_extra_args(false) + ->each(file_appender); + + install_flag->needs(file_opt); + install_flag->excludes(write_flag, + ts_flag, + no_default_flag, + rotated_flag, + recurse_flag, + headless_flag, + cmd_opt, + exec_file_opt); + } + + auto is_mmode = argc >= 2 && strcmp(argv[1], "-m") == 0; + try { + if (is_mmode) { + mmode_ops = lnav::management::describe_cli(app, argc, argv); + } else { + app.parse(argc, argv); + } + } catch (const CLI::CallForHelp& e) { + if (is_mmode) { + fmt::print("{}\n", app.help()); + } else { + usage(); + } + return EXIT_SUCCESS; + } catch (const CLI::CallForVersion& e) { + fmt::print("{}\n", VCS_PACKAGE_STRING); + return EXIT_SUCCESS; + } catch (const CLI::ParseError& e) { + if (!arg_errors.empty()) { + print_user_msgs(arg_errors); + return e.get_exit_code(); } + + lnav::console::print( + stderr, + lnav::console::user_message::error("invalid command-line arguments") + .with_reason(e.what())); + return e.get_exit_code(); } - argc -= optind; - argv += optind; + if (lnav_data.ld_debug_log_name != DEFAULT_DEBUG_LOG) { + lnav_log_level = lnav_log_level_t::TRACE; + } - lnav_log_file - = make_optional_from_nullable(fopen(lnav_data.ld_debug_log_name, "a")); + lnav_log_file = make_optional_from_nullable( + fopen(lnav_data.ld_debug_log_name.c_str(), "a")); log_info("lnav started"); { @@ -2386,19 +2042,19 @@ main(int argc, char* argv[]) load_config(lnav_data.ld_config_paths, config_errors); if (!config_errors.empty()) { - print_errors(config_errors); + print_user_msgs(config_errors); return EXIT_FAILURE; } add_global_vars(ec); - if (lnav_data.ld_flags & LNF_UPDATE_FORMATS) { + if (mode_flags.mf_update_formats) { if (!update_installs_from_git()) { return EXIT_FAILURE; } return EXIT_SUCCESS; } - if (lnav_data.ld_flags & LNF_INSTALL) { + if (mode_flags.mf_install) { auto formats_installed_path = lnav::paths::dotlnav() / "formats/installed"; auto configs_installed_path @@ -2428,32 +2084,32 @@ main(int argc, char* argv[]) return EXIT_FAILURE; } - for (lpc = 0; lpc < argc; lpc++) { - if (endswith(argv[lpc], ".git")) { - if (!install_from_git(argv[lpc])) { + for (auto& file_path : file_args) { + if (endswith(file_path, ".git")) { + if (!install_from_git(file_path)) { return EXIT_FAILURE; } continue; } - if (strcmp(argv[lpc], "extra") == 0) { + if (file_path == "extra") { install_extra_formats(); continue; } - auto file_type_result = detect_config_file_type(argv[lpc]); + auto file_type_result = detect_config_file_type(file_path); if (file_type_result.isErr()) { lnav::console::print( stderr, lnav::console::user_message::error( attr_line_t("unable to open configuration file: ") - .append(lnav::roles::file(argv[lpc]))) + .append(lnav::roles::file(file_path))) .with_reason(file_type_result.unwrapErr())); return EXIT_FAILURE; } auto file_type = file_type_result.unwrap(); - auto src_path = ghc::filesystem::path(argv[lpc]); + auto src_path = ghc::filesystem::path(file_path); ghc::filesystem::path dst_name; if (file_type == config_file_type::CONFIG) { dst_name = src_path.filename(); @@ -2461,7 +2117,7 @@ main(int argc, char* argv[]) auto format_list = load_format_file(src_path, loader_errors); if (!loader_errors.empty()) { - print_errors(loader_errors); + print_user_msgs(loader_errors); return EXIT_FAILURE; } if (format_list.empty()) { @@ -2483,7 +2139,7 @@ main(int argc, char* argv[]) / dst_name; auto_fd in_fd, out_fd; - if ((in_fd = open(argv[lpc], O_RDONLY)) == -1) { + if ((in_fd = open(file_path.c_str(), O_RDONLY)) == -1) { perror("unable to open file to install"); } else if ((out_fd = lnav::filesystem::openp( dst_path, O_WRONLY | O_CREAT | O_TRUNC, 0644)) @@ -2619,7 +2275,7 @@ main(int argc, char* argv[]) lnav_data.ld_user_message_view.set_sub_source( &lnav_data.ld_user_message_source); - for (lpc = 0; lpc < LNV__MAX; lpc++) { + for (int lpc = 0; lpc < LNV__MAX; lpc++) { lnav_data.ld_views[lpc].set_gutter_source(new log_gutter_source()); } @@ -2633,8 +2289,8 @@ main(int argc, char* argv[]) hs.set_time_slice(ZOOM_LEVELS[lnav_data.ld_zoom_level]); } - for (lpc = 0; lpc < LNV__MAX; lpc++) { - lnav_data.ld_views[lpc].set_title(view_titles[lpc]); + for (int lpc = 0; lpc < LNV__MAX; lpc++) { + lnav_data.ld_views[lpc].set_title(lnav_view_titles[lpc]); } load_formats(lnav_data.ld_config_paths, loader_errors); @@ -2716,43 +2372,41 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' }); if (!loader_errors.empty()) { - print_errors(loader_errors); - return EXIT_FAILURE; + print_user_msgs(loader_errors); + if (mmode_ops == nullptr) { + return EXIT_FAILURE; + } + } + + if (mmode_ops) { + auto perform_res = lnav::management::perform(mmode_ops); + + return print_user_msgs(perform_res); } - if (!(lnav_data.ld_flags & LNF_CHECK_CONFIG)) { - DEFAULT_FILES.insert( - make_pair(LNF_SYSLOG, std::string("var/log/messages"))); - DEFAULT_FILES.insert( - make_pair(LNF_SYSLOG, std::string("var/log/system.log"))); - DEFAULT_FILES.insert( - make_pair(LNF_SYSLOG, std::string("var/log/syslog"))); - DEFAULT_FILES.insert( - make_pair(LNF_SYSLOG, std::string("var/log/syslog.log"))); + if (!mode_flags.mf_check_configs) { + DEFAULT_FILES.emplace_back("var/log/messages"); + DEFAULT_FILES.emplace_back("var/log/system.log"); + DEFAULT_FILES.emplace_back("var/log/syslog"); + DEFAULT_FILES.emplace_back("var/log/syslog.log"); } init_lnav_commands(lnav_commands); lnav_data.ld_looping = true; - lnav_data.ld_mode = LNM_PAGING; + lnav_data.ld_mode = ln_mode_t::PAGING; - if ((isatty(STDIN_FILENO) || is_dev_null(STDIN_FILENO)) && argc == 0 - && !(lnav_data.ld_flags & LNF__ALL) - && !(lnav_data.ld_flags & LNF_NO_DEFAULT)) + if ((isatty(STDIN_FILENO) || is_dev_null(STDIN_FILENO)) && file_args.empty() + && !mode_flags.mf_no_default) { - lnav_data.ld_flags |= LNF_SYSLOG; - } - if (lnav_data.ld_flags != 0) { char start_dir[FILENAME_MAX]; if (getcwd(start_dir, sizeof(start_dir)) == nullptr) { perror("getcwd"); } else { do { - for (lpc = 0; lpc < LNB__MAX; lpc++) { - if (!append_default_files((lnav_flags_t) (1L << lpc))) { - retval = EXIT_FAILURE; - } + if (!append_default_files()) { + retval = EXIT_FAILURE; } } while (lnav_data.ld_active_files.fc_file_names.empty() && change_to_parent_dir()); @@ -2764,118 +2418,77 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' } { - const auto internals_dir = getenv("DUMP_INTERNALS_DIR"); - - if (internals_dir) { - dump_schema_to( - lnav_config_handlers, internals_dir, "config-v1.schema.json"); - dump_schema_to( - root_format_handler, internals_dir, "format-v1.schema.json"); - - execute_examples(); - - auto cmd_ref_path - = ghc::filesystem::path(internals_dir) / "cmd-ref.rst"; - auto cmd_file = std::unique_ptr( - fopen(cmd_ref_path.c_str(), "w+"), fclose); - - if (cmd_file.get()) { - std::set unique_cmds; - - for (auto& cmd : lnav_commands) { - if (unique_cmds.find(cmd.second) != unique_cmds.end()) { - continue; - } - unique_cmds.insert(cmd.second); - format_help_text_for_rst( - cmd.second->c_help, eval_example, cmd_file.get()); - } - } + const auto internals_dir_opt = getenv_opt("DUMP_INTERNALS_DIR"); - auto sql_ref_path - = ghc::filesystem::path(internals_dir) / "sql-ref.rst"; - auto sql_file = std::unique_ptr( - fopen(sql_ref_path.c_str(), "w+"), fclose); - std::set unique_sql_help; - - if (sql_file.get()) { - for (auto& sql : sqlite_function_help) { - if (unique_sql_help.find(sql.second) - != unique_sql_help.end()) { - continue; - } - unique_sql_help.insert(sql.second); - format_help_text_for_rst( - *sql.second, eval_example, sql_file.get()); - } - } + if (internals_dir_opt) { + lnav::dump_internals(internals_dir_opt.value()); return EXIT_SUCCESS; } } - if (argc == 0) { + if (file_args.empty()) { load_stdin = true; } - for (lpc = 0; lpc < argc; lpc++) { + for (auto& file_path : file_args) { auto_mem abspath; struct stat st; - if (strcmp(argv[lpc], "-") == 0) { + if (file_path == "-") { load_stdin = true; - } else if (startswith(argv[lpc], "pt:")) { + } else if (startswith(file_path, "pt:")) { #ifdef HAVE_LIBCURL - lnav_data.ld_pt_search = argv[lpc]; + lnav_data.ld_pt_search = file_path; #else fprintf(stderr, "error: lnav is not compiled with libcurl\n"); retval = EXIT_FAILURE; #endif } #ifdef HAVE_LIBCURL - else if (is_url(argv[lpc])) + else if (is_url(file_path)) { - auto ul = std::make_shared(argv[lpc]); + auto ul = std::make_shared(file_path); - lnav_data.ld_active_files.fc_file_names[argv[lpc]].with_fd( + lnav_data.ld_active_files.fc_file_names[file_path].with_fd( ul->copy_fd()); isc::to().send( [ul](auto& clooper) { clooper.add_request(ul); }); } #endif - else if (is_glob(argv[lpc])) + else if (is_glob(file_path)) { - lnav_data.ld_active_files.fc_file_names[argv[lpc]].with_tail( + lnav_data.ld_active_files.fc_file_names[file_path].with_tail( !(lnav_data.ld_flags & LNF_HEADLESS)); - } else if (stat(argv[lpc], &st) == -1) { - if (strchr(argv[lpc], ':') != nullptr) { - lnav_data.ld_active_files.fc_file_names[argv[lpc]].with_tail( + } else if (stat(file_path.c_str(), &st) == -1) { + if (file_path.find(':') != std::string::npos) { + lnav_data.ld_active_files.fc_file_names[file_path].with_tail( !(lnav_data.ld_flags & LNF_HEADLESS)); } else { lnav::console::print( stderr, lnav::console::user_message::error( attr_line_t("unable to open file: ") - .append(lnav::roles::file(argv[lpc]))) + .append(lnav::roles::file(file_path))) .with_errno_reason()); retval = EXIT_FAILURE; } - } else if (access(argv[lpc], R_OK) == -1) { + } else if (access(file_path.c_str(), R_OK) == -1) { lnav::console::print(stderr, lnav::console::user_message::error( attr_line_t("cannot read file: ") - .append(lnav::roles::file(argv[lpc]))) + .append(lnav::roles::file(file_path))) .with_errno_reason()); retval = EXIT_FAILURE; } else if (S_ISFIFO(st.st_mode)) { auto_fd fifo_fd; - if ((fifo_fd = open(argv[lpc], O_RDONLY)) == -1) { + if ((fifo_fd = open(file_path.c_str(), O_RDONLY)) == -1) { lnav::console::print( stderr, lnav::console::user_message::error( attr_line_t("cannot open fifo: ") - .append(lnav::roles::file(argv[lpc]))) + .append(lnav::roles::file(file_path))) .with_errno_reason()); retval = EXIT_FAILURE; } else { @@ -2899,7 +2512,8 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' std::move(fifo_out_fd)); lnav_data.ld_pipers.push_back(fifo_piper); } - } else if ((abspath = realpath(argv[lpc], nullptr)) == nullptr) { + } else if ((abspath = realpath(file_path.c_str(), nullptr)) == nullptr) + { perror("Cannot find file"); retval = EXIT_FAILURE; } else if (S_ISDIR(st.st_mode)) { @@ -2916,7 +2530,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' } } - if (lnav_data.ld_flags & LNF_CHECK_CONFIG) { + if (mode_flags.mf_check_configs) { rescan_files(true); for (auto& lf : lnav_data.ld_active_files.fc_files) { logfile::rebuild_result_t rebuild_result; @@ -2980,9 +2594,8 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' return retval; } - if (!(lnav_data.ld_flags & (LNF_HEADLESS | LNF_CHECK_CONFIG)) - && !isatty(STDOUT_FILENO)) - { + if (lnav_data.ld_flags & LNF_HEADLESS || mode_flags.mf_check_configs) { + } else if (!isatty(STDOUT_FILENO)) { lnav::console::print( stderr, lnav::console::user_message::error( @@ -2998,7 +2611,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' if (load_stdin && !isatty(STDIN_FILENO) && !is_dev_null(STDIN_FILENO) && !exec_stdin) { - if (stdin_out.empty()) { + if (stdin_opts.so_out.empty()) { auto pattern = lnav::paths::dotlnav() / "stdin-captures/stdin.XXXXXX"; @@ -3012,23 +2625,23 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' auto temp_pair = open_result.unwrap(); stdin_tmp_path = temp_pair.first; - stdin_out_fd = std::move(temp_pair.second); + stdin_opts.so_out_fd = std::move(temp_pair.second); } else { auto open_res = lnav::filesystem::open_file( - stdin_out, O_RDWR | O_CREAT | O_TRUNC, 0600); + stdin_opts.so_out, O_RDWR | O_CREAT | O_TRUNC, 0600); if (open_res.isErr()) { fmt::print(stderr, "error: {}\n", open_res.unwrapErr()); return EXIT_FAILURE; } - stdin_out_fd = open_res.unwrap(); + stdin_opts.so_out_fd = open_res.unwrap(); } stdin_captured = true; stdin_reader = std::make_shared(auto_fd(STDIN_FILENO), - lnav_data.ld_flags & LNF_TIMESTAMP, - std::move(stdin_out_fd)); + stdin_opts.so_timestamp, + std::move(stdin_opts.so_out_fd)); lnav_data.ld_active_files.fc_file_names["stdin"] .with_fd(stdin_reader->get_fd()) .with_include_in_session(false); @@ -3043,7 +2656,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' if (lnav_data.ld_active_files.fc_file_names.empty() && lnav_data.ld_commands.empty() && lnav_data.ld_pt_search.empty() - && !(lnav_data.ld_flags & (LNF_HELP | LNF_NO_DEFAULT))) + && !(lnav_data.ld_show_help_view || mode_flags.mf_no_default)) { lnav::console::print( stderr, @@ -3165,7 +2778,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' auto msg = pair.first.unwrap(); if (startswith(msg, "info:")) { - if (lnav_data.ld_flags & LNF_VERBOSE) { + if (verbosity == verbosity_t::verbose) { printf("%s\n", msg.c_str()); } } else if (!msg.empty()) { @@ -3175,12 +2788,11 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' } } - if (output_view && !(lnav_data.ld_flags & LNF_QUIET) + if (output_view && verbosity != verbosity_t::quiet && !lnav_data.ld_view_stack.empty() && !lnav_data.ld_stdout_used) { bool suppress_empty_lines = false; - list_overlay_source* los; unsigned long view_index; vis_line_t y; @@ -3195,24 +2807,18 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' break; } - los = tc->get_overlay_source(); + auto* los = tc->get_overlay_source(); vis_line_t vl; for (vl = tc->get_top(); vl < tc->get_inner_height(); ++vl, ++y) { attr_line_t al; - auto& line = al.get_string(); + while (los != nullptr && los->list_value_for_overlay( *tc, y, tc->get_inner_height(), vl, al)) { - if (write( - STDOUT_FILENO, line.c_str(), line.length()) - == -1 - || write(STDOUT_FILENO, "\n", 1) == -1) - { - perror("1 write to STDOUT"); - } + write_line_to(stdout, al); ++y; } @@ -3222,33 +2828,17 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' continue; } - struct line_range lr = find_string_attr_range( - rows[0].get_attrs(), &SA_ORIGINAL_LINE); - if (write(STDOUT_FILENO, - lr.substr(rows[0].get_string()), - lr.sublen(rows[0].get_string())) - == -1 - || write(STDOUT_FILENO, "\n", 1) == -1) - { - perror("2 write to STDOUT"); - } + write_line_to(stdout, rows[0]); } { attr_line_t al; - auto& line = al.get_string(); while (los != nullptr && los->list_value_for_overlay( *tc, y, tc->get_inner_height(), vl, al) && !al.empty()) { - if (write( - STDOUT_FILENO, line.c_str(), line.length()) - == -1 - || write(STDOUT_FILENO, "\n", 1) == -1) - { - perror("1 write to STDOUT"); - } + write_line_to(stdout, al); ++y; } } @@ -3267,20 +2857,24 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' save_session(); } + } catch (const std::system_error& e) { + if (e.code().value() != EPIPE) { + fprintf(stderr, "error: %s\n", e.what()); + } } catch (line_buffer::error& e) { fprintf(stderr, "error: %s\n", strerror(e.e_err)); } // When reading from stdin, tell the user where the capture file is // stored so they can look at it later. - if (stdin_captured && stdin_out.empty() - && !(lnav_data.ld_flags & LNF_QUIET) + if (stdin_captured && stdin_opts.so_out.empty() && !(lnav_data.ld_flags & LNF_HEADLESS)) { ghc::filesystem::permissions(stdin_tmp_path, ghc::filesystem::perms::owner_read); auto stdin_size = ghc::filesystem::file_size(stdin_tmp_path); - if (stdin_size > MAX_STDIN_CAPTURE_SIZE) { + if (verbosity == verbosity_t::quiet + || stdin_size > MAX_STDIN_CAPTURE_SIZE) { log_info("not saving large stdin capture -- %s", stdin_tmp_path.c_str()); ghc::filesystem::remove(stdin_tmp_path); diff --git a/src/lnav.hh b/src/lnav.hh index 2f24d758..d9317c9f 100644 --- a/src/lnav.hh +++ b/src/lnav.hh @@ -75,53 +75,17 @@ #include "top_status_source.hh" #include "view_helpers.hh" -/** The command modes that are available while viewing a file. */ -typedef enum { - LNM_PAGING, - LNM_FILTER, - LNM_FILES, - LNM_COMMAND, - LNM_SEARCH, - LNM_SEARCH_FILTERS, - LNM_SEARCH_FILES, - LNM_CAPTURE, - LNM_SQL, - LNM_EXEC, - LNM_USER, -} ln_mode_t; - enum { - LNB_SYSLOG, - LNB__MAX, - - LNB_TIMESTAMP, - LNB_HELP, LNB_HEADLESS, - LNB_QUIET, - LNB_CHECK_CONFIG, - LNB_INSTALL, - LNB_UPDATE_FORMATS, - LNB_VERBOSE, + LNB_MANAGEMENT, LNB_SECURE_MODE, - LNB_NO_DEFAULT, }; /** Flags set on the lnav command-line. */ typedef enum { - LNF_SYSLOG = (1L << LNB_SYSLOG), - - LNF_TIMESTAMP = (1L << LNB_TIMESTAMP), - LNF_HELP = (1L << LNB_HELP), LNF_HEADLESS = (1L << LNB_HEADLESS), - LNF_QUIET = (1L << LNB_QUIET), - LNF_CHECK_CONFIG = (1L << LNB_CHECK_CONFIG), - LNF_INSTALL = (1L << LNB_INSTALL), - LNF_UPDATE_FORMATS = (1L << LNB_UPDATE_FORMATS), - LNF_VERBOSE = (1L << LNB_VERBOSE), + LNF_MANAGEMENT = (1L << LNB_MANAGEMENT), LNF_SECURE_MODE = (1L << LNB_SECURE_MODE), - LNF_NO_DEFAULT = (1L << LNB_NO_DEFAULT), - - LNF__ALL = (LNF_SYSLOG | LNF_HELP), } lnav_flags_t; extern const char* lnav_zoom_strings[]; @@ -138,17 +102,17 @@ typedef enum { LNS__MAX } lnav_status_t; -typedef std::pair ppid_time_pair_t; -typedef std::pair session_pair_t; +using ppid_time_pair_t = std::pair; +using session_pair_t = std::pair; class input_state_tracker : public log_state_dumper { public: - input_state_tracker() : ist_index(0) + input_state_tracker() { memset(this->ist_recent_key_presses, 0, sizeof(this->ist_recent_key_presses)); - }; + } void log_state() override { @@ -159,19 +123,19 @@ public: this->ist_recent_key_presses[lpc]); } log_msg_extra_complete(); - }; + } void push_back(int ch) { this->ist_recent_key_presses[this->ist_index % COUNT] = ch; this->ist_index = (this->ist_index + 1) % COUNT; - }; + } private: static const int COUNT = 10; int ist_recent_key_presses[COUNT]; - size_t ist_index; + size_t ist_index{0}; }; struct key_repeat_history { @@ -209,7 +173,7 @@ struct lnav_data_t { time_t ld_session_time; time_t ld_session_load_time; const char* ld_program_name; - const char* ld_debug_log_name; + std::string ld_debug_log_name; std::list ld_commands; bool ld_cmd_init_done; @@ -228,7 +192,7 @@ struct lnav_data_t { unsigned long ld_flags; WINDOW* ld_window; ln_mode_t ld_mode; - ln_mode_t ld_last_config_mode{LNM_FILTER}; + ln_mode_t ld_last_config_mode{ln_mode_t::FILTER}; statusview_curses ld_status[LNS__MAX]; top_status_source ld_top_source; @@ -301,6 +265,9 @@ struct lnav_data_t { int ld_fifo_counter; struct key_repeat_history ld_key_repeat_history; + + bool ld_initial_build{false}; + bool ld_show_help_view{false}; }; struct static_service { @@ -322,18 +289,7 @@ extern const ssize_t ZOOM_COUNT; #define HELP_MSG_2(x, y, msg) "Press " ANSI_BOLD(#x) "/" ANSI_BOLD(#y) " " msg -void rebuild_hist(); -size_t rebuild_indexes(nonstd::optional deadline - = nonstd::nullopt); -void rebuild_indexes_repeatedly(); - bool setup_logline_table(exec_context& ec); - -bool rescan_files(bool required = false); -bool update_active_files(file_collection& new_files); - void wait_for_children(); -textview_curses* get_textview_for_mode(ln_mode_t mode); - #endif diff --git a/src/lnav.indexing.cc b/src/lnav.indexing.cc new file mode 100644 index 00000000..acd2ebe2 --- /dev/null +++ b/src/lnav.indexing.cc @@ -0,0 +1,385 @@ +/** + * Copyright (c) 2022, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "lnav.indexing.hh" + +#include "lnav.hh" +#include "service_tags.hh" +#include "session_data.hh" + +using namespace std::chrono_literals; + +/** + * Observer for loading progress that updates the bottom status bar. + */ +class loading_observer : public logfile_observer { +public: + loading_observer() : lo_last_offset(0){}; + + indexing_result logfile_indexing(const std::shared_ptr& lf, + file_off_t off, + file_size_t total) override + { + static sig_atomic_t index_counter = 0; + + if (lnav_data.ld_window == nullptr) { + return indexing_result::CONTINUE; + } + + /* XXX require(off <= total); */ + if (off > (off_t) total) { + off = total; + } + + if ((((size_t) off == total) && (this->lo_last_offset != off)) + || ui_periodic_timer::singleton().time_to_update(index_counter)) + { + lnav_data.ld_bottom_source.update_loading(off, total); + do_observer_update(lf); + this->lo_last_offset = off; + } + + if (!lnav_data.ld_looping) { + return indexing_result::BREAK; + } + return indexing_result::CONTINUE; + } + + off_t lo_last_offset; +}; + +void +do_observer_update(const std::shared_ptr& lf) +{ + if (isendwin()) { + return; + } + lnav_data.ld_top_source.update_time(); + for (auto& sc : lnav_data.ld_status) { + sc.do_update(); + } + if (lf && lnav_data.ld_mode == ln_mode_t::FILES + && !lnav_data.ld_initial_build) { + auto& fc = lnav_data.ld_active_files; + auto iter = std::find(fc.fc_files.begin(), fc.fc_files.end(), lf); + + if (iter != fc.fc_files.end()) { + auto index = std::distance(fc.fc_files.begin(), iter); + lnav_data.ld_files_view.set_selection( + vis_line_t(fc.fc_other_files.size() + index)); + lnav_data.ld_files_view.reload_data(); + lnav_data.ld_files_view.do_update(); + } + } + refresh(); +} + +void +rebuild_hist() +{ + logfile_sub_source& lss = lnav_data.ld_log_source; + hist_source2& hs = lnav_data.ld_hist_source2; + int zoom = lnav_data.ld_zoom_level; + + hs.set_time_slice(ZOOM_LEVELS[zoom]); + lss.reload_index_delegate(); +} + +class textfile_callback { +public: + void closed_files(const std::vector>& files) + { + for (const auto& lf : files) { + log_info("closed text files: %s", lf->get_filename().c_str()); + } + lnav_data.ld_active_files.close_files(files); + } + + void promote_file(const std::shared_ptr& lf) + { + if (lnav_data.ld_log_source.insert_file(lf)) { + this->did_promotion = true; + log_info("promoting text file to log file: %s (%s)", + lf->get_filename().c_str(), + lf->get_content_id().c_str()); + auto format = lf->get_format(); + if (format->lf_is_self_describing) { + auto vt = format->get_vtab_impl(); + + if (vt != nullptr) { + lnav_data.ld_vtab_manager->register_vtab(vt); + } + } + + auto iter = session_data.sd_file_states.find(lf->get_filename()); + if (iter != session_data.sd_file_states.end()) { + log_debug("found state for log file %d", + iter->second.fs_is_visible); + + lnav_data.ld_log_source.find_data(lf) | [&iter](auto ld) { + ld->set_visibility(iter->second.fs_is_visible); + }; + } + } else { + this->closed_files({lf}); + } + } + + void scanned_file(const std::shared_ptr& lf) + { + if (!lnav_data.ld_files_to_front.empty() + && lnav_data.ld_files_to_front.front().first == lf->get_filename()) + { + this->front_file = lf; + this->front_top = lnav_data.ld_files_to_front.front().second; + + lnav_data.ld_files_to_front.pop_front(); + } + } + + std::shared_ptr front_file; + int front_top{-1}; + bool did_promotion{false}; +}; + +size_t +rebuild_indexes(nonstd::optional deadline) +{ + logfile_sub_source& lss = lnav_data.ld_log_source; + textview_curses& log_view = lnav_data.ld_views[LNV_LOG]; + textview_curses& text_view = lnav_data.ld_views[LNV_TEXT]; + vis_line_t old_bottoms[LNV__MAX]; + bool scroll_downs[LNV__MAX]; + size_t retval = 0; + + for (int lpc = 0; lpc < LNV__MAX; lpc++) { + old_bottoms[lpc] = lnav_data.ld_views[lpc].get_top_for_last_row(); + scroll_downs[lpc] + = (lnav_data.ld_views[lpc].get_top() >= old_bottoms[lpc]) + && !(lnav_data.ld_flags & LNF_HEADLESS); + } + + { + textfile_sub_source* tss = &lnav_data.ld_text_source; + textfile_callback cb; + + if (tss->rescan_files(cb, deadline)) { + text_view.reload_data(); + retval += 1; + } + + if (cb.front_file != nullptr) { + ensure_view(&text_view); + + if (tss->current_file() != cb.front_file) { + tss->to_front(cb.front_file); + old_bottoms[LNV_TEXT] = -1_vl; + } + + if (cb.front_top < 0) { + cb.front_top += text_view.get_inner_height(); + } + if (cb.front_top < text_view.get_inner_height()) { + text_view.set_top(vis_line_t(cb.front_top)); + scroll_downs[LNV_TEXT] = false; + } + } + if (cb.did_promotion && deadline) { + // If there's a new log file, extend the deadline so it can be + // indexed quickly. + deadline = deadline.value() + 500ms; + } + } + + std::vector> closed_files; + for (auto& lf : lnav_data.ld_active_files.fc_files) { + if ((!lf->exists() || lf->is_closed())) { + log_info("closed log file: %s", lf->get_filename().c_str()); + lnav_data.ld_text_source.remove(lf); + lnav_data.ld_log_source.remove_file(lf); + closed_files.emplace_back(lf); + } + } + if (!closed_files.empty()) { + lnav_data.ld_active_files.close_files(closed_files); + } + + auto result = lss.rebuild_index(deadline); + if (result != logfile_sub_source::rebuild_result::rr_no_change) { + size_t new_count = lss.text_line_count(); + bool force + = result == logfile_sub_source::rebuild_result::rr_full_rebuild; + + if ((!scroll_downs[LNV_LOG] + || log_view.get_top() > vis_line_t(new_count)) + && force) + { + scroll_downs[LNV_LOG] = false; + } + + log_view.reload_data(); + + { + std::unordered_map>> + id_to_files; + bool reload = false; + + for (const auto& lf : lnav_data.ld_active_files.fc_files) { + id_to_files[lf->get_content_id()].push_back(lf); + } + + for (auto& pair : id_to_files) { + if (pair.second.size() == 1) { + continue; + } + + pair.second.sort([](const auto& left, const auto& right) { + return right->get_stat().st_size < left->get_stat().st_size; + }); + + auto dupe_name = pair.second.front()->get_unique_path(); + pair.second.pop_front(); + for_each(pair.second.begin(), + pair.second.end(), + [&dupe_name](auto& lf) { + log_info("Hiding duplicate file: %s", + lf->get_filename().c_str()); + lf->mark_as_duplicate(dupe_name); + lnav_data.ld_log_source.find_data(lf) | + [](auto ld) { ld->set_visibility(false); }; + }); + reload = true; + } + + if (reload) { + lss.text_filters_changed(); + } + } + + retval += 1; + } + + for (int lpc = 0; lpc < LNV__MAX; lpc++) { + textview_curses& scroll_view = lnav_data.ld_views[lpc]; + + if (scroll_downs[lpc] + && scroll_view.get_top_for_last_row() > scroll_view.get_top()) + { + scroll_view.set_top(scroll_view.get_top_for_last_row()); + } + } + + lnav_data.ld_view_stack.top() | [](auto tc) { + lnav_data.ld_filter_status_source.update_filtered(tc->get_sub_source()); + lnav_data.ld_scroll_broadcaster(tc); + }; + + return retval; +} + +void +rebuild_indexes_repeatedly() +{ + for (size_t attempt = 0; attempt < 10 && rebuild_indexes() > 0; attempt++) { + log_info("continuing to rebuild indexes..."); + } +} + +bool +update_active_files(file_collection& new_files) +{ + static loading_observer obs; + + if (lnav_data.ld_active_files.fc_invalidate_merge) { + lnav_data.ld_active_files.fc_invalidate_merge = false; + + return true; + } + + for (const auto& lf : new_files.fc_files) { + lf->set_logfile_observer(&obs); + lnav_data.ld_text_source.push_back(lf); + } + for (const auto& other_pair : new_files.fc_other_files) { + switch (other_pair.second.ofd_format) { + case file_format_t::SQLITE_DB: + attach_sqlite_db(lnav_data.ld_db.in(), other_pair.first); + break; + default: + break; + } + } + lnav_data.ld_active_files.merge(new_files); + if (!new_files.fc_files.empty() || !new_files.fc_other_files.empty() + || !new_files.fc_name_to_errors.empty()) + { + lnav_data.ld_active_files.regenerate_unique_file_names(); + } + lnav_data.ld_child_pollers.insert( + lnav_data.ld_child_pollers.begin(), + std::make_move_iterator( + lnav_data.ld_active_files.fc_child_pollers.begin()), + std::make_move_iterator( + lnav_data.ld_active_files.fc_child_pollers.end())); + + return true; +} + +bool +rescan_files(bool req) +{ + auto& mlooper = injector::get(); + bool done = false; + auto delay = 0ms; + + do { + auto fc = lnav_data.ld_active_files.rescan_files(req); + bool all_synced = true; + + update_active_files(fc); + mlooper.get_port().process_for(delay); + if (lnav_data.ld_flags & LNF_HEADLESS) { + for (const auto& pair : lnav_data.ld_active_files.fc_other_files) { + if (pair.second.ofd_format != file_format_t::REMOTE) { + continue; + } + + if (lnav_data.ld_active_files.fc_synced_files.count(pair.first) + == 0) { + all_synced = false; + } + } + if (!all_synced) { + delay = 30ms; + } + } + done = fc.fc_file_names.empty() && all_synced; + } while (!done); + return true; +} diff --git a/src/lnav.indexing.hh b/src/lnav.indexing.hh new file mode 100644 index 00000000..a37745b6 --- /dev/null +++ b/src/lnav.indexing.hh @@ -0,0 +1,45 @@ +/** + * Copyright (c) 2022, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef lnav_indexing_hh +#define lnav_indexing_hh + +#include "file_collection.hh" +#include "logfile_fwd.hh" +#include "optional.hpp" + +void rebuild_hist(); +size_t rebuild_indexes(nonstd::optional deadline + = nonstd::nullopt); +void rebuild_indexes_repeatedly(); +bool rescan_files(bool required = false); +bool update_active_files(file_collection& new_files); +void do_observer_update(const std::shared_ptr& lf); + +#endif diff --git a/src/lnav.management_cli.cc b/src/lnav.management_cli.cc new file mode 100644 index 00000000..020bc939 --- /dev/null +++ b/src/lnav.management_cli.cc @@ -0,0 +1,970 @@ +/** + * Copyright (c) 2022, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "lnav.management_cli.hh" + +#include "base/itertools.hh" +#include "base/result.h" +#include "base/string_util.hh" +#include "fmt/format.h" +#include "fts_fuzzy_match.hh" +#include "log_format.hh" +#include "log_format_ext.hh" +#include "mapbox/variant.hpp" +#include "regex101.import.hh" +#include "session_data.hh" + +using namespace lnav::roles::literals; + +namespace lnav { +namespace itertools { + +namespace details { + +struct similar_to { + std::string st_pattern; + size_t st_count; +}; + +} // namespace details + +details::similar_to +similar_to(std::string pattern, size_t count = 5) +{ + return lnav::itertools::details::similar_to{std::move(pattern), count}; +} + +} // namespace itertools +} // namespace lnav + +template +std::vector +operator|(const T& in, const lnav::itertools::details::similar_to& st) +{ + using score_pair = std::pair; + + struct score_cmp { + bool operator()(const score_pair& lhs, const score_pair& rhs) + { + return lhs.first > rhs.first; + } + }; + + std::priority_queue, score_cmp> pq; + + for (const auto& elem : in) { + int score = 0; + + if (!fts::fuzzy_match(st.st_pattern.c_str(), elem.c_str(), score)) { + continue; + } + if (score <= 0) { + continue; + } + pq.push(std::make_pair(score, elem)); + + if (pq.size() > st.st_count) { + pq.pop(); + } + } + + std::vector> retval; + + while (!pq.empty()) { + retval.template emplace_back(pq.top().second); + pq.pop(); + } + std::reverse(retval.begin(), retval.end()); + + return retval; +} + +namespace lnav { + +namespace management { + +struct no_subcmd_t { + CLI::App* ns_root_app{nullptr}; +}; + +inline attr_line_t& +symbol_reducer(const std::string& elem, attr_line_t& accum) +{ + return accum.append("\n ").append(lnav::roles::symbol(elem)); +} + +inline attr_line_t& +subcmd_reducer(const CLI::App* app, attr_line_t& accum) +{ + return accum.append("\n \u2022 ") + .append(lnav::roles::keyword(app->get_name())) + .append(": ") + .append(app->get_description()); +} + +struct subcmd_format_t { + using action_t = std::function; + + CLI::App* sf_format_app{nullptr}; + std::string sf_name; + CLI::App* sf_regex_app{nullptr}; + std::string sf_regex_name; + CLI::App* sf_regex101_app{nullptr}; + action_t sf_action; + + subcmd_format_t& set_action(action_t act) + { + if (!this->sf_action) { + this->sf_action = std::move(act); + } + return *this; + } + + Result, console::user_message> validate_format() + const + { + if (this->sf_name.empty()) { + auto um = console::user_message::error( + "expecting a format name to operate on"); + um.with_note( + (log_format::get_root_formats() + | lnav::itertools::map(&log_format::get_name) + | lnav::itertools::sort_with(intern_string_t::case_lt) + | lnav::itertools::map(&intern_string_t::to_string) + | lnav::itertools::fold(symbol_reducer, attr_line_t{})) + .add_header("the available formats are:")); + + return Err(um); + } + + auto lformat = log_format::find_root_format(this->sf_name.c_str()); + if (lformat == nullptr) { + auto um = console::user_message::error( + attr_line_t("unknown format: ") + .append(lnav::roles::symbol(this->sf_name))); + um.with_note( + (log_format::get_root_formats() + | lnav::itertools::map(&log_format::get_name) + | lnav::itertools::similar_to(this->sf_name) + | lnav::itertools::map(&intern_string_t::to_string) + | lnav::itertools::fold(symbol_reducer, attr_line_t{})) + .add_header("did you mean one of the following?")); + + return Err(um); + } + + return Ok(lformat); + } + + Result + validate_external_format() const + { + auto lformat = TRY(this->validate_format()); + auto* ext_lformat = dynamic_cast(lformat.get()); + + if (ext_lformat == nullptr) { + return Err(console::user_message::error( + attr_line_t() + .append_quoted(lnav::roles::symbol(this->sf_name)) + .append(" is an internal format that is not defined in a " + "configuration file"))); + } + + return Ok(ext_lformat); + } + + Result>, + console::user_message> + validate_regex() const + { + auto* ext_lformat = TRY(this->validate_external_format()); + + if (this->sf_regex_name.empty()) { + auto um = console::user_message::error( + "expecting a regex name to operate on"); + um.with_note( + ext_lformat->elf_pattern_order + | lnav::itertools::map(&external_log_format::pattern::p_name) + | lnav::itertools::fold( + symbol_reducer, attr_line_t{"the available regexes are:"})); + + return Err(um); + } + + for (const auto& pat : ext_lformat->elf_pattern_order) { + if (pat->p_name == this->sf_regex_name) { + return Ok(std::make_pair(ext_lformat, pat)); + } + } + + auto um = console::user_message::error( + attr_line_t("unknown regex: ") + .append(lnav::roles::symbol(this->sf_regex_name))); + um.with_note( + (ext_lformat->elf_pattern_order + | lnav::itertools::map(&external_log_format::pattern::p_name) + | lnav::itertools::similar_to(this->sf_regex_name) + | lnav::itertools::fold(symbol_reducer, attr_line_t{})) + .add_header("did you mean one of the following?")); + + return Err(um); + } + + static perform_result_t default_action(const subcmd_format_t& sf) + { + auto validate_res = sf.validate_format(); + if (validate_res.isErr()) { + return {validate_res.unwrapErr()}; + } + + auto lformat = validate_res.unwrap(); + auto* ext_format = dynamic_cast(lformat.get()); + + attr_line_t ext_details; + if (ext_format != nullptr) { + ext_details.append("\n ") + .append("Regexes"_h3) + .append(": ") + .join(ext_format->elf_pattern_order + | lnav::itertools::map( + &external_log_format::pattern::p_name), + VC_ROLE.value(role_t::VCR_SYMBOL), + ", "); + } + + auto um = console::user_message::error( + attr_line_t("expecting an operation to perform on the ") + .append(lnav::roles::symbol(sf.sf_name)) + .append(" format")); + um.with_note(attr_line_t() + .append(lnav::roles::symbol(sf.sf_name)) + .append(": ") + .append(lformat->lf_description) + .append(ext_details)); + um.with_help( + sf.sf_format_app->get_subcommands({}) + | lnav::itertools::fold( + subcmd_reducer, attr_line_t{"the available operations are:"})); + + return {um}; + } + + static perform_result_t default_regex_action(const subcmd_format_t& sf) + { + auto validate_res = sf.validate_regex(); + + if (validate_res.isErr()) { + return {validate_res.unwrapErr()}; + } + + auto um = console::user_message::error( + attr_line_t("expecting an operation to perform on the ") + .append(lnav::roles::symbol(sf.sf_regex_name)) + .append(" regular expression using regex101.com")); + + um.with_help(attr_line_t{"the available subcommands are:"}.append( + sf.sf_regex101_app->get_subcommands({}) + | lnav::itertools::fold(subcmd_reducer, attr_line_t{}))); + + return {um}; + } + + static perform_result_t get_action(const subcmd_format_t& sf) + { + auto validate_res = sf.validate_format(); + + if (validate_res.isErr()) { + return {validate_res.unwrapErr()}; + } + + auto format = validate_res.unwrap(); + + auto um = console::user_message::raw( + attr_line_t() + .append(lnav::roles::symbol(sf.sf_name)) + .append(": ") + .append(on_blank(format->lf_description, ""))); + + return {um}; + } + + static perform_result_t source_action(const subcmd_format_t& sf) + { + auto validate_res = sf.validate_external_format(); + + if (validate_res.isErr()) { + return {validate_res.unwrapErr()}; + } + + auto* format = validate_res.unwrap(); + + if (format->elf_format_source_order.empty()) { + return { + console::user_message::error( + "format is builtin, there is no source file"), + }; + } + + auto um = console::user_message::raw( + format->elf_format_source_order[0].string()); + + return {um}; + } + + static perform_result_t sources_action(const subcmd_format_t& sf) + { + auto validate_res = sf.validate_external_format(); + + if (validate_res.isErr()) { + return {validate_res.unwrapErr()}; + } + + auto* format = validate_res.unwrap(); + + if (format->elf_format_source_order.empty()) { + return { + console::user_message::error( + "format is builtin, there is no source file"), + }; + } + + auto um = console::user_message::raw( + attr_line_t().join(format->elf_format_source_order, + VC_ROLE.value(role_t::VCR_TEXT), + "\n")); + + return {um}; + } + + static perform_result_t regex101_pull_action(const subcmd_format_t& sf) + { + auto validate_res = sf.validate_regex(); + if (validate_res.isErr()) { + return {validate_res.unwrapErr()}; + } + + auto format_regex_pair = validate_res.unwrap(); + auto get_meta_res + = lnav::session::regex101::get_entry(sf.sf_name, sf.sf_regex_name); + + return get_meta_res.match( + [&sf]( + const lnav::session::regex101::error& err) -> perform_result_t { + return { + console::user_message::error( + attr_line_t("unable to get DB entry for: ") + .append(lnav::roles::symbol(sf.sf_name)) + .append("/") + .append(lnav::roles::symbol(sf.sf_regex_name))) + .with_reason(err.e_msg), + }; + }, + [&sf]( + const lnav::session::regex101::no_entry&) -> perform_result_t { + return { + console::user_message::error( + attr_line_t("regex ") + .append_quoted( + lnav::roles::symbol(sf.sf_regex_name)) + .append(" of format ") + .append_quoted(lnav::roles::symbol(sf.sf_name)) + .append(" has not been pushed to regex101.com")) + .with_help( + attr_line_t("use the ") + .append_quoted("push"_keyword) + .append(" subcommand to create the regex on " + "regex101.com for easy editing")), + }; + }, + [&](const lnav::session::regex101::entry& en) -> perform_result_t { + auto retrieve_res = regex101::client::retrieve(en.re_permalink); + + return retrieve_res.match( + [&](const console::user_message& um) -> perform_result_t { + return { + console::user_message::error( + attr_line_t("unable to retrieve entry ") + .append_quoted( + lnav::roles::symbol(en.re_permalink)) + .append(" from regex101.com")) + .with_reason(um), + }; + }, + [&](const regex101::client::no_entry&) -> perform_result_t { + lnav::session::regex101::delete_entry(sf.sf_name, + sf.sf_regex_name); + return { + console::user_message::error( + attr_line_t("entry ") + .append_quoted( + lnav::roles::symbol(en.re_permalink)) + .append( + " no longer exists on regex101.com")) + .with_help(attr_line_t("use the ") + .append_quoted("delete"_keyword) + .append(" subcommand to delete " + "the association")), + }; + }, + [&](const regex101::client::entry& remote_entry) + -> perform_result_t { + auto curr_entry = regex101::convert_format_pattern( + format_regex_pair.first, format_regex_pair.second); + + if (curr_entry.e_regex == remote_entry.e_regex) { + return { + console::user_message::ok( + attr_line_t("local regex is in sync " + "with entry ") + .append_quoted(lnav::roles::symbol( + en.re_permalink)) + .append(" on regex101.com")) + .with_help( + attr_line_t("make edits on ") + .append_quoted(lnav::roles::file( + regex101::client::to_edit_url( + en.re_permalink))) + .append(" and then run this " + "command again to update " + "the local values")), + }; + } + + auto patch_res + = regex101::patch(format_regex_pair.first, + sf.sf_regex_name, + remote_entry); + + if (patch_res.isErr()) { + return { + console::user_message::error( + attr_line_t( + "unable to patch format regex: ") + .append(lnav::roles::symbol(sf.sf_name)) + .append("/") + .append(lnav::roles::symbol( + sf.sf_regex_name))) + .with_reason(patch_res.unwrapErr()), + }; + } + + auto um = console::user_message::ok( + attr_line_t("format patch file written to: ") + .append(lnav::roles::file( + patch_res.unwrap().string()))); + if (!format_regex_pair.first->elf_builtin_format) { + um.with_help( + attr_line_t("once the regex has been found " + "to be working correctly, move the " + "contents of the patch file to the " + "original file at:\n ") + .append(lnav::roles::file( + format_regex_pair.first + ->elf_format_source_order.front() + .string()))); + } + + return {um}; + }); + }); + } + + static perform_result_t regex101_default_action(const subcmd_format_t& sf) + { + auto validate_res = sf.validate_regex(); + + if (validate_res.isErr()) { + return {validate_res.unwrapErr()}; + } + + auto um = console::user_message::error( + attr_line_t("expecting an operation to perform on the ") + .append(lnav::roles::symbol(sf.sf_regex_name)) + .append(" regex")); + + auto get_res + = lnav::session::regex101::get_entry(sf.sf_name, sf.sf_regex_name); + if (get_res.is()) { + auto local_entry = get_res.get(); + um.with_note( + attr_line_t("this regex is currently associated with the " + "following regex101.com entry:\n ") + .append(lnav::roles::file(regex101::client::to_edit_url( + local_entry.re_permalink)))); + } + + um.with_help(attr_line_t{"the available subcommands are:"}.append( + sf.sf_regex_app->get_subcommands({}) + | lnav::itertools::fold(subcmd_reducer, attr_line_t{}))); + + return {um}; + } + + static perform_result_t regex101_push_action(const subcmd_format_t& sf) + { + auto validate_res = sf.validate_regex(); + if (validate_res.isErr()) { + return {validate_res.unwrapErr()}; + } + + auto format_regex_pair = validate_res.unwrap(); + auto entry = regex101::convert_format_pattern(format_regex_pair.first, + format_regex_pair.second); + auto get_meta_res + = lnav::session::regex101::get_entry(sf.sf_name, sf.sf_regex_name); + + if (get_meta_res.is()) { + auto entry_meta + = get_meta_res.get(); + auto retrieve_res + = regex101::client::retrieve(entry_meta.re_permalink); + + if (retrieve_res.is()) { + auto remote_entry = retrieve_res.get(); + + if (remote_entry == entry) { + return { + console::user_message::ok( + attr_line_t("regex101 entry ") + .append(lnav::roles::symbol( + entry_meta.re_permalink)) + .append(" is already up-to-date")), + }; + } + } else if (retrieve_res.is()) { + return { + retrieve_res.get(), + }; + } + + entry.e_permalink_fragment = entry_meta.re_permalink; + } + + auto upsert_res = regex101::client::upsert(entry); + auto upsert_info = upsert_res.unwrap(); + + if (get_meta_res.is()) { + lnav::session::regex101::insert_entry({ + format_regex_pair.first->get_name().to_string(), + format_regex_pair.second->p_name, + upsert_info.cr_permalink_fragment, + upsert_info.cr_delete_code, + }); + } + + return { + console::user_message::ok( + attr_line_t("pushed regex to -- ") + .append(lnav::roles::file(regex101::client::to_edit_url( + upsert_info.cr_permalink_fragment)))) + .with_help(attr_line_t("use the ") + .append_quoted("pull"_keyword) + .append(" subcommand to update the format after " + "you make changes on regex101.com")), + }; + } + + static perform_result_t regex101_delete_action(const subcmd_format_t& sf) + { + auto get_res + = lnav::session::regex101::get_entry(sf.sf_name, sf.sf_regex_name); + + return get_res.match( + [&sf]( + const lnav::session::regex101::entry& en) -> perform_result_t { + { + auto validate_res = sf.validate_external_format(); + + if (validate_res.isOk()) { + auto ppath = regex101::patch_path(validate_res.unwrap(), + en.re_permalink); + + if (ghc::filesystem::exists(ppath)) { + return { + console::user_message::error( + attr_line_t("cannot delete regex101 entry " + "while patch file exists")) + .with_note(attr_line_t(" ").append( + lnav::roles::file(ppath.string()))) + .with_help(attr_line_t( + "move the contents of the patch file " + "to the main log format and then " + "delete the file to continue")), + }; + } + } + } + + perform_result_t retval; + if (en.re_delete_code.empty()) { + retval.emplace_back( + console::user_message::warning( + attr_line_t("not deleting regex101 entry ") + .append_quoted( + lnav::roles::symbol(en.re_permalink))) + .with_reason( + "delete code is not known for this entry") + .with_note( + "formats created by importing a regex101.com " + "entry will not have a delete code")); + } else { + auto delete_res + = regex101::client::delete_entry(en.re_delete_code); + + if (delete_res.isErr()) { + return { + console::user_message::error( + "unable to delete regex101 entry") + .with_reason(delete_res.unwrapErr()), + }; + } + } + + lnav::session::regex101::delete_entry(sf.sf_name, + sf.sf_regex_name); + + retval.emplace_back(console::user_message::ok( + attr_line_t("deleted regex101 entry: ") + .append(lnav::roles::symbol(en.re_permalink)))); + + return retval; + }, + [&sf]( + const lnav::session::regex101::no_entry&) -> perform_result_t { + return { + console::user_message::error( + attr_line_t("no regex101 entry for ") + .append(lnav::roles::symbol(sf.sf_name)) + .append("/") + .append(lnav::roles::symbol(sf.sf_regex_name))), + }; + }, + [&sf]( + const lnav::session::regex101::error& err) -> perform_result_t { + return { + console::user_message::error( + attr_line_t("unable to get regex101 entry for ") + .append(lnav::roles::symbol(sf.sf_name)) + .append("/") + .append(lnav::roles::symbol(sf.sf_regex_name))) + .with_reason(err.e_msg), + }; + }); + } +}; + +struct subcmd_regex101_t { + using action_t = std::function; + + CLI::App* sr_app{nullptr}; + action_t sr_action; + std::string sr_import_url; + std::string sr_import_name; + std::string sr_import_regex_name{"std"}; + + subcmd_regex101_t& set_action(action_t act) + { + if (!this->sr_action) { + this->sr_action = std::move(act); + } + return *this; + } + + static perform_result_t default_action(const subcmd_regex101_t& sr) + { + auto um = console::user_message::error( + "expecting an operation related to the regex101.com integration"); + um.with_help( + sr.sr_app->get_subcommands({}) + | lnav::itertools::fold( + subcmd_reducer, attr_line_t{"the available operations are:"})); + + return {um}; + } + + static perform_result_t list_action(const subcmd_regex101_t&) + { + auto get_res = lnav::session::regex101::get_entries(); + + if (get_res.isErr()) { + return { + console::user_message::error( + "unable to read regex101 entries from DB") + .with_reason(get_res.unwrapErr()), + }; + } + + auto entries + = get_res.unwrap() | lnav::itertools::map([](const auto& elem) { + return fmt::format( + FMT_STRING(" format {} regex {} regex101\n"), + elem.re_format_name, + elem.re_regex_name); + }) + | lnav::itertools::fold( + [](const auto& elem, auto& accum) { + return accum.append(elem); + }, + attr_line_t{}); + + auto um = console::user_message::ok( + entries.add_header("the following regex101 entries were found:\n") + .with_default("no regex101 entries found")); + + return {um}; + } + + static perform_result_t import_action(const subcmd_regex101_t& sr) + { + auto import_res = regex101::import( + sr.sr_import_url, sr.sr_import_name, sr.sr_import_regex_name); + + if (import_res.isOk()) { + return { + lnav::console::user_message::ok( + attr_line_t("converted regex101 entry to format file: ") + .append(lnav::roles::file(import_res.unwrap()))) + .with_note("the converted format may still have errors") + .with_help( + attr_line_t( + "use the following command to patch the regex as " + "more changes are made on regex101.com:\n") + .append(FMT_STRING(" lnav -m format {} regex {} " + "regex101 pull"), + sr.sr_import_name, + sr.sr_import_regex_name)), + }; + } + + return { + import_res.unwrapErr(), + }; + } +}; + +using operations_v + = mapbox::util::variant; + +class operations { +public: + operations_v o_ops; +}; + +std::shared_ptr +describe_cli(CLI::App& app, int argc, char* argv[]) +{ + auto retval = std::make_shared(); + + retval->o_ops = no_subcmd_t{ + &app, + }; + + app.add_flag("-m", "Switch to the management CLI mode."); + + subcmd_format_t format_args; + subcmd_regex101_t regex101_args; + + { + auto* subcmd_format + = app.add_subcommand("format", + "perform operations on log file formats") + ->callback([&]() { + format_args.set_action(subcmd_format_t::default_action); + retval->o_ops = format_args; + }); + format_args.sf_format_app = subcmd_format; + subcmd_format + ->add_option( + "format_name", format_args.sf_name, "the name of the format") + ->expected(1); + + { + subcmd_format + ->add_subcommand("get", "print information about a format") + ->callback([&]() { + format_args.set_action(subcmd_format_t::get_action); + }); + } + + { + subcmd_format + ->add_subcommand("source", + "print the path of the first source file " + "containing this format") + ->callback([&]() { + format_args.set_action(subcmd_format_t::source_action); + }); + } + + { + subcmd_format + ->add_subcommand("sources", + "print the paths of all source files " + "containing this format") + ->callback([&]() { + format_args.set_action(subcmd_format_t::sources_action); + }); + } + + { + auto* subcmd_format_regex + = subcmd_format + ->add_subcommand( + "regex", + "operate on the format's regular expressions") + ->callback([&]() { + format_args.set_action( + subcmd_format_t::default_regex_action); + }); + format_args.sf_regex_app = subcmd_format_regex; + subcmd_format_regex->add_option( + "regex-name", + format_args.sf_regex_name, + "the name of the regular expression to operate on"); + + { + auto* subcmd_format_regex_regex101 + = subcmd_format_regex + ->add_subcommand("regex101", + "use regex101.com to edit this " + "regular expression") + ->callback([&]() { + format_args.set_action( + subcmd_format_t::regex101_default_action); + }); + format_args.sf_regex101_app = subcmd_format_regex_regex101; + + { + subcmd_format_regex_regex101 + ->add_subcommand("push", + "create/update an entry for " + "this regex on regex101.com") + ->callback([&]() { + format_args.set_action( + subcmd_format_t::regex101_push_action); + }); + subcmd_format_regex_regex101 + ->add_subcommand( + "pull", + "create a patch format file for this " + "regular expression based on the entry in " + "regex101.com") + ->callback([&]() { + format_args.set_action( + subcmd_format_t::regex101_pull_action); + }); + subcmd_format_regex_regex101 + ->add_subcommand( + "delete", + "delete the entry regex101.com that was " + "created by a push operation") + ->callback([&]() { + format_args.set_action( + subcmd_format_t::regex101_delete_action); + }); + } + } + } + } + + { + auto* subcmd_regex101 + = app.add_subcommand("regex101", + "create and edit log message regular " + "expressions using regex101.com") + ->callback([&]() { + regex101_args.set_action( + subcmd_regex101_t::default_action); + retval->o_ops = regex101_args; + }); + regex101_args.sr_app = subcmd_regex101; + + { + subcmd_regex101 + ->add_subcommand("list", + "list the log format regular expression " + "linked to entries on regex101.com") + ->callback([&]() { + regex101_args.set_action(subcmd_regex101_t::list_action); + }); + } + { + auto* subcmd_regex101_import + = subcmd_regex101 + ->add_subcommand("import", + "create a new format from a regular " + "expression on regex101.com") + ->callback([&]() { + regex101_args.set_action( + subcmd_regex101_t::import_action); + }); + + subcmd_regex101_import->add_option( + "url", + regex101_args.sr_import_url, + "The regex101.com url to construct a log format from"); + subcmd_regex101_import->add_option("name", + regex101_args.sr_import_name, + "The name for the log format"); + subcmd_regex101_import + ->add_option("regex-name", + regex101_args.sr_import_regex_name, + "The name for the new regex") + ->always_capture_default(); + } + } + + app.parse(argc, argv); + + return retval; +} + +perform_result_t +perform(std::shared_ptr opts) +{ + return opts->o_ops.match( + [](const no_subcmd_t& ns) -> perform_result_t { + auto um = console::user_message::error( + attr_line_t("expecting an operation to perform")); + um.with_help(ns.ns_root_app->get_subcommands({}) + | lnav::itertools::fold( + subcmd_reducer, + attr_line_t{"the available operations are:"})); + + return {um}; + }, + [](const subcmd_format_t& sf) { return sf.sf_action(sf); }, + [](const subcmd_regex101_t& sr) { return sr.sr_action(sr); }); +} + +} // namespace management +} // namespace lnav diff --git a/src/lnav.management_cli.hh b/src/lnav.management_cli.hh new file mode 100644 index 00000000..41c26150 --- /dev/null +++ b/src/lnav.management_cli.hh @@ -0,0 +1,53 @@ +/** + * Copyright (c) 2022, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef lnav_management_cli_hh +#define lnav_management_cli_hh + +#include +#include + +#include "base/lnav.console.hh" +#include "CLI/CLI.hpp" + +namespace lnav { +namespace management { + +class operations; + +std::shared_ptr describe_cli(CLI::App& app, int argc, char* argv[]); + +using perform_result_t = std::vector; + +perform_result_t perform(std::shared_ptr opts); + +} // namespace management +} // namespace lnav + +#endif diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index 93e82e51..9472d11c 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -55,6 +55,7 @@ #include "db_sub_source.hh" #include "field_overlay_source.hh" #include "fmt/printf.h" +#include "lnav.indexing.hh" #include "lnav_commands.hh" #include "lnav_config.hh" #include "lnav_util.hh" @@ -613,11 +614,12 @@ com_goto_mark(exec_context& ec, if (args.size() > 1) { for (size_t lpc = 1; lpc < args.size(); lpc++) { - auto bt = bookmark_type_t::find_type(args[lpc]); - if (bt == nullptr) { - return ec.make_error("unknown bookmark type"); + auto bt_opt = bookmark_type_t::find_type(args[lpc]); + if (!bt_opt) { + return ec.make_error("unknown bookmark type: {}", + args[lpc]); } - mark_types.insert(bt); + mark_types.insert(bt_opt.value()); } } else { mark_types = DEFAULT_TYPES; @@ -826,34 +828,6 @@ json_write_row(yajl_gen handle, int row) } } -static void -write_line_to(FILE* outfile, const attr_line_t& al) -{ - const auto& al_attrs = al.get_attrs(); - auto lr = find_string_attr_range(al_attrs, &SA_ORIGINAL_LINE); - const auto line_meta_opt = get_string_attr(al_attrs, logline::L_META); - - if (lr.lr_start > 1) { - // If the line is prefixed with some extra information, include that - // in the output. For example, the log file name or time offset. - lr = line_range{0, -1}; - } - - fwrite(lr.substr(al.get_string()), 1, lr.sublen(al.get_string()), outfile); - fwrite("\n", 1, 1, outfile); - - if (line_meta_opt) { - auto bm = line_meta_opt.value().get(); - - if (!bm->bm_comment.empty()) { - fprintf(outfile, " // %s\n", bm->bm_comment.c_str()); - } - if (!bm->bm_tags.empty()) { - fmt::print(outfile, " -- {}\n", fmt::join(bm->bm_tags, " ")); - } - } -} - static Result com_save_to(exec_context& ec, std::string cmdline, @@ -1129,13 +1103,33 @@ com_save_to(exec_context& ec, vis_line_t top = tc->get_top(); vis_line_t bottom = tc->get_bottom(); + auto y = 0_vl; std::vector rows(bottom - top + 1); + attr_line_t ov_al; + auto* los = tc->get_overlay_source(); tc->listview_value_for_rows(*tc, top, rows); - for (auto& al : rows) { + for (const auto& al : rows) { + while (los != nullptr + && los->list_value_for_overlay( + *tc, y, tc->get_inner_height(), top, ov_al)) + { + write_line_to(outfile, ov_al); + ++y; + } write_line_to(outfile, al); line_count += 1; + ++top; + ++y; + } + while (los != nullptr + && los->list_value_for_overlay( + *tc, y, tc->get_inner_height(), top, ov_al) + && !ov_al.empty()) + { + write_line_to(outfile, ov_al); + ++y; } tc->set_word_wrap(wrapped); @@ -1225,9 +1219,10 @@ com_save_to(exec_context& ec, tc->set_word_wrap(wrapped); } else { + auto* los = tc->get_overlay_source(); std::vector rows(1); + attr_line_t ov_al; size_t count = 0; - std::string line; for (auto iter = all_user_marks.begin(); iter != all_user_marks.end(); iter++, count++) @@ -1238,6 +1233,15 @@ com_save_to(exec_context& ec, tc->listview_value_for_rows(*tc, *iter, rows); write_line_to(outfile, rows[0]); + auto y = 1_vl; + while (los != nullptr + && los->list_value_for_overlay( + *tc, y, tc->get_inner_height(), *iter, ov_al)) + { + write_line_to(outfile, ov_al); + ++y; + } + line_count += 1; } } @@ -1551,7 +1555,7 @@ com_highlight(exec_context& ec, if (lnav_data.ld_rl_view != nullptr) { lnav_data.ld_rl_view->add_possibility( - LNM_COMMAND, "highlight", args[1]); + ln_mode_t::COMMAND, "highlight", args[1]); } retval = "info: highlight pattern now active"; @@ -1591,7 +1595,7 @@ com_clear_highlight(exec_context& ec, if (lnav_data.ld_rl_view != NULL) { lnav_data.ld_rl_view->rem_possibility( - LNM_COMMAND, "highlight", args[1]); + ln_mode_t::COMMAND, "highlight", args[1]); } } } else { @@ -1991,7 +1995,7 @@ com_create_logline_table(exec_context& ec, custom_logline_tables.insert(args[1]); if (lnav_data.ld_rl_view != NULL) { lnav_data.ld_rl_view->add_possibility( - LNM_COMMAND, "custom-table", args[1]); + ln_mode_t::COMMAND, "custom-table", args[1]); } retval = "info: created new log table -- " + args[1]; } else { @@ -2032,7 +2036,7 @@ com_delete_logline_table(exec_context& ec, if (rc.empty()) { if (lnav_data.ld_rl_view != NULL) { lnav_data.ld_rl_view->rem_possibility( - LNM_COMMAND, "custom-table", args[1]); + ln_mode_t::COMMAND, "custom-table", args[1]); } retval = "info: deleted logline table"; } else { @@ -2103,7 +2107,7 @@ com_create_search_table(exec_context& ec, custom_search_tables.insert(args[1]); if (lnav_data.ld_rl_view != NULL) { lnav_data.ld_rl_view->add_possibility( - LNM_COMMAND, "search-table", args[1]); + ln_mode_t::COMMAND, "search-table", args[1]); } retval = "info: created new search table -- " + args[1]; } else { @@ -2140,7 +2144,7 @@ com_delete_search_table(exec_context& ec, if (rc.empty()) { if (lnav_data.ld_rl_view != NULL) { lnav_data.ld_rl_view->rem_possibility( - LNM_COMMAND, "search-table", args[1]); + ln_mode_t::COMMAND, "search-table", args[1]); } retval = "info: deleted search table"; } else { @@ -3156,8 +3160,7 @@ com_summarize(exec_context& ec, const auto& top_source = ec.ec_source.top(); sql_progress_guard progress_guard(sql_progress, sql_progress_finished, - top_source.s_source, - top_source.s_line, + top_source.s_location, top_source.s_content); auto_mem stmt(sqlite3_finalize); int retcode; @@ -3848,6 +3851,14 @@ com_poll_now(exec_context& ec, return Ok(std::string()); } +static Result +com_test_comment(exec_context& ec, + std::string cmdline, + std::vector& args) +{ + return Ok(std::string()); +} + static Result com_redraw(exec_context& ec, std::string cmdline, @@ -4014,7 +4025,9 @@ com_config(exec_context& ec, if (args.empty()) { args.emplace_back("config-option"); } else if (args.size() > 1) { - yajlpp_parse_context ypc("input", &lnav_config_handlers); + static const auto INPUT_SRC = intern_string::lookup("input"); + + yajlpp_parse_context ypc(INPUT_SRC, &lnav_config_handlers); std::vector errors; std::string option = args[1]; @@ -4129,10 +4142,7 @@ com_config(exec_context& ec, if (changed) { intern_string_t path = intern_string::lookup(option); - lnav_config_locations[path] = { - intern_string::lookup(ec.ec_source.top().s_source), - ec.ec_source.top().s_line, - }; + lnav_config_locations[path] = ec.ec_source.top().s_location; reload_config(errors); if (!errors.empty()) { @@ -4170,7 +4180,9 @@ com_reset_config(exec_context& ec, } else if (args.size() == 1) { return ec.make_error("expecting a configuration option to reset"); } else { - yajlpp_parse_context ypc("input", &lnav_config_handlers); + static const auto INPUT_SRC = intern_string::lookup("input"); + + yajlpp_parse_context ypc(INPUT_SRC, &lnav_config_handlers); std::string option = args[1]; lnav_config = rollback_lnav_config; @@ -4401,10 +4413,10 @@ public: class db_spectro_value_source : public spectrogram_value_source { public: db_spectro_value_source(std::string colname) - : dsvs_colname(std::move(colname)), dsvs_begin_time(0), dsvs_end_time(0) + : dsvs_colname(std::move(colname)) { this->update_stats(); - }; + } void update_stats() { @@ -4425,12 +4437,12 @@ public: return; } - if (this->dsvs_column_index == -1) { + if (!this->dsvs_column_index) { this->dsvs_error_msg = "unknown column -- " + this->dsvs_colname; return; } - if (!dls.dls_headers[this->dsvs_column_index].hm_graphable) { + if (!dls.dls_headers[this->dsvs_column_index.value()].hm_graphable) { this->dsvs_error_msg = "column is not numeric -- " + this->dsvs_colname; return; @@ -4449,9 +4461,9 @@ public: this->dsvs_stats.lvs_min_value = bs.bs_min_value; this->dsvs_stats.lvs_max_value = bs.bs_max_value; this->dsvs_stats.lvs_count = dls.dls_rows.size(); - }; + } - void spectro_bounds(spectrogram_bounds& sb_out) + void spectro_bounds(spectrogram_bounds& sb_out) override { db_label_source& dls = lnav_data.ld_db_row_source; @@ -4466,9 +4478,9 @@ public: sb_out.sb_min_value_out = this->dsvs_stats.lvs_min_value; sb_out.sb_max_value_out = this->dsvs_stats.lvs_max_value; sb_out.sb_count = this->dsvs_stats.lvs_count; - }; + } - void spectro_row(spectrogram_request& sr, spectrogram_row& row_out) + void spectro_row(spectrogram_request& sr, spectrogram_row& row_out) override { db_label_source& dls = lnav_data.ld_db_row_source; auto begin_row = dls.row_for_time({sr.sr_begin_time, 0}).value_or(0_vl); @@ -4478,23 +4490,25 @@ public: for (auto lpc = begin_row; lpc < end_row; ++lpc) { double value = 0.0; - sscanf(dls.dls_rows[lpc][this->dsvs_column_index], "%lf", &value); + sscanf(dls.dls_rows[lpc][this->dsvs_column_index.value()], + "%lf", + &value); row_out.add_value(sr, value, false); } - }; + } void spectro_mark(textview_curses& tc, time_t begin_time, time_t end_time, double range_min, - double range_max){}; + double range_max) override{}; std::string dsvs_colname; logline_value_stats dsvs_stats; - time_t dsvs_begin_time; - time_t dsvs_end_time; - int dsvs_column_index; + time_t dsvs_begin_time{0}; + time_t dsvs_end_time{0}; + nonstd::optional dsvs_column_index; std::string dsvs_error_msg; }; @@ -4589,9 +4603,10 @@ command_prompt(std::vector& args) lnav_data.ld_exec_context.ec_top_line = tc->get_top(); - lnav_data.ld_rl_view->clear_possibilities(LNM_COMMAND, + lnav_data.ld_rl_view->clear_possibilities(ln_mode_t::COMMAND, "numeric-colname"); - lnav_data.ld_rl_view->clear_possibilities(LNM_COMMAND, "colname"); + lnav_data.ld_rl_view->clear_possibilities(ln_mode_t::COMMAND, + "colname"); ldh.parse_line(log_view.get_top(), true); @@ -4604,7 +4619,7 @@ command_prompt(std::vector& args) } lnav_data.ld_rl_view->add_possibility( - LNM_COMMAND, "numeric-colname", dls_header.hm_name); + ln_mode_t::COMMAND, "numeric-colname", dls_header.hm_name); } } else { for (auto& ldh_line_value : ldh.ldh_line_values) { @@ -4622,67 +4637,74 @@ command_prompt(std::vector& args) } lnav_data.ld_rl_view->add_possibility( - LNM_COMMAND, "numeric-colname", meta.lvm_name.to_string()); + ln_mode_t::COMMAND, + "numeric-colname", + meta.lvm_name.to_string()); } } for (auto& cn_name : ldh.ldh_namer->cn_names) { lnav_data.ld_rl_view->add_possibility( - LNM_COMMAND, "colname", cn_name); + ln_mode_t::COMMAND, "colname", cn_name); } for (const auto& iter : ldh.ldh_namer->cn_builtin_names) { if (iter == "col") { continue; } - lnav_data.ld_rl_view->add_possibility(LNM_COMMAND, "colname", iter); + lnav_data.ld_rl_view->add_possibility( + ln_mode_t::COMMAND, "colname", iter); } ldh.clear(); readline_curses* rlc = lnav_data.ld_rl_view; - rlc->clear_possibilities(LNM_COMMAND, "move-time"); - rlc->add_possibility(LNM_COMMAND, "move-time", MOVE_TIMES); - rlc->clear_possibilities(LNM_COMMAND, "line-time"); + rlc->clear_possibilities(ln_mode_t::COMMAND, "move-time"); + rlc->add_possibility(ln_mode_t::COMMAND, "move-time", MOVE_TIMES); + rlc->clear_possibilities(ln_mode_t::COMMAND, "line-time"); { struct timeval tv = lf->get_time_offset(); char buffer[64]; sql_strftime( buffer, sizeof(buffer), ll->get_time(), ll->get_millis(), 'T'); - rlc->add_possibility(LNM_COMMAND, "line-time", buffer); - rlc->add_possibility(LNM_COMMAND, "move-time", buffer); + rlc->add_possibility(ln_mode_t::COMMAND, "line-time", buffer); + rlc->add_possibility(ln_mode_t::COMMAND, "move-time", buffer); sql_strftime(buffer, sizeof(buffer), ll->get_time() - tv.tv_sec, ll->get_millis() - (tv.tv_usec / 1000), 'T'); - rlc->add_possibility(LNM_COMMAND, "line-time", buffer); - rlc->add_possibility(LNM_COMMAND, "move-time", buffer); + rlc->add_possibility(ln_mode_t::COMMAND, "line-time", buffer); + rlc->add_possibility(ln_mode_t::COMMAND, "move-time", buffer); } } rollback_lnav_config = lnav_config; lnav_data.ld_doc_status_source.set_title("Command Help"); - add_view_text_possibilities( - lnav_data.ld_rl_view, LNM_COMMAND, "filter", tc); + add_view_text_possibilities(lnav_data.ld_rl_view, + ln_mode_t::COMMAND, + "filter", + tc, + text_quoting::none); lnav_data.ld_rl_view->add_possibility( - LNM_COMMAND, "filter", tc->get_current_search()); + ln_mode_t::COMMAND, "filter", tc->get_current_search()); add_filter_possibilities(tc); add_mark_possibilities(); add_config_possibilities(); - add_env_possibilities(LNM_COMMAND); + add_env_possibilities(ln_mode_t::COMMAND); add_tag_possibilities(); add_file_possibilities(); add_recent_netlocs_possibilities(); if (tc == &lnav_data.ld_views[LNV_LOG]) { add_filter_expr_possibilities( - lnav_data.ld_rl_view, LNM_COMMAND, "filter-expr-syms"); + lnav_data.ld_rl_view, ln_mode_t::COMMAND, "filter-expr-syms"); } - lnav_data.ld_mode = LNM_COMMAND; - lnav_data.ld_rl_view->focus( - LNM_COMMAND, cget(args, 2).value_or(":"), cget(args, 3).value_or("")); + lnav_data.ld_mode = ln_mode_t::COMMAND; + lnav_data.ld_rl_view->focus(ln_mode_t::COMMAND, + cget(args, 2).value_or(":"), + cget(args, 3).value_or("")); } static void @@ -4691,19 +4713,21 @@ script_prompt(std::vector& args) textview_curses* tc = *lnav_data.ld_view_stack.top(); auto& scripts = injector::get(); - lnav_data.ld_mode = LNM_EXEC; + lnav_data.ld_mode = ln_mode_t::EXEC; lnav_data.ld_exec_context.ec_top_line = tc->get_top(); - lnav_data.ld_rl_view->clear_possibilities(LNM_EXEC, "__command"); + lnav_data.ld_rl_view->clear_possibilities(ln_mode_t::EXEC, "__command"); find_format_scripts(lnav_data.ld_config_paths, scripts); for (const auto& iter : scripts.as_scripts) { lnav_data.ld_rl_view->add_possibility( - LNM_EXEC, "__command", iter.first); + ln_mode_t::EXEC, "__command", iter.first); } - add_view_text_possibilities(lnav_data.ld_rl_view, LNM_EXEC, "*", tc); - add_env_possibilities(LNM_EXEC); - lnav_data.ld_rl_view->focus( - LNM_EXEC, cget(args, 2).value_or("|"), cget(args, 3).value_or("")); + add_view_text_possibilities( + lnav_data.ld_rl_view, ln_mode_t::EXEC, "*", tc, text_quoting::regex); + add_env_possibilities(ln_mode_t::EXEC); + lnav_data.ld_rl_view->focus(ln_mode_t::EXEC, + cget(args, 2).value_or("|"), + cget(args, 3).value_or("")); lnav_data.ld_bottom_source.set_prompt( "Enter a script to execute: (Press " ANSI_BOLD("CTRL+]") " to abort)"); } @@ -4713,11 +4737,13 @@ search_prompt(std::vector& args) { textview_curses* tc = *lnav_data.ld_view_stack.top(); - lnav_data.ld_mode = LNM_SEARCH; + lnav_data.ld_mode = ln_mode_t::SEARCH; lnav_data.ld_search_start_line = tc->get_top(); - add_view_text_possibilities(lnav_data.ld_rl_view, LNM_SEARCH, "*", tc); - lnav_data.ld_rl_view->focus( - LNM_SEARCH, cget(args, 2).value_or("/"), cget(args, 3).value_or("")); + add_view_text_possibilities( + lnav_data.ld_rl_view, ln_mode_t::SEARCH, "*", tc, text_quoting::regex); + lnav_data.ld_rl_view->focus(ln_mode_t::SEARCH, + cget(args, 2).value_or("/"), + cget(args, 3).value_or("")); lnav_data.ld_doc_status_source.set_title("Syntax Help"); rl_set_help(); lnav_data.ld_bottom_source.set_prompt( @@ -4729,13 +4755,14 @@ search_prompt(std::vector& args) static void search_filters_prompt(std::vector& args) { - lnav_data.ld_mode = LNM_SEARCH_FILTERS; + lnav_data.ld_mode = ln_mode_t::SEARCH_FILTERS; lnav_data.ld_filter_view.reload_data(); add_view_text_possibilities(lnav_data.ld_rl_view, - LNM_SEARCH_FILTERS, + ln_mode_t::SEARCH_FILTERS, "*", - &lnav_data.ld_filter_view); - lnav_data.ld_rl_view->focus(LNM_SEARCH_FILTERS, + &lnav_data.ld_filter_view, + text_quoting::regex); + lnav_data.ld_rl_view->focus(ln_mode_t::SEARCH_FILTERS, cget(args, 2).value_or("/"), cget(args, 3).value_or("")); lnav_data.ld_bottom_source.set_prompt( @@ -4749,12 +4776,13 @@ search_files_prompt(std::vector& args) { static const std::regex re_escape(R"(([.\^$*+?()\[\]{}\\|]))"); - lnav_data.ld_mode = LNM_SEARCH_FILES; + lnav_data.ld_mode = ln_mode_t::SEARCH_FILES; for (const auto& lf : lnav_data.ld_active_files.fc_files) { auto path = pcrepp::quote(lf->get_unique_path()); - lnav_data.ld_rl_view->add_possibility(LNM_SEARCH_FILES, "*", path); + lnav_data.ld_rl_view->add_possibility( + ln_mode_t::SEARCH_FILES, "*", path); } - lnav_data.ld_rl_view->focus(LNM_SEARCH_FILES, + lnav_data.ld_rl_view->focus(ln_mode_t::SEARCH_FILES, cget(args, 2).value_or("/"), cget(args, 3).value_or("")); lnav_data.ld_bottom_source.set_prompt( @@ -4771,10 +4799,11 @@ sql_prompt(std::vector& args) lnav_data.ld_exec_context.ec_top_line = tc->get_top(); - lnav_data.ld_mode = LNM_SQL; + lnav_data.ld_mode = ln_mode_t::SQL; setup_logline_table(lnav_data.ld_exec_context); - lnav_data.ld_rl_view->focus( - LNM_SQL, cget(args, 2).value_or(";"), cget(args, 3).value_or("")); + lnav_data.ld_rl_view->focus(ln_mode_t::SQL, + cget(args, 2).value_or(";"), + cget(args, 3).value_or("")); lnav_data.ld_doc_status_source.set_title("Query Help"); rl_set_help(); @@ -4794,10 +4823,11 @@ user_prompt(std::vector& args) textview_curses* tc = *lnav_data.ld_view_stack.top(); lnav_data.ld_exec_context.ec_top_line = tc->get_top(); - lnav_data.ld_mode = LNM_USER; + lnav_data.ld_mode = ln_mode_t::USER; setup_logline_table(lnav_data.ld_exec_context); - lnav_data.ld_rl_view->focus( - LNM_USER, cget(args, 2).value_or("? "), cget(args, 3).value_or("")); + lnav_data.ld_rl_view->focus(ln_mode_t::USER, + cget(args, 2).value_or("? "), + cget(args, 3).value_or("")); lnav_data.ld_bottom_source.update_loading(0, 0); lnav_data.ld_status[LNS_BOTTOM].do_update(); @@ -5700,10 +5730,12 @@ init_lnav_commands(readline_context::command_map_t& cmd_map) } if (getenv("lnav_test") != nullptr) { static readline_context::command_t rebuild(com_rebuild), - shexec(com_shexec), poll_now(com_poll_now); + shexec(com_shexec), poll_now(com_poll_now), + test_comment(com_test_comment); cmd_map["rebuild"] = &rebuild; cmd_map["shexec"] = &shexec; cmd_map["poll-now"] = &poll_now; + cmd_map["test-comment"] = &test_comment; } } diff --git a/src/lnav_config.cc b/src/lnav_config.cc index 539bbb50..17d90ff3 100644 --- a/src/lnav_config.cc +++ b/src/lnav_config.cc @@ -173,7 +173,7 @@ ensure_dotlnav() } bool -install_from_git(const char* repo) +install_from_git(const std::string& repo) { static const std::regex repo_name_converter("[^\\w]"); @@ -197,18 +197,18 @@ install_from_git(const char* repo) auto git_cmd = fork_res.unwrap(); if (git_cmd.in_child()) { if (ghc::filesystem::is_directory(local_formats_path)) { - printf("Updating format repo: %s\n", repo); + fmt::print("Updating format repo: {}\n", repo); log_perror(chdir(local_formats_path.c_str())); execlp("git", "git", "pull", nullptr); } else if (ghc::filesystem::is_directory(local_configs_path)) { - printf("Updating config repo: %s\n", repo); + fmt::print("Updating config repo: {}\n", repo); log_perror(chdir(local_configs_path.c_str())); execlp("git", "git", "pull", nullptr); } else { execlp("git", "git", "clone", - repo, + repo.c_str(), local_staging_path.c_str(), nullptr); } @@ -344,7 +344,8 @@ install_extra_formats() if ((fd = lnav::filesystem::openp(config_json, O_RDONLY)) == -1) { perror("Unable to open remote-config.json"); } else { - yajlpp_parse_context ypc_config(config_root.string(), &format_handlers); + yajlpp_parse_context ypc_config( + intern_string::lookup(config_root.string()), &format_handlers); auto_mem jhandle(yajl_free); unsigned char buffer[4096]; ssize_t rc; @@ -639,6 +640,13 @@ static const struct json_path_container theme_styles_handlers = { return &root->lt_style_header[5]; }) .with_children(style_config_handlers), + yajlpp::property_handler("list-glyph") + .with_description("Styling for glyphs that prefix a list item") + .with_obj_provider( + [](const yajlpp_provider_context& ypc, lnav_theme* root) { + return &root->lt_style_list_glyph; + }) + .with_children(style_config_handlers), }; static const struct json_path_container theme_syntax_styles_handlers = { @@ -1271,7 +1279,8 @@ load_config_from(_lnav_config& lconfig, const ghc::filesystem::path& path, std::vector& errors) { - yajlpp_parse_context ypc(path.string(), &lnav_config_handlers); + yajlpp_parse_context ypc(intern_string::lookup(path.string()), + &lnav_config_handlers); struct userdata ud(errors); auto_fd fd; @@ -1326,7 +1335,8 @@ load_default_config(struct _lnav_config& config_obj, const bin_src_file& bsf, std::vector& errors) { - yajlpp_parse_context ypc_builtin(bsf.get_name(), &lnav_config_handlers); + yajlpp_parse_context ypc_builtin(intern_string::lookup(bsf.get_name()), + &lnav_config_handlers); auto_mem handle(yajl_free); struct userdata ud(errors); @@ -1501,7 +1511,7 @@ reload_config(std::vector& errors) .with_reason(errmsg) .with_snippet( lnav::console::snippet::from( - loc_iter->second.sl_source.to_string(), "") + loc_iter->second.sl_source, "") .with_line(loc_iter->second.sl_line_number))); }; diff --git a/src/lnav_config.hh b/src/lnav_config.hh index 4e27c57c..5eda1d53 100644 --- a/src/lnav_config.hh +++ b/src/lnav_config.hh @@ -69,7 +69,7 @@ bool check_experimental(const char* feature_name); */ void ensure_dotlnav(); -bool install_from_git(const char* repo); +bool install_from_git(const std::string& repo); bool update_installs_from_git(); void install_extra_formats(); diff --git a/src/lnav_util.cc b/src/lnav_util.cc index ffb672a4..1970e18d 100644 --- a/src/lnav_util.cc +++ b/src/lnav_util.cc @@ -37,9 +37,12 @@ #include #include "base/ansi_scrubber.hh" +#include "base/itertools.hh" #include "base/result.h" +#include "bookmarks.hh" #include "config.h" #include "fmt/format.h" +#include "log_format_fwd.hh" #include "view_curses.hh" #include "yajlpp/yajlpp.hh" #include "yajlpp/yajlpp_def.hh" @@ -109,8 +112,44 @@ err_to_ok(const lnav::console::user_message msg) return Ok(msg.to_attr_line().get_string()); } +short +pollfd_revents(const std::vector& pollfds, int fd) +{ + return pollfds | lnav::itertools::find_if([fd](const auto& entry) { + return entry.fd == fd; + }) + | lnav::itertools::map(&pollfd::revents) + | lnav::itertools::unwrap_or((short) 0); +} + +void +write_line_to(FILE* outfile, const attr_line_t& al) +{ + const auto& al_attrs = al.get_attrs(); + auto lr = find_string_attr_range(al_attrs, &SA_ORIGINAL_LINE); + + if (!lr.is_valid() || lr.lr_start > 1) { + // If the line is prefixed with some extra information, include that + // in the output. For example, the log file name or time offset. + lnav::console::println(outfile, al); + } else { + lnav::console::println(outfile, al.subline(lr.lr_start, lr.length())); + } +} + namespace lnav { +std::string +to_json(const std::string& str) +{ + yajlpp_gen gen; + + yajl_gen_config(gen, yajl_gen_beautify, false); + yajl_gen_string(gen, str); + + return gen.to_string_fragment().to_string(); +} + static void to_json(yajlpp_gen& gen, const attr_line_t& al) { @@ -173,6 +212,9 @@ to_json(const lnav::console::user_message& um) root_map.gen("level"); switch (um.um_level) { + case console::user_message::level::raw: + root_map.gen("raw"); + break; case console::user_message::level::ok: root_map.gen("ok"); break; @@ -199,11 +241,9 @@ to_json(const lnav::console::user_message& um) yajlpp_map snip_map(gen); snip_map.gen("source"); - snip_map.gen(snip.s_source); + snip_map.gen(snip.s_location.sl_source); snip_map.gen("line"); - snip_map.gen(snip.s_line); - snip_map.gen("column"); - snip_map.gen(snip.s_column); + snip_map.gen(snip.s_location.sl_line_number); snip_map.gen("content"); to_json(gen, snip.s_content); } @@ -234,10 +274,10 @@ read_string_attr_type(yajlpp_parse_context* ypc, static int read_string_attr_int_value(yajlpp_parse_context* ypc, long long in) { - auto sa = (string_attr*) ypc->ypc_obj_stack.top(); + auto* sa = (string_attr*) ypc->ypc_obj_stack.top(); if (sa->sa_type == &VC_ROLE) { - sa->sa_value = (role_t) in; + sa->sa_value = static_cast(in); } return 1; } @@ -251,39 +291,27 @@ static const struct json_path_container string_attr_handlers = { yajlpp::property_handler("value").add_cb(read_string_attr_int_value), }; -static const struct json_path_container attr_line_handlers = { +static const typed_json_path_container attr_line_handlers = { yajlpp::property_handler("str").for_field(&attr_line_t::al_string), yajlpp::property_handler("attrs#") - .with_obj_provider( - [](const yajlpp_provider_context& ypc, attr_line_t* root) { - root->al_attrs.resize(ypc.ypc_index + 1); - - return &root->al_attrs[ypc.ypc_index]; - }) + .for_field(&attr_line_t::al_attrs) .with_children(string_attr_handlers), }; template<> -attr_line_t +Result> from_json(const std::string& json) { - yajlpp_parse_context ypc("string", &attr_line_handlers); - auto_mem handle(yajl_free); - attr_line_t retval; + static const auto STRING_SRC = intern_string::lookup("string"); - handle = yajl_alloc(&ypc.ypc_callbacks, nullptr, &ypc); - ypc.with_handle(handle); - ypc.with_obj(retval); - ypc.parse(json); - ypc.complete_parse(); - - return retval; + return attr_line_handlers.parser_for(STRING_SRC).of(json); } static const json_path_container snippet_handlers = { - yajlpp::property_handler("source").for_field(&console::snippet::s_source), - yajlpp::property_handler("line").for_field(&console::snippet::s_line), - yajlpp::property_handler("column").for_field(&console::snippet::s_column), + yajlpp::property_handler("source").for_field(&console::snippet::s_location, + &source_location::sl_source), + yajlpp::property_handler("line").for_field( + &console::snippet::s_location, &source_location::sl_line_number), yajlpp::property_handler("content") .with_obj_provider( [](const yajlpp_provider_context& ypc, console::snippet* snip) { @@ -293,6 +321,7 @@ static const json_path_container snippet_handlers = { }; static const json_path_handler_base::enum_value_t LEVEL_ENUM[] = { + {"raw", lnav::console::user_message::level::raw}, {"ok", lnav::console::user_message::level::ok}, {"info", lnav::console::user_message::level::info}, {"warning", lnav::console::user_message::level::warning}, @@ -301,51 +330,44 @@ static const json_path_handler_base::enum_value_t LEVEL_ENUM[] = { json_path_handler_base::ENUM_TERMINATOR, }; -static const struct json_path_container user_message_handlers = { - yajlpp::property_handler("level") - .with_enum_values(LEVEL_ENUM) - .for_field(&console::user_message::um_level), - yajlpp::property_handler("message") - .with_obj_provider( - [](const yajlpp_provider_context& ypc, - console::user_message* root) { return &root->um_message; }) - .with_children(attr_line_handlers), - yajlpp::property_handler("reason") - .with_obj_provider( - [](const yajlpp_provider_context& ypc, - console::user_message* root) { return &root->um_reason; }) - .with_children(attr_line_handlers), - yajlpp::property_handler("snippets#") - .with_obj_provider( - [](const yajlpp_provider_context& ypc, - console::user_message* root) { - root->um_snippets.resize(ypc.ypc_index + 1); - - return &root->um_snippets[ypc.ypc_index]; - }) - .with_children(snippet_handlers), - yajlpp::property_handler("help") - .with_obj_provider( - [](const yajlpp_provider_context& ypc, - console::user_message* root) { return &root->um_help; }) - .with_children(attr_line_handlers), +static const typed_json_path_container + user_message_handlers = { + yajlpp::property_handler("level") + .with_enum_values(LEVEL_ENUM) + .for_field(&console::user_message::um_level), + yajlpp::property_handler("message") + .with_obj_provider( + [](const yajlpp_provider_context& ypc, + console::user_message* root) { return &root->um_message; }) + .with_children(attr_line_handlers), + yajlpp::property_handler("reason") + .with_obj_provider( + [](const yajlpp_provider_context& ypc, + console::user_message* root) { return &root->um_reason; }) + .with_children(attr_line_handlers), + yajlpp::property_handler("snippets#") + .with_obj_provider( + [](const yajlpp_provider_context& ypc, + console::user_message* root) { + root->um_snippets.resize(ypc.ypc_index + 1); + + return &root->um_snippets[ypc.ypc_index]; + }) + .with_children(snippet_handlers), + yajlpp::property_handler("help") + .with_obj_provider( + [](const yajlpp_provider_context& ypc, + console::user_message* root) { return &root->um_help; }) + .with_children(attr_line_handlers), }; template<> -lnav::console::user_message +Result> from_json(const std::string& json) { - yajlpp_parse_context ypc("string", &user_message_handlers); - auto_mem handle(yajl_free); - lnav::console::user_message retval; - - handle = yajl_alloc(&ypc.ypc_callbacks, nullptr, &ypc); - ypc.with_handle(handle); - ypc.with_obj(retval); - ypc.parse(json); - ypc.complete_parse(); + static const auto STRING_SRC = intern_string::lookup("string"); - return retval; + return user_message_handlers.parser_for(STRING_SRC).of(json); } } // namespace lnav diff --git a/src/lnav_util.hh b/src/lnav_util.hh index 2bb53f25..a7025b02 100644 --- a/src/lnav_util.hh +++ b/src/lnav_util.hh @@ -132,26 +132,14 @@ to_string(const char* s) } // namespace std inline bool -is_glob(const char* fn) +is_glob(const std::string& fn) { - return (strchr(fn, '*') != nullptr || strchr(fn, '?') != nullptr - || strchr(fn, '[') != nullptr); + return (fn.find('*') != std::string::npos + || fn.find('?') != std::string::npos + || fn.find('[') != std::string::npos); }; -inline short -pollfd_revents(const std::vector& pollfds, int fd) -{ - auto iter - = std::find_if(pollfds.begin(), pollfds.end(), [fd](const auto& entry) { - return entry.fd == fd; - }); - - if (iter == pollfds.end()) { - return 0; - } - - return iter->revents; -} +short pollfd_revents(const std::vector& pollfds, int fd); inline bool pollfd_ready(const std::vector& pollfds, @@ -233,13 +221,17 @@ std::string err_prefix(std::string msg); Result err_to_ok( lnav::console::user_message msg); +void write_line_to(FILE* outfile, const attr_line_t& al); + namespace lnav { +std::string to_json(const std::string& str); std::string to_json(const lnav::console::user_message& um); std::string to_json(const attr_line_t& al); template -T from_json(const std::string& str); +Result> from_json( + const std::string& json); } // namespace lnav diff --git a/src/log_actions.hh b/src/log_actions.hh index 8a9220c4..02dc4763 100644 --- a/src/log_actions.hh +++ b/src/log_actions.hh @@ -46,9 +46,9 @@ public: std::function)> piper_cb) : ad_log_helper(lss), ad_child_cb(std::move(child_cb)), - ad_piper_cb(std::move(piper_cb)){ - - }; + ad_piper_cb(std::move(piper_cb)) + { + } bool text_handle_mouse(textview_curses& tc, mouse_event& me) override; diff --git a/src/log_data_helper.hh b/src/log_data_helper.hh index f86b77be..d40fa900 100644 --- a/src/log_data_helper.hh +++ b/src/log_data_helper.hh @@ -48,10 +48,7 @@ class log_data_helper { public: - explicit log_data_helper(logfile_sub_source& lss) - : ldh_log_source(lss){ - - }; + explicit log_data_helper(logfile_sub_source& lss) : ldh_log_source(lss) {} void clear(); @@ -70,7 +67,7 @@ public: return std::count(this->ldh_msg.get_data(), this->ldh_msg.get_data() + lv.lv_origin.lr_start, '\n'); - }; + } std::string format_json_getter(const intern_string_t field, int index); diff --git a/src/log_data_table.hh b/src/log_data_table.hh index e6296814..80356c19 100644 --- a/src/log_data_table.hh +++ b/src/log_data_table.hh @@ -53,13 +53,13 @@ public: void get_columns(std::vector& cols) const override { cols = this->ldt_cols; - }; + } void get_foreign_keys(std::vector& keys_inout) const override { log_vtab_impl::get_foreign_keys(keys_inout); keys_inout.emplace_back("log_msg_instance"); - }; + } bool next(log_cursor& lc, logfile_sub_source& lss) override; diff --git a/src/log_format.cc b/src/log_format.cc index 4fe2d9b3..2038dea4 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -46,6 +46,8 @@ #include "yajlpp/yajlpp.hh" #include "yajlpp/yajlpp_def.hh" +using namespace lnav::roles::literals; + static auto intern_lifetime = intern_string::get_table_lifetime(); string_attr_type logline::L_PREFIX("prefix"); @@ -1165,9 +1167,7 @@ external_log_format::rewrite(exec_context& ec, } auto _sg = ec.enter_source( - this->elf_name.to_string() + ":" + vd_iter->first.to_string(), - 1, - vd.vd_rewriter); + vd_iter->second->vd_rewrite_src_name, 1, vd.vd_rewriter); auto field_value = execute_any(ec, vd.vd_rewriter).orElse(err_to_ok).unwrap(); struct line_range adj_origin @@ -1391,12 +1391,12 @@ external_log_format::get_subline(const logline& ll, size_t begin_size = this->jlf_cached_line.size(); switch (jfe.jfe_type) { - case JLF_CONSTANT: + case json_log_field::CONSTANT: this->json_append_to_cache( jfe.jfe_default_value.c_str(), jfe.jfe_default_value.size()); break; - case JLF_VARIABLE: + case json_log_field::VARIABLE: lv_iter = find_if( this->jlf_line_values.begin(), this->jlf_line_values.end(), @@ -1792,8 +1792,8 @@ external_log_format::build(std::vector& errors) .with_snippets(this->get_snippets())); } if (this->elf_type == elf_type_t::ELF_TYPE_JSON) { - this->jlf_parse_context = std::make_shared( - this->elf_name.to_string()); + this->jlf_parse_context + = std::make_shared(this->elf_name); this->jlf_yajl_handle.reset( yajl_alloc(&this->jlf_parse_context->ypc_callbacks, nullptr, @@ -1820,6 +1820,7 @@ external_log_format::build(std::vector& errors) for (auto& vd : this->elf_value_def_order) { std::vector::iterator act_iter; + vd->vd_meta.lvm_format = this; if (!vd->vd_internal && vd->vd_meta.lvm_column == -1) { vd->vd_meta.lvm_column = this->elf_column_count++; } @@ -1841,6 +1842,8 @@ external_log_format::build(std::vector& errors) #endif } } + + vd->set_rewrite_src_name(); } if (this->elf_type == elf_type_t::ELF_TYPE_TEXT @@ -1857,8 +1860,10 @@ external_log_format::build(std::vector& errors) } for (auto& elf_sample : this->elf_samples) { + auto sample_lines + = string_fragment(elf_sample.s_line.pp_value).split_lines(); pcre_context_static<128> pc; - pcre_input pi(elf_sample.s_line.pp_value); + pcre_input pi(sample_lines[0]); bool found = false; for (auto pat_iter = this->elf_pattern_order.begin(); @@ -1930,8 +1935,8 @@ external_log_format::build(std::vector& errors) || dts.scan(ts, ts_len, custom_formats, &tm, tv) == nullptr) { attr_line_t notes; - notes.append("the following formats were tried:"); if (custom_formats == nullptr) { + notes.append("the following built-in formats were tried:"); for (int lpc = 0; PTIMEC_FORMATS[lpc].pf_fmt != nullptr; lpc++) { off_t off = 0; @@ -1941,13 +1946,13 @@ external_log_format::build(std::vector& errors) .append(ts, (size_t) ts_len) .append("\n") .append(2 + off, ' ') - .append(lnav::roles::comment("^ ")) + .append("^ "_comment) .append_quoted( lnav::roles::symbol(PTIMEC_FORMATS[lpc].pf_fmt)) - .append( - lnav::roles::comment(" matched up to here")); + .append(" matched up to here"_comment); } } else { + notes.append("the following custom formats were tried:"); for (int lpc = 0; custom_formats[lpc] != nullptr; lpc++) { off_t off = 0; @@ -1956,22 +1961,26 @@ external_log_format::build(std::vector& errors) .append(ts, (size_t) ts_len) .append("\n") .append(2 + off, ' ') - .append(lnav::roles::comment("^ ")) + .append("^ "_comment) .append_quoted( lnav::roles::symbol(custom_formats[lpc])) - .append( - lnav::roles::comment(" matched up to here")); + .append(" matched up to here"_comment); } } errors.emplace_back( lnav::console::user_message::error( - attr_line_t("invalid sample log message ") - .append_quoted(elf_sample.s_line.pp_value)) + attr_line_t("invalid sample log message: ") + .append(lnav::to_json(elf_sample.s_line.pp_value))) .with_reason(attr_line_t("unrecognized timestamp -- ") .append(ts, (size_t) ts_len)) .with_snippet(elf_sample.s_line.to_snippet()) - .with_note(notes)); + .with_note(notes) + .with_help(attr_line_t("If the timestamp format is not " + "supported by default, you can " + "add a custom format with the ") + .append_quoted("timestamp-format"_symbol) + .append(" property"))); } log_level_t level = this->convert_level(pi, level_cap); @@ -1980,8 +1989,8 @@ external_log_format::build(std::vector& errors) && elf_sample.s_level != level) { errors.emplace_back( lnav::console::user_message::error( - attr_line_t("invalid sample log message ") - .append_quoted(elf_sample.s_line.pp_value)) + attr_line_t("invalid sample log message: ") + .append(lnav::to_json(elf_sample.s_line.pp_value))) .with_reason(attr_line_t() .append_quoted(lnav::roles::symbol( level_names[level])) @@ -1991,14 +2000,35 @@ external_log_format::build(std::vector& errors) level_names[elf_sample.s_level]))) .with_snippet(elf_sample.s_line.to_snippet())); } + + { + pcre_context_static<128> pc_full; + pcre_input pi_full(elf_sample.s_line.pp_value); + + if (!pat.p_pcre->match(pc_full, pi_full) + || pc_full.all()->length() + != elf_sample.s_line.pp_value.length()) + { + errors.emplace_back( + lnav::console::user_message::error( + attr_line_t("invalid pattern: ") + .append_quoted(lnav::roles::symbol(pat.p_name))) + .with_reason("pattern does not match entire " + "multiline message") + .with_snippet(elf_sample.s_line.to_snippet()) + .with_help(attr_line_t("using ") + .append_quoted(".*") + .append(" when capturing the body " + "will match new-lines"))); + } + } } if (!found && !this->elf_pattern_order.empty()) { - attr_line_t notes( - "the following shows how each pattern matched this sample:\n"); + std::vector> partial_indexes; + attr_line_t notes; size_t max_name_width = 0; - notes.append(" ").append_quoted(elf_sample.s_line.pp_value); for (const auto& pat_iter : this->elf_pattern_order) { pattern& pat = *pat_iter; @@ -2006,21 +2036,37 @@ external_log_format::build(std::vector& errors) continue; } - size_t partial_len = pat.p_pcre->match_partial(pi); - - notes.append("\n ") - .append(partial_len, ' ') - .append(lnav::roles::comment("^ ")) - .append(lnav::roles::symbol(pat.p_name)) - .append(lnav::roles::comment(" matched up to here")); - + partial_indexes.emplace_back(pat.p_pcre->match_partial(pi), + pat.p_name); max_name_width = std::max(max_name_width, pat.p_name.size()); } + for (const auto& line_frag : sample_lines) { + auto src_line = line_frag.to_string(); + if (!endswith(src_line, "\n")) { + src_line.append("\n"); + } + notes.append(" ").append(src_line); + for (auto& part_pair : partial_indexes) { + if (part_pair.first >= 0 + && part_pair.first < line_frag.length()) { + notes.append(" ") + .append(part_pair.first, ' ') + .append("^ "_comment) + .append(lnav::roles::symbol(part_pair.second)) + .append(" matched up to here"_comment) + .append("\n"); + } + part_pair.first -= line_frag.length(); + } + } + notes.add_header( + "the following shows how each pattern matched this sample:\n"); - attr_line_t help; + attr_line_t regex_note; for (const auto& pat_iter : this->elf_pattern_order) { if (!pat_iter->p_pcre) { - help.append( + regex_note + .append( lnav::roles::symbol(fmt::format(FMT_STRING("{:{}}"), pat_iter->p_name, max_name_width))) @@ -2028,22 +2074,22 @@ external_log_format::build(std::vector& errors) continue; } - help + regex_note .append(lnav::roles::symbol(fmt::format( FMT_STRING("{:{}}"), pat_iter->p_name, max_name_width))) .append(" = ") - .append(pat_iter->p_pcre->get_pattern()) + .append_quoted(pat_iter->p_pcre->get_pattern()) .append("\n"); } errors.emplace_back( lnav::console::user_message::error( - attr_line_t("invalid sample log message ") - .append_quoted(elf_sample.s_line.pp_value)) + attr_line_t("invalid sample log message: ") + .append(lnav::to_json(elf_sample.s_line.pp_value))) .with_reason("sample does not match any patterns") .with_snippet(elf_sample.s_line.to_snippet()) - .with_note(notes) - .with_help(help)); + .with_note(notes.rtrim()) + .with_note(regex_note)); } } @@ -2097,7 +2143,7 @@ external_log_format::build(std::vector& errors) } switch (jfe.jfe_type) { - case JLF_VARIABLE: { + case json_log_field::VARIABLE: { auto vd_iter = this->elf_value_defs.find(jfe.jfe_value.pp_value); if (jfe.jfe_value.pp_value == ts) { @@ -2124,7 +2170,7 @@ external_log_format::build(std::vector& errors) } break; } - case JLF_CONSTANT: + case json_log_field::CONSTANT: this->jlf_line_format_init_count += std::count(jfe.jfe_default_value.begin(), jfe.jfe_default_value.end(), @@ -2404,8 +2450,8 @@ external_log_format::specialized(int fmt_lock) } if (this->elf_type == elf_type_t::ELF_TYPE_JSON) { - this->jlf_parse_context = std::make_shared( - this->elf_name.to_string()); + this->jlf_parse_context + = std::make_shared(this->elf_name); this->jlf_yajl_handle.reset( yajl_alloc(&this->jlf_parse_context->ypc_callbacks, nullptr, @@ -2444,6 +2490,103 @@ external_log_format::match_mime_type(const file_format_t ff) const return this->elf_mime_types.count(ff) == 1; } +long +external_log_format::value_line_count(const intern_string_t ist, + bool top_level, + const unsigned char* str, + ssize_t len) const +{ + const auto iter = this->elf_value_defs.find(ist); + long line_count + = (str != nullptr) ? std::count(&str[0], &str[len], '\n') + 1 : 1; + + if (iter == this->elf_value_defs.end()) { + return (this->jlf_hide_extra || !top_level) ? 0 : line_count; + } + + if (iter->second->vd_meta.lvm_hidden) { + return 0; + } + + if (std::find_if(this->jlf_line_format.begin(), + this->jlf_line_format.end(), + json_field_cmp(json_log_field::VARIABLE, ist)) + != this->jlf_line_format.end()) + { + return line_count - 1; + } + + return line_count; +} + +log_level_t +external_log_format::convert_level( + const pcre_input& pi, const pcre_context::capture_t* level_cap) const +{ + log_level_t retval = LEVEL_INFO; + + if (level_cap != nullptr && level_cap->is_valid()) { + pcre_context_static<128> pc_level; + pcre_input pi_level( + pi.get_substr_start(level_cap), 0, level_cap->length()); + + if (this->elf_level_patterns.empty()) { + retval = string2level(pi_level.get_string(), level_cap->length()); + } else { + for (const auto& elf_level_pattern : this->elf_level_patterns) { + if (elf_level_pattern.second.lp_pcre->match(pc_level, pi_level)) + { + retval = elf_level_pattern.first; + break; + } + } + } + } + + return retval; +} + +logline_value_meta +external_log_format::get_value_meta(intern_string_t field_name, + value_kind_t kind) +{ + auto iter = this->elf_value_defs.find(field_name); + + if (iter == this->elf_value_defs.end()) { + auto retval = logline_value_meta(field_name, kind, -1, this); + + retval.lvm_hidden = this->jlf_hide_extra; + return retval; + } + + auto lvm = iter->second->vd_meta; + + lvm.lvm_kind = kind; + return lvm; +} + +void +external_log_format::json_append( + const external_log_format::json_format_element& jfe, + const char* value, + ssize_t len) +{ + if (len == -1) { + len = strlen(value); + } + if (jfe.jfe_align == json_format_element::align_t::RIGHT) { + if (len < jfe.jfe_min_width) { + this->json_append_to_cache(jfe.jfe_min_width - len); + } + } + this->json_append_to_cache(value, len); + if (jfe.jfe_align == json_format_element::align_t::LEFT) { + if (len < jfe.jfe_min_width) { + this->json_append_to_cache(jfe.jfe_min_width - len); + } + } +} + int log_format::pattern_index_for_line(uint64_t line_number) const { @@ -2468,11 +2611,58 @@ log_format::get_pattern_name(uint64_t line_number) const return fmt::format(FMT_STRING("builtin ({})"), pat_index); } +std::shared_ptr +log_format::find_root_format(const char* name) +{ + auto& fmts = get_root_formats(); + for (auto& lf : fmts) { + if (lf->get_name() == name) { + return lf; + } + } + return nullptr; +} + log_format::pattern_for_lines::pattern_for_lines(uint32_t pfl_line, uint32_t pfl_pat_index) : pfl_line(pfl_line), pfl_pat_index(pfl_pat_index) { } +void +logline_value_stats::merge(const logline_value_stats& other) +{ + if (other.lvs_count == 0) { + return; + } + + require(other.lvs_min_value <= other.lvs_max_value); + + if (other.lvs_min_value < this->lvs_min_value) { + this->lvs_min_value = other.lvs_min_value; + } + if (other.lvs_max_value > this->lvs_max_value) { + this->lvs_max_value = other.lvs_max_value; + } + this->lvs_count += other.lvs_count; + this->lvs_total += other.lvs_total; + + ensure(this->lvs_count >= 0); + ensure(this->lvs_min_value <= this->lvs_max_value); +} + +void +logline_value_stats::add_value(double value) +{ + if (value < this->lvs_min_value) { + this->lvs_min_value = value; + } + if (value > this->lvs_max_value) { + this->lvs_max_value = value; + } + this->lvs_count += 1; + this->lvs_total += value; +} + /* XXX */ #include "log_format_impls.cc" diff --git a/src/log_format.hh b/src/log_format.hh index 2e2ad6ff..ad53fe21 100644 --- a/src/log_format.hh +++ b/src/log_format.hh @@ -51,7 +51,6 @@ #include "base/date_time_scanner.hh" #include "base/intern_string.hh" #include "base/lnav_log.hh" -#include "byte_array.hh" #include "file_format.hh" #include "highlighter.hh" #include "line_buffer.hh" @@ -73,8 +72,6 @@ enum class scale_op_t { }; struct scaling_factor { - scaling_factor() : sf_op(scale_op_t::SO_IDENTITY), sf_value(1){}; - template void scale(T& val) const { @@ -90,8 +87,8 @@ struct scaling_factor { } } - scale_op_t sf_op; - double sf_value; + scale_op_t sf_op{scale_op_t::SO_IDENTITY}; + double sf_value{1}; }; enum class value_kind_t : int { @@ -227,10 +224,7 @@ public: }; struct logline_value_stats { - logline_value_stats() - { - this->clear(); - }; + logline_value_stats() { this->clear(); } void clear() { @@ -238,40 +232,11 @@ struct logline_value_stats { this->lvs_total = 0; this->lvs_min_value = std::numeric_limits::max(); this->lvs_max_value = -std::numeric_limits::max(); - }; - - void merge(const logline_value_stats& other) - { - if (other.lvs_count == 0) { - return; - } - - require(other.lvs_min_value <= other.lvs_max_value); + } - if (other.lvs_min_value < this->lvs_min_value) { - this->lvs_min_value = other.lvs_min_value; - } - if (other.lvs_max_value > this->lvs_max_value) { - this->lvs_max_value = other.lvs_max_value; - } - this->lvs_count += other.lvs_count; - this->lvs_total += other.lvs_total; + void merge(const logline_value_stats& other); - ensure(this->lvs_count >= 0); - ensure(this->lvs_min_value <= this->lvs_max_value); - }; - - void add_value(double value) - { - if (value < this->lvs_min_value) { - this->lvs_min_value = value; - } - if (value > this->lvs_max_value) { - this->lvs_max_value = value; - } - this->lvs_count += 1; - this->lvs_total += value; - }; + void add_value(double value); int64_t lvs_count; double lvs_total; @@ -282,9 +247,9 @@ struct logline_value_stats { struct logline_value_cmp { explicit logline_value_cmp(const intern_string_t* name = nullptr, int col = -1) - : lvc_name(name), lvc_column(col){ - - }; + : lvc_name(name), lvc_column(col) + { + } bool operator()(const logline_value& lv) const { @@ -298,7 +263,7 @@ struct logline_value_cmp { } return retval; - }; + } const intern_string_t* lvc_name; int lvc_column; @@ -316,29 +281,18 @@ public: */ static std::vector>& get_root_formats(); - static std::shared_ptr find_root_format(const char* name) - { - auto& fmts = get_root_formats(); - for (auto& lf : fmts) { - if (lf->get_name() == name) { - return lf; - } - } - return nullptr; - } + static std::shared_ptr find_root_format(const char* name); struct action_def { std::string ad_name; std::string ad_label; std::vector ad_cmdline; - bool ad_capture_output; - - action_def() : ad_capture_output(false){}; + bool ad_capture_output{false}; bool operator<(const action_def& rhs) const { return this->ad_name < rhs.ad_name; - }; + } }; virtual ~log_format() = default; @@ -347,7 +301,7 @@ public: { this->lf_pattern_locks.clear(); this->lf_date_time.clear(); - }; + } /** * Get the name of this log format. @@ -356,10 +310,7 @@ public: */ virtual const intern_string_t get_name() const = 0; - virtual bool match_name(const std::string& filename) - { - return true; - }; + virtual bool match_name(const std::string& filename) { return true; } virtual bool match_mime_type(const file_format_t ff) const { @@ -367,7 +318,7 @@ public: return true; } return false; - }; + } enum scan_result_t { SCAN_MATCH, @@ -393,7 +344,7 @@ public: virtual bool scan_for_partial(shared_buffer_ref& sbr, size_t& len_out) const { return false; - }; + } /** * Remove redundant data from the log line string. @@ -409,7 +360,9 @@ public: shared_buffer_ref& sbr, string_attrs_t& sa, std::vector& values, - bool annotate_module = true) const {}; + bool annotate_module = true) const + { + } virtual void rewrite(exec_context& ec, shared_buffer_ref& line, @@ -417,20 +370,20 @@ public: std::string& value_out) { value_out.assign(line.get_data(), line.length()); - }; + } virtual const logline_value_stats* stats_for_value( const intern_string_t& name) const { return nullptr; - }; + } virtual std::shared_ptr specialized(int fmt_lock = -1) = 0; virtual std::shared_ptr get_vtab_impl() const { return nullptr; - }; + } virtual void get_subline(const logline& ll, shared_buffer_ref& sbr, @@ -440,7 +393,7 @@ public: const logline_value& lv) const { return nullptr; - }; + } virtual std::set get_source_path() const { @@ -449,12 +402,12 @@ public: retval.insert("default"); return retval; - }; + } virtual bool hide_field(const intern_string_t field_name, bool val) { return false; - }; + } const char* const* get_timestamp_formats() const { @@ -463,7 +416,7 @@ public: } return &this->lf_timestamp_format[0]; - }; + } void check_for_new_year(std::vector& dst, exttm log_tv, @@ -474,7 +427,7 @@ public: virtual std::string get_pattern_regex(uint64_t line_number) const { return ""; - }; + } struct pattern_for_lines { pattern_for_lines(uint32_t pfl_line, uint32_t pfl_pat_index); @@ -494,6 +447,18 @@ public: int pattern_index_for_line(uint64_t line_number) const; + bool operator<(const log_format& rhs) const + { + return this->get_name() < rhs.get_name(); + } + + static bool name_lt(const std::shared_ptr& lhs, + const std::shared_ptr& rhs) + { + return intern_string_t::case_lt(lhs->get_name(), rhs->get_name()); + } + + std::string lf_description; uint8_t lf_mod_index{0}; bool lf_multiline{true}; date_time_scanner lf_date_time; @@ -515,9 +480,9 @@ protected: pcre_format(const char* regex) : name(regex), pcre(regex) { this->pf_timestamp_index = this->pcre.name_index("timestamp"); - }; + } - pcre_format() : name(nullptr), pcre(""){}; + pcre_format() : name(nullptr), pcre("") {} const char* name; pcrepp pcre; diff --git a/src/log_format_ext.hh b/src/log_format_ext.hh index 56311865..a5925ff8 100644 --- a/src/log_format_ext.hh +++ b/src/log_format_ext.hh @@ -42,10 +42,9 @@ class module_format; class external_log_format : public log_format { public: struct sample { - sample() : s_level(LEVEL_UNKNOWN) {} - positioned_property s_line; - log_level_t s_level; + std::string s_description; + log_level_t s_level{LEVEL_UNKNOWN}; }; struct value_def { @@ -53,7 +52,17 @@ public: value_kind_t kind, int col, log_format* format) - : vd_meta(name, kind, col, format){}; + : vd_meta(name, kind, col, format) + { + } + + void set_rewrite_src_name() + { + this->vd_rewrite_src_name = intern_string::lookup( + fmt::format(FMT_STRING("{}:{}"), + this->vd_meta.lvm_format.value()->get_name(), + this->vd_meta.lvm_name)); + } logline_value_meta vd_meta; std::string vd_collate; @@ -65,6 +74,7 @@ public: std::vector vd_action_list; std::string vd_rewriter; std::string vd_description; + intern_string_t vd_rewrite_src_name; }; struct indexed_value_def { @@ -89,7 +99,7 @@ public: struct pattern { std::string p_name; std::string p_config_path; - std::shared_ptr p_pcre; + std::shared_ptr> p_pcre; std::vector p_value_by_index; std::vector p_numeric_value_indexes; int p_timestamp_field_index{-1}; @@ -204,16 +214,16 @@ public: } return retval; - }; + } std::set get_source_path() const { return this->elf_source_path; - }; + } - enum json_log_field { - JLF_CONSTANT, - JLF_VARIABLE + enum class json_log_field { + CONSTANT, + VARIABLE }; struct json_format_element { @@ -235,81 +245,52 @@ public: CAPITALIZE, }; - json_format_element() - : jfe_type(JLF_CONSTANT), jfe_default_value("-"), jfe_min_width(0), - jfe_max_width(LLONG_MAX), jfe_align(align_t::LEFT), - jfe_overflow(overflow_t::ABBREV), - jfe_text_transform(transform_t::NONE){}; - - json_log_field jfe_type; + json_log_field jfe_type{json_log_field::CONSTANT}; positioned_property jfe_value; - std::string jfe_default_value; - long long jfe_min_width; - long long jfe_max_width; - align_t jfe_align; - overflow_t jfe_overflow; - transform_t jfe_text_transform; + std::string jfe_default_value{"-"}; + long long jfe_min_width{0}; + long long jfe_max_width{LLONG_MAX}; + align_t jfe_align{align_t::LEFT}; + overflow_t jfe_overflow{overflow_t::ABBREV}; + transform_t jfe_text_transform{transform_t::NONE}; std::string jfe_ts_format; }; struct json_field_cmp { json_field_cmp(json_log_field type, const intern_string_t name) - : jfc_type(type), jfc_field_name(name){}; + : jfc_type(type), jfc_field_name(name) + { + } bool operator()(const json_format_element& jfe) const { return (this->jfc_type == jfe.jfe_type && this->jfc_field_name == jfe.jfe_value.pp_value); - }; + } json_log_field jfc_type; const intern_string_t jfc_field_name; }; struct highlighter_def { - highlighter_def() : hd_underline(false), hd_blink(false) {} - std::shared_ptr hd_pattern; positioned_property hd_color; positioned_property hd_background_color; - bool hd_underline; - bool hd_blink; + bool hd_underline{false}; + bool hd_blink{false}; }; long value_line_count(const intern_string_t ist, bool top_level, const unsigned char* str = nullptr, - ssize_t len = -1) const - { - const auto iter = this->elf_value_defs.find(ist); - long line_count - = (str != NULL) ? std::count(&str[0], &str[len], '\n') + 1 : 1; - - if (iter == this->elf_value_defs.end()) { - return (this->jlf_hide_extra || !top_level) ? 0 : line_count; - } - - if (iter->second->vd_meta.lvm_hidden) { - return 0; - } - - if (std::find_if(this->jlf_line_format.begin(), - this->jlf_line_format.end(), - json_field_cmp(JLF_VARIABLE, ist)) - != this->jlf_line_format.end()) - { - return line_count - 1; - } - - return line_count; - }; + ssize_t len = -1) const; bool has_value_def(const intern_string_t ist) const { const auto iter = this->elf_value_defs.find(ist); return iter != this->elf_value_defs.end(); - }; + } std::string get_pattern_name(uint64_t line_number) const { @@ -330,31 +311,7 @@ public: } log_level_t convert_level(const pcre_input& pi, - const pcre_context::capture_t* level_cap) const - { - log_level_t retval = LEVEL_INFO; - - if (level_cap != nullptr && level_cap->is_valid()) { - pcre_context_static<128> pc_level; - pcre_input pi_level( - pi.get_substr_start(level_cap), 0, level_cap->length()); - - if (this->elf_level_patterns.empty()) { - retval - = string2level(pi_level.get_string(), level_cap->length()); - } else { - for (const auto& elf_level_pattern : this->elf_level_patterns) { - if (elf_level_pattern.second.lp_pcre->match(pc_level, - pi_level)) { - retval = elf_level_pattern.first; - break; - } - } - } - } - - return retval; - } + const pcre_context::capture_t* level_cap) const; using mod_map_t = std::map; static mod_map_t MODULE_FORMATS; @@ -362,7 +319,8 @@ public: GRAPH_ORDERED_FORMATS; std::set elf_source_path; - std::unordered_map elf_format_sources; + std::vector elf_format_source_order; + std::map elf_format_sources; std::list elf_collision; std::string elf_file_pattern; std::set elf_mime_types; @@ -410,52 +368,21 @@ public: } this->jlf_cached_line.resize(old_size + len); memcpy(&(this->jlf_cached_line[old_size]), value, len); - }; + } void json_append_to_cache(ssize_t len) { size_t old_size = this->jlf_cached_line.size(); this->jlf_cached_line.resize(old_size + len); memset(&this->jlf_cached_line[old_size], ' ', len); - }; + } void json_append(const json_format_element& jfe, const char* value, - ssize_t len) - { - if (len == -1) { - len = strlen(value); - } - if (jfe.jfe_align == json_format_element::align_t::RIGHT) { - if (len < jfe.jfe_min_width) { - this->json_append_to_cache(jfe.jfe_min_width - len); - } - } - this->json_append_to_cache(value, len); - if (jfe.jfe_align == json_format_element::align_t::LEFT) { - if (len < jfe.jfe_min_width) { - this->json_append_to_cache(jfe.jfe_min_width - len); - } - } - }; + ssize_t len); logline_value_meta get_value_meta(intern_string_t field_name, - value_kind_t kind) - { - auto iter = this->elf_value_defs.find(field_name); - - if (iter == this->elf_value_defs.end()) { - auto retval = logline_value_meta(field_name, kind, -1, this); - - retval.lvm_hidden = this->jlf_hide_extra; - return retval; - } - - auto lvm = iter->second->vd_meta; - - lvm.lvm_kind = kind; - return lvm; - } + value_kind_t kind); std::vector get_snippets() const; diff --git a/src/log_format_fwd.hh b/src/log_format_fwd.hh index 10961de3..6055dfef 100644 --- a/src/log_format_fwd.hh +++ b/src/log_format_fwd.hh @@ -73,7 +73,7 @@ public: ll_expr_mark(0) { memset(this->ll_schema, 0, sizeof(this->ll_schema)); - }; + } logline(file_off_t off, const struct timeval& tv, @@ -85,69 +85,48 @@ public: { this->set_time(tv); memset(this->ll_schema, 0, sizeof(this->ll_schema)); - }; + } /** @return The offset of the line in the file. */ - file_off_t get_offset() const - { - return this->ll_offset; - }; + file_off_t get_offset() const { return this->ll_offset; } - uint16_t get_sub_offset() const - { - return this->ll_sub_offset; - }; + uint16_t get_sub_offset() const { return this->ll_sub_offset; } - void set_sub_offset(uint16_t suboff) - { - this->ll_sub_offset = suboff; - }; + void set_sub_offset(uint16_t suboff) { this->ll_sub_offset = suboff; } /** @return The timestamp for the line. */ - time_t get_time() const - { - return this->ll_time; - }; + time_t get_time() const { return this->ll_time; } void to_exttm(struct exttm& tm_out) const { tm_out.et_tm = *gmtime(&this->ll_time); tm_out.et_nsec = this->ll_millis * 1000 * 1000; - }; + } - void set_time(time_t t) - { - this->ll_time = t; - }; + void set_time(time_t t) { this->ll_time = t; } /** @return The millisecond timestamp for the line. */ - uint16_t get_millis() const - { - return this->ll_millis; - }; + uint16_t get_millis() const { return this->ll_millis; } - void set_millis(uint16_t m) - { - this->ll_millis = m; - }; + void set_millis(uint16_t m) { this->ll_millis = m; } uint64_t get_time_in_millis() const { return (this->ll_time * 1000ULL + (uint64_t) this->ll_millis); - }; + } struct timeval get_timeval() const { struct timeval retval = {this->ll_time, this->ll_millis * 1000}; return retval; - }; + } void set_time(const struct timeval& tv) { this->ll_time = tv.tv_sec; this->ll_millis = tv.tv_usec / 1000; - }; + } void set_ignore(bool val) { @@ -156,7 +135,7 @@ public: } else { this->ll_level &= ~LEVEL_IGNORE; } - }; + } bool is_ignored() const { @@ -170,22 +149,13 @@ public: } else { this->ll_level &= ~LEVEL_MARK; } - }; + } - bool is_marked() const - { - return this->ll_level & LEVEL_MARK; - }; + bool is_marked() const { return this->ll_level & LEVEL_MARK; } - void set_expr_mark(bool val) - { - this->ll_expr_mark = val; - }; + void set_expr_mark(bool val) { this->ll_expr_mark = val; } - bool is_expr_marked() const - { - return this->ll_expr_mark; - }; + bool is_expr_marked() const { return this->ll_expr_mark; } void set_time_skew(bool val) { @@ -194,22 +164,13 @@ public: } else { this->ll_level &= ~LEVEL_TIME_SKEW; } - }; + } - bool is_time_skewed() const - { - return this->ll_level & LEVEL_TIME_SKEW; - }; + bool is_time_skewed() const { return this->ll_level & LEVEL_TIME_SKEW; } - void set_valid_utf(bool v) - { - this->ll_valid_utf = v; - } + void set_valid_utf(bool v) { this->ll_valid_utf = v; } - bool is_valid_utf() const - { - return this->ll_valid_utf; - } + bool is_valid_utf() const { return this->ll_valid_utf; } /** @param l The logging level. */ void set_level(log_level_t l) @@ -221,42 +182,30 @@ public: log_level_t get_level_and_flags() const { return (log_level_t) this->ll_level; - }; + } log_level_t get_msg_level() const { return (log_level_t) (this->ll_level & ~LEVEL__FLAGS); - }; + } const char* get_level_name() const { return level_names[this->ll_level & ~LEVEL__FLAGS]; - }; + } bool is_message() const { return (this->ll_level & (LEVEL_IGNORE | LEVEL_CONTINUED)) == 0; } - bool is_continued() const - { - return this->ll_level & LEVEL_CONTINUED; - }; + bool is_continued() const { return this->ll_level & LEVEL_CONTINUED; } - uint8_t get_module_id() const - { - return this->ll_module_id; - }; + uint8_t get_module_id() const { return this->ll_module_id; } - void set_opid(uint8_t opid) - { - this->ll_opid = opid; - }; + void set_opid(uint8_t opid) { this->ll_opid = opid; } - uint8_t get_opid() const - { - return this->ll_opid; - }; + uint8_t get_opid() const { return this->ll_opid; } /** * @return True if there is a schema value set for this log line. @@ -264,7 +213,7 @@ public: bool has_schema() const { return (this->ll_schema[0] != 0 || this->ll_schema[1] != 0); - }; + } /** * Set the "schema" for this log line. The schema ID is used to match log @@ -276,12 +225,9 @@ public: void set_schema(const byte_array<2, uint64_t>& ba) { memcpy(this->ll_schema, ba.in(), sizeof(this->ll_schema)); - }; + } - char get_schema() const - { - return this->ll_schema[0]; - }; + char get_schema() const { return this->ll_schema[0]; } /** * Perform a partial match of the given schema against this log line. @@ -309,26 +255,23 @@ public: || (this->ll_time == rhs.ll_time && this->ll_millis == rhs.ll_millis && this->ll_offset == rhs.ll_offset && this->ll_sub_offset < rhs.ll_sub_offset); - }; + } - bool operator<(const time_t& rhs) const - { - return this->ll_time < rhs; - }; + bool operator<(const time_t& rhs) const { return this->ll_time < rhs; } bool operator<(const struct timeval& rhs) const { return ((this->ll_time < rhs.tv_sec) || ((this->ll_time == rhs.tv_sec) && (this->ll_millis < (rhs.tv_usec / 1000)))); - }; + } bool operator<=(const struct timeval& rhs) const { return ((this->ll_time < rhs.tv_sec) || ((this->ll_time == rhs.tv_sec) && (this->ll_millis <= (rhs.tv_usec / 1000)))); - }; + } private: file_off_t ll_offset; diff --git a/src/log_format_loader.cc b/src/log_format_loader.cc index 6d82cafb..21686217 100644 --- a/src/log_format_loader.cc +++ b/src/log_format_loader.cc @@ -93,11 +93,14 @@ ensure_format(const yajlpp_provider_context& ypc, userdata* ud) formats->push_back(name); } - auto srcs_iter - = retval->elf_format_sources.find(ud->ud_format_path.string()); - if (srcs_iter == retval->elf_format_sources.end()) { - retval->elf_format_sources[ud->ud_format_path.string()] - = ud->ud_parse_context->get_line_number(); + if (!ud->ud_format_path.empty()) { + auto i_src_path = intern_string::lookup(ud->ud_format_path.string()); + auto srcs_iter = retval->elf_format_sources.find(i_src_path); + if (srcs_iter == retval->elf_format_sources.end()) { + retval->elf_format_source_order.emplace_back(ud->ud_format_path); + retval->elf_format_sources[i_src_path] + = ud->ud_parse_context->get_line_number(); + } } if (ud->ud_format_path.empty()) { @@ -170,7 +173,7 @@ line_format_provider(const yajlpp_provider_context& ypc, { auto& jfe = ensure_json_format_element(elf, ypc.ypc_index); - jfe.jfe_type = external_log_format::JLF_VARIABLE; + jfe.jfe_type = external_log_format::json_log_field::VARIABLE; return &jfe; } @@ -394,7 +397,7 @@ read_json_constant(yajlpp_parse_context* ypc, ypc->ypc_array_index.back() += 1; auto& jfe = ensure_json_format_element(elf, ypc->ypc_array_index.back()); - jfe.jfe_type = external_log_format::JLF_CONSTANT; + jfe.jfe_type = external_log_format::json_log_field::CONSTANT; jfe.jfe_default_value = val; return 1; @@ -647,6 +650,10 @@ static const json_path_handler_base::enum_value_t LEVEL_ENUM[] = { }; static struct json_path_container sample_handlers = { + yajlpp::property_handler("description") + .with_synopsis("") + .with_description("A description of this sample.") + .for_field(&external_log_format::sample::s_description), yajlpp::property_handler("line") .with_synopsis("") .with_description( @@ -800,7 +807,8 @@ struct json_path_container format_handlers = { json_path_handler("title", read_format_field) .with_description("The human-readable name for this log format"), json_path_handler("description", read_format_field) - .with_description("A longer description of this log format"), + .with_description("A longer description of this log format") + .for_field(&external_log_format::lf_description), json_path_handler("timestamp-format#", read_format_field) .with_description("An array of strptime(3)-like timestamp formats"), json_path_handler("module-field", read_format_field) @@ -983,7 +991,8 @@ load_format_file(const ghc::filesystem::path& filename, auto_fd fd; log_info("loading formats from file: %s", filename.c_str()); - yajlpp_parse_context ypc(filename, &root_format_handler); + yajlpp_parse_context ypc(intern_string::lookup(filename.string()), + &root_format_handler); ud.ud_parse_context = &ypc; ud.ud_format_path = filename; ud.ud_format_names = &retval; @@ -1082,7 +1091,8 @@ load_formats(const std::vector& extra_paths, log_debug("Loading default formats"); for (const auto& bsf : lnav_format_json) { - yajlpp_parse_context ypc_builtin(bsf.get_name(), &root_format_handler); + yajlpp_parse_context ypc_builtin(intern_string::lookup(bsf.get_name()), + &root_format_handler); handle = yajl_alloc(&ypc_builtin.ypc_callbacks, nullptr, &ypc_builtin); ud.ud_parse_context = &ypc_builtin; ud.ud_format_names = &retval; @@ -1140,13 +1150,7 @@ load_formats(const std::vector& extra_paths, } } - if (errors.empty()) { - alpha_ordered_formats.push_back(elf); - } - } - - if (!errors.empty()) { - return; + alpha_ordered_formats.push_back(elf); } auto& graph_ordered_formats = external_log_format::GRAPH_ORDERED_FORMATS; diff --git a/src/log_gutter_source.hh b/src/log_gutter_source.hh index 496f8241..738d6961 100644 --- a/src/log_gutter_source.hh +++ b/src/log_gutter_source.hh @@ -72,7 +72,7 @@ public: bar_role_out = role_t::VCR_SCROLLBAR_WARNING; } } - }; + } }; #endif diff --git a/src/log_level.hh b/src/log_level.hh index be0f1225..1c6cb32b 100644 --- a/src/log_level.hh +++ b/src/log_level.hh @@ -34,37 +34,7 @@ #include -/** - * The logging level identifiers for a line(s). - */ -enum log_level_t : int { - LEVEL_UNKNOWN, - LEVEL_TRACE, - LEVEL_DEBUG5, - LEVEL_DEBUG4, - LEVEL_DEBUG3, - LEVEL_DEBUG2, - LEVEL_DEBUG, - LEVEL_INFO, - LEVEL_STATS, - LEVEL_NOTICE, - LEVEL_WARNING, - LEVEL_ERROR, - LEVEL_CRITICAL, - LEVEL_FATAL, - LEVEL_INVALID, - - LEVEL__MAX, - - LEVEL_IGNORE = 0x10, /*< Ignore */ - LEVEL_TIME_SKEW = 0x20, /*< Received after timestamp. */ - LEVEL_MARK = 0x40, /*< Bookmarked line. */ - LEVEL_CONTINUED = 0x80, /*< Continuation of multiline entry. */ - - /** Mask of flags for the level field. */ - LEVEL__FLAGS - = (LEVEL_IGNORE | LEVEL_TIME_SKEW | LEVEL_MARK | LEVEL_CONTINUED) -}; +#include "base/log_level_enum.hh" extern const char* level_names[LEVEL__MAX + 1]; diff --git a/src/log_vtab_impl.cc b/src/log_vtab_impl.cc index 70079b27..ef343a85 100644 --- a/src/log_vtab_impl.cc +++ b/src/log_vtab_impl.cc @@ -168,6 +168,41 @@ log_vtab_impl::logline_value_to_sqlite_type(value_kind_t kind) return std::make_pair(type, subtype); } +void +log_vtab_impl::get_foreign_keys(std::vector& keys_inout) const +{ + keys_inout.emplace_back("log_line"); + keys_inout.emplace_back("min(log_line)"); + keys_inout.emplace_back("log_mark"); + keys_inout.emplace_back("log_time_msecs"); +} + +void +log_vtab_impl::extract(std::shared_ptr lf, + uint64_t line_number, + shared_buffer_ref& line, + std::vector& values) +{ + auto format = lf->get_format(); + + this->vi_attrs.clear(); + format->annotate(line_number, line, this->vi_attrs, values, false); +} + +bool +log_vtab_impl::is_valid(log_cursor& lc, logfile_sub_source& lss) +{ + content_line_t cl(lss.at(lc.lc_curr_line)); + std::shared_ptr lf = lss.find(cl); + auto lf_iter = lf->begin() + cl; + + if (!lf_iter->is_message()) { + return false; + } + + return true; +} + struct vtab { sqlite3_vtab base; sqlite3* db; @@ -984,14 +1019,12 @@ vt_update(sqlite3_vtab* tab, if (log_tags) { std::vector errors; - yajlpp_parse_context ypc( - fmt::format(FMT_STRING("{}.log_tags"), vt->vi->get_name()), - &tags_handler); + yajlpp_parse_context ypc(vt->vi->get_tags_name(), &tags_handler); auto_mem handle(yajl_free); handle = yajl_alloc(&ypc.ypc_callbacks, nullptr, &ypc); ypc.ypc_userdata = &errors; - ypc.ypc_line_number = log_vtab_data.lvd_line_number; + ypc.ypc_line_number = log_vtab_data.lvd_location.sl_line_number; ypc.with_handle(handle) .with_error_reporter([](const yajlpp_parse_context& ypc, auto msg) { @@ -1002,19 +1035,16 @@ vt_update(sqlite3_vtab* tab, .with_obj(tmp_bm); ypc.parse_doc(string_fragment{log_tags}); if (!errors.empty()) { - auto top_error - = lnav::console::user_message::error( - attr_line_t("invalid value for ") - .append_quoted("log_tags"_symbol) - .append(" column of table ") - .append_quoted(lnav::roles::symbol( - vt->vi->get_name().to_string()))) - .with_reason(errors[0].to_attr_line({})) - .with_snippet( - lnav::console::snippet::from( - log_vtab_data.lvd_source, - log_vtab_data.lvd_content) - .with_line(log_vtab_data.lvd_line_number)); + auto top_error = lnav::console::user_message::error( + attr_line_t("invalid value for ") + .append_quoted("log_tags"_symbol) + .append(" column of table ") + .append_quoted(lnav::roles::symbol( + vt->vi->get_name().to_string()))) + .with_reason(errors[0].to_attr_line({})) + .with_snippet(lnav::console::snippet::from( + log_vtab_data.lvd_location, + log_vtab_data.lvd_content)); auto json_error = lnav::to_json(top_error); tab->zErrMsg = sqlite3_mprintf("lnav-error:%s", json_error.c_str()); @@ -1185,3 +1215,33 @@ log_vtab_manager::unregister_vtab(intern_string_t name) return retval; } + +bool +log_format_vtab_impl::next(log_cursor& lc, logfile_sub_source& lss) +{ + lc.lc_curr_line = lc.lc_curr_line + vis_line_t(1); + lc.lc_sub_index = 0; + + if (lc.is_eof()) { + return true; + } + + auto cl = content_line_t(lss.at(lc.lc_curr_line)); + auto lf = lss.find(cl); + auto lf_iter = lf->begin() + cl; + uint8_t mod_id = lf_iter->get_module_id(); + + if (!lf_iter->is_message()) { + return false; + } + + auto format = lf->get_format(); + if (format->get_name() == this->lfvi_format.get_name()) { + return true; + } else if (mod_id && mod_id == this->lfvi_format.lf_mod_index) { + // XXX + return true; + } + + return false; +} diff --git a/src/log_vtab_impl.hh b/src/log_vtab_impl.hh index 61eb4698..2ed0fc77 100644 --- a/src/log_vtab_impl.hh +++ b/src/log_vtab_impl.hh @@ -107,61 +107,40 @@ public: value_kind_t kind); log_vtab_impl(const intern_string_t name) - : vi_supports_indexes(true), vi_name(name) + : vi_name(name), vi_tags_name(intern_string::lookup( + fmt::format(FMT_STRING("{}.log_tags"), name))) { this->vi_attrs.resize(128); - }; - virtual ~log_vtab_impl() = default; + } - const intern_string_t get_name() const - { - return this->vi_name; - }; + virtual ~log_vtab_impl() = default; - std::string get_table_statement(); + const intern_string_t get_name() const { return this->vi_name; } - virtual bool is_valid(log_cursor& lc, logfile_sub_source& lss) - { - content_line_t cl(lss.at(lc.lc_curr_line)); - std::shared_ptr lf = lss.find(cl); - auto lf_iter = lf->begin() + cl; + intern_string_t get_tags_name() const { return this->vi_tags_name; } - if (!lf_iter->is_message()) { - return false; - } + std::string get_table_statement(); - return true; - }; + virtual bool is_valid(log_cursor& lc, logfile_sub_source& lss); virtual bool next(log_cursor& lc, logfile_sub_source& lss) = 0; virtual void get_columns(std::vector& cols) const {}; - virtual void get_foreign_keys(std::vector& keys_inout) const - { - keys_inout.emplace_back("log_line"); - keys_inout.emplace_back("min(log_line)"); - keys_inout.emplace_back("log_mark"); - keys_inout.emplace_back("log_time_msecs"); - }; + virtual void get_foreign_keys(std::vector& keys_inout) const; virtual void extract(std::shared_ptr lf, uint64_t line_number, shared_buffer_ref& line, - std::vector& values) - { - auto format = lf->get_format(); + std::vector& values); - this->vi_attrs.clear(); - format->annotate(line_number, line, this->vi_attrs, values, false); - }; - - bool vi_supports_indexes; + bool vi_supports_indexes{true}; int vi_column_count; string_attrs_t vi_attrs; protected: const intern_string_t vi_name; + const intern_string_t vi_tags_name; }; class log_format_vtab_impl : public log_vtab_impl { @@ -171,34 +150,7 @@ public: { } - virtual bool next(log_cursor& lc, logfile_sub_source& lss) - { - lc.lc_curr_line = lc.lc_curr_line + vis_line_t(1); - lc.lc_sub_index = 0; - - if (lc.is_eof()) { - return true; - } - - auto cl = content_line_t(lss.at(lc.lc_curr_line)); - auto lf = lss.find(cl); - auto lf_iter = lf->begin() + cl; - uint8_t mod_id = lf_iter->get_module_id(); - - if (!lf_iter->is_message()) { - return false; - } - - auto format = lf->get_format(); - if (format->get_name() == this->lfvi_format.get_name()) { - return true; - } else if (mod_id && mod_id == this->lfvi_format.lf_mod_index) { - // XXX - return true; - } - - return false; - }; + virtual bool next(log_cursor& lc, logfile_sub_source& lss); protected: const log_format& lfvi_format; @@ -210,8 +162,7 @@ typedef void (*sql_progress_finished_callback_t)(); struct _log_vtab_data { sql_progress_callback_t lvd_progress; sql_progress_finished_callback_t lvd_finished; - std::string lvd_source; - int lvd_line_number{0}; + source_location lvd_location; attr_line_t lvd_content; }; @@ -221,14 +172,12 @@ class sql_progress_guard { public: sql_progress_guard(sql_progress_callback_t cb, sql_progress_finished_callback_t fcb, - const std::string& source, - int line_number, + source_location loc, const attr_line_t& content) { log_vtab_data.lvd_progress = cb; log_vtab_data.lvd_finished = fcb; - log_vtab_data.lvd_source = source; - log_vtab_data.lvd_line_number = line_number; + log_vtab_data.lvd_location = loc; log_vtab_data.lvd_content = content; } @@ -239,8 +188,7 @@ public: } log_vtab_data.lvd_progress = nullptr; log_vtab_data.lvd_finished = nullptr; - log_vtab_data.lvd_source.clear(); - log_vtab_data.lvd_line_number = 0; + log_vtab_data.lvd_location = source_location{}; log_vtab_data.lvd_content.clear(); } }; @@ -253,15 +201,9 @@ public: log_vtab_manager(sqlite3* db, textview_curses& tc, logfile_sub_source& lss); ~log_vtab_manager(); - textview_curses* get_view() const - { - return &this->vm_textview; - }; + textview_curses* get_view() const { return &this->vm_textview; } - logfile_sub_source* get_source() - { - return &this->vm_source; - }; + logfile_sub_source* get_source() { return &this->vm_source; } std::string register_vtab(std::shared_ptr vi); std::string unregister_vtab(intern_string_t name); @@ -274,17 +216,11 @@ public: return iter->second; } return nullptr; - }; + } - iterator begin() const - { - return this->vm_impls.begin(); - }; + iterator begin() const { return this->vm_impls.begin(); } - iterator end() const - { - return this->vm_impls.end(); - }; + iterator end() const { return this->vm_impls.end(); } private: sqlite3* vm_db; diff --git a/src/logfile.cc b/src/logfile.cc index 90b1b34a..1239ed5f 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -796,3 +796,49 @@ logfile::mark_as_duplicate(const std::string& name) note_type::duplicate, fmt::format(FMT_STRING("hiding duplicate of {}"), name)); } + +void +logfile::adjust_content_time(int line, const timeval& tv, bool abs_offset) +{ + struct timeval old_time = this->lf_time_offset; + + this->lf_time_offset_line = line; + if (abs_offset) { + this->lf_time_offset = tv; + } else { + timeradd(&old_time, &tv, &this->lf_time_offset); + } + for (auto& iter : *this) { + struct timeval curr, diff, new_time; + + curr = iter.get_timeval(); + timersub(&curr, &old_time, &diff); + timeradd(&diff, &this->lf_time_offset, &new_time); + iter.set_time(new_time); + } + this->lf_sort_needed = true; +} + +void +logfile::set_filename(const std::string& filename) +{ + if (this->lf_filename != filename) { + this->lf_filename = filename; + ghc::filesystem::path p(filename); + this->lf_basename = p.filename(); + } +} + +struct timeval +logfile::original_line_time(logfile::iterator ll) +{ + if (this->is_time_adjusted()) { + struct timeval line_time = ll->get_timeval(); + struct timeval retval; + + timersub(&line_time, &this->lf_time_offset, &retval); + return retval; + } + + return ll->get_timeval(); +} diff --git a/src/logfile.hh b/src/logfile.hh index b76ae9d3..88c096ae 100644 --- a/src/logfile.hh +++ b/src/logfile.hh @@ -111,10 +111,7 @@ public: ~logfile() override; - const logfile_activity& get_activity() const - { - return this->lf_activity; - }; + const logfile_activity& get_activity() const { return this->lf_activity; } nonstd::optional get_actual_path() const { @@ -122,128 +119,63 @@ public: } /** @return The filename as given in the constructor. */ - const std::string& get_filename() const - { - return this->lf_filename; - }; + const std::string& get_filename() const { return this->lf_filename; } /** @return The filename as given in the constructor, excluding the path * prefix. */ - const std::string& get_basename() const - { - return this->lf_basename; - }; + const std::string& get_basename() const { return this->lf_basename; } - int get_fd() const - { - return this->lf_line_buffer.get_fd(); - }; + int get_fd() const { return this->lf_line_buffer.get_fd(); } /** @param filename The new filename for this log file. */ - void set_filename(const std::string& filename) - { - if (this->lf_filename != filename) { - this->lf_filename = filename; - ghc::filesystem::path p(filename); - this->lf_basename = p.filename(); - } - }; + void set_filename(const std::string& filename); - const std::string& get_content_id() const - { - return this->lf_content_id; - }; + const std::string& get_content_id() const { return this->lf_content_id; } /** @return The inode for this log file. */ - const struct stat& get_stat() const - { - return this->lf_stat; - }; + const struct stat& get_stat() const { return this->lf_stat; } - size_t get_longest_line_length() const - { - return this->lf_longest_line; - } + size_t get_longest_line_length() const { return this->lf_longest_line; } - bool is_compressed() const - { - return this->lf_line_buffer.is_compressed(); - }; + bool is_compressed() const { return this->lf_line_buffer.is_compressed(); } - bool is_valid_filename() const - { - return this->lf_valid_filename; - }; + bool is_valid_filename() const { return this->lf_valid_filename; } - file_off_t get_index_size() const - { - return this->lf_index_size; - } + file_off_t get_index_size() const { return this->lf_index_size; } /** * @return The detected format, rebuild_index() must be called before this * will return a value other than NULL. */ - std::shared_ptr get_format() const - { - return this->lf_format; - }; + std::shared_ptr get_format() const { return this->lf_format; } intern_string_t get_format_name() const; - text_format_t get_text_format() const - { - return this->lf_text_format; - } + text_format_t get_text_format() const { return this->lf_text_format; } /** * @return The last modified time of the file when the file was last * indexed. */ - time_t get_modified_time() const - { - return this->lf_index_time; - }; + time_t get_modified_time() const { return this->lf_index_time; } - int get_time_offset_line() const - { - return this->lf_time_offset_line; - }; + int get_time_offset_line() const { return this->lf_time_offset_line; } const struct timeval& get_time_offset() const { return this->lf_time_offset; - }; + } void adjust_content_time(int line, const struct timeval& tv, - bool abs_offset = true) - { - struct timeval old_time = this->lf_time_offset; - - this->lf_time_offset_line = line; - if (abs_offset) { - this->lf_time_offset = tv; - } else { - timeradd(&old_time, &tv, &this->lf_time_offset); - } - for (auto& iter : *this) { - struct timeval curr, diff, new_time; - - curr = iter.get_timeval(); - timersub(&curr, &old_time, &diff); - timeradd(&diff, &this->lf_time_offset, &new_time); - iter.set_time(new_time); - } - this->lf_sort_needed = true; - }; + bool abs_offset = true); void clear_time_offset() { struct timeval tv = {0, 0}; this->adjust_content_time(-1, tv); - }; + } void mark_as_duplicate(const std::string& name); @@ -299,46 +231,20 @@ public: nonstd::optional find_from_time( const struct timeval& tv) const; - logline& operator[](int index) - { - return this->lf_index[index]; - }; + logline& operator[](int index) { return this->lf_index[index]; } - logline& front() - { - return this->lf_index.front(); - } + logline& front() { return this->lf_index.front(); } - logline& back() - { - return this->lf_index.back(); - }; + logline& back() { return this->lf_index.back(); } /** @return True if this log file still exists. */ bool exists() const; - void close() - { - this->lf_is_closed = true; - }; - - bool is_closed() const - { - return this->lf_is_closed; - }; - - struct timeval original_line_time(iterator ll) - { - if (this->is_time_adjusted()) { - struct timeval line_time = ll->get_timeval(); - struct timeval retval; + void close() { this->lf_is_closed = true; } - timersub(&line_time, &this->lf_time_offset, &retval); - return retval; - } + bool is_closed() const { return this->lf_is_closed; } - return ll->get_timeval(); - }; + struct timeval original_line_time(iterator ll); Result read_line(iterator ll); @@ -351,7 +257,7 @@ public: } return retval; - }; + } iterator message_start(iterator ll) { @@ -370,8 +276,10 @@ public: file_range get_file_range(const_iterator ll, bool include_continues = true) { - return {ll->get_offset(), - (file_ssize_t) this->line_length(ll, include_continues)}; + return { + ll->get_offset(), + (file_ssize_t) this->line_length(ll, include_continues), + }; } void read_full_message(const_iterator ll, @@ -402,14 +310,14 @@ public: void set_logfile_observer(logfile_observer* lo) { this->lf_logfile_observer = lo; - }; + } void set_logline_observer(logline_observer* llo); logline_observer* get_logline_observer() const { return this->lf_logline_observer; - }; + } bool operator<(const logfile& rhs) const { @@ -424,7 +332,7 @@ public: } return retval; - }; + } bool is_indexing() const { diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc index a067c712..5bdbd070 100644 --- a/src/logfile_sub_source.cc +++ b/src/logfile_sub_source.cc @@ -36,6 +36,7 @@ #include "base/ansi_scrubber.hh" #include "base/humanize.time.hh" +#include "base/itertools.hh" #include "base/string_util.hh" #include "command_executor.hh" #include "config.h" @@ -349,8 +350,6 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv, value_out = this->lss_token_attrs; - attrs = vc.vc_level_attrs[this->lss_token_line->get_msg_level()].first; - if ((row + 1) < (int) this->lss_filtered_index.size()) { next_line = this->find_line(this->at(vis_line_t(row + 1))); } @@ -367,6 +366,8 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv, lr.lr_start = 0; lr.lr_end = this->lss_token_value.length(); value_out.emplace_back(lr, SA_ORIGINAL_LINE.value()); + value_out.emplace_back( + lr, SA_LEVEL.value(this->lss_token_line->get_msg_level())); lr.lr_start = time_offset_end; lr.lr_end = -1; @@ -1731,15 +1732,9 @@ logfile_sub_source::reload_index_delegate() nonstd::optional> logfile_sub_source::get_sql_filter() { - auto iter - = std::find_if(this->tss_filters.begin(), - this->tss_filters.end(), - [](const auto& filt) { return filt->get_index() == 0; }); - - if (iter != this->tss_filters.end()) { - return *iter; - } - return nonstd::nullopt; + return this->tss_filters | lnav::itertools::find_if([](const auto& filt) { + return filt->get_index() == 0; + }); } void diff --git a/src/logfile_sub_source.hh b/src/logfile_sub_source.hh index 88aaabfb..4bec016d 100644 --- a/src/logfile_sub_source.hh +++ b/src/logfile_sub_source.hh @@ -65,19 +65,15 @@ class index_delegate { public: virtual ~index_delegate() = default; - virtual void index_start(logfile_sub_source& lss){ - - }; + virtual void index_start(logfile_sub_source& lss) {} virtual void index_line(logfile_sub_source& lss, logfile* lf, - logfile::iterator ll){ - - }; - - virtual void index_complete(logfile_sub_source& lss){ + logfile::iterator ll) + { + } - }; + virtual void index_complete(logfile_sub_source& lss) {} }; class pcre_filter : public text_filter { @@ -95,14 +91,14 @@ public: pcre_input pi(line.get_data(), 0, line.length()); return this->pf_pcre.match(pc, pi); - }; + } std::string to_command() const override { return (this->lf_type == text_filter::INCLUDE ? "filter-in " : "filter-out ") + this->lf_id; - }; + } protected: pcrepp pf_pcre; @@ -177,7 +173,7 @@ public: { this->lss_flags ^= F_TIME_OFFSET; this->clear_line_size_cache(); - }; + } void increase_line_context() { @@ -194,7 +190,7 @@ public: if (old_flags != this->lss_flags) { this->clear_line_size_cache(); } - }; + } bool decrease_line_context() { @@ -213,7 +209,7 @@ public: } return false; - }; + } size_t get_filename_offset() const { @@ -233,27 +229,24 @@ public: else this->lss_flags &= ~F_TIME_OFFSET; this->clear_line_size_cache(); - }; + } bool is_time_offset_enabled() const { return (bool) (this->lss_flags & F_TIME_OFFSET); - }; + } bool is_filename_enabled() const { return (bool) (this->lss_flags & F_FILENAME); - }; + } bool is_basename_enabled() const { return (bool) (this->lss_flags & F_BASENAME); - }; + } - log_level_t get_min_log_level() const - { - return this->lss_min_log_level; - }; + log_level_t get_min_log_level() const { return this->lss_min_log_level; } void set_force_rebuild() { @@ -266,14 +259,14 @@ public: this->lss_min_log_level = level; this->text_filters_changed(); } - }; + } bool get_min_log_time(struct timeval& tv_out) const { tv_out = this->lss_min_log_time; return (this->lss_min_log_time.tv_sec != 0 || this->lss_min_log_time.tv_usec != 0); - }; + } void set_min_log_time(const struct timeval& tv) { @@ -281,7 +274,7 @@ public: this->lss_min_log_time = tv; this->text_filters_changed(); } - }; + } bool get_max_log_time(struct timeval& tv_out) const { @@ -289,7 +282,7 @@ public: return (this->lss_max_log_time.tv_sec != std::numeric_limits::max() || this->lss_max_log_time.tv_usec != 0); - }; + } void set_max_log_time(struct timeval& tv) { @@ -297,7 +290,7 @@ public: this->lss_max_log_time = tv; this->text_filters_changed(); } - }; + } void clear_min_max_log_times() { @@ -312,7 +305,7 @@ public: this->lss_max_log_time.tv_usec = 0; this->text_filters_changed(); } - }; + } bool list_input_handle_key(listview_curses& lv, int ch); @@ -322,22 +315,16 @@ public: this->lss_marked_only = val; this->text_filters_changed(); } - }; + } - bool get_marked_only() - { - return this->lss_marked_only; - }; + bool get_marked_only() { return this->lss_marked_only; } - size_t text_line_count() - { - return this->lss_filtered_index.size(); - }; + size_t text_line_count() { return this->lss_filtered_index.size(); } size_t text_line_width(textview_curses& curses) { return this->lss_longest_line; - }; + } size_t file_count() const { @@ -353,10 +340,7 @@ public: return retval; } - bool empty() const - { - return this->lss_filtered_index.empty(); - }; + bool empty() const { return this->lss_filtered_index.empty(); } void text_value_for_line(textview_curses& tc, int row, @@ -404,22 +388,22 @@ public: void set_user_mark(const bookmark_type_t* bm, content_line_t cl) { this->lss_user_marks[bm].insert_once(cl); - }; + } bookmarks::type& get_user_bookmarks() { return this->lss_user_marks; - }; + } std::map& get_user_bookmark_metadata() { return this->lss_user_mark_metadata; - }; + } int get_filtered_count() const { return this->lss_index.size() - this->lss_filtered_index.size(); - }; + } int get_filtered_count_for(size_t filter_index) const { @@ -468,7 +452,7 @@ public: line = content_line_t(line % MAX_LINES_PER_FILE); return retval; - }; + } logfile* find_file_ptr(content_line_t& line) { @@ -477,7 +461,7 @@ public: line = content_line_t(line % MAX_LINES_PER_FILE); return retval; - }; + } logline* find_line(content_line_t line) const { @@ -491,7 +475,7 @@ public: } return retval; - }; + } nonstd::optional find_from_time( const struct timeval& start) const; @@ -501,12 +485,12 @@ public: struct timeval tv = {start, 0}; return this->find_from_time(tv); - }; + } nonstd::optional find_from_time(const exttm& etm) const { return this->find_from_time(etm.to_timeval()); - }; + } nonstd::optional find_from_content(content_line_t cl); @@ -516,17 +500,17 @@ public: return this->find_line(this->at(row))->get_timeval(); } return nonstd::nullopt; - }; + } nonstd::optional row_for_time(struct timeval time_bucket) { return this->find_from_time(time_bucket); - }; + } content_line_t at(vis_line_t vl) { return this->lss_index[this->lss_filtered_index[vl]]; - }; + } content_line_t at_base(vis_line_t vl) { @@ -535,7 +519,7 @@ public: } return this->at(vl); - }; + } log_accel::direction_t get_line_accel_direction(vis_line_t vl); @@ -551,28 +535,25 @@ public: ld_visible(lf->is_indexing()) { lf->set_logline_observer(&this->ld_filter_state); - }; + } - void clear() - { - this->ld_filter_state.lfo_filter_state.clear(); - }; + void clear() { this->ld_filter_state.lfo_filter_state.clear(); } void set_file(const std::shared_ptr& lf) { this->ld_filter_state.lfo_filter_state.tfs_logfile = lf; lf->set_logline_observer(&this->ld_filter_state); - }; + } std::shared_ptr get_file() const { return this->ld_filter_state.lfo_filter_state.tfs_logfile; - }; + } logfile* get_file_ptr() const { return this->ld_filter_state.lfo_filter_state.tfs_logfile.get(); - }; + } bool is_visible() const { @@ -594,25 +575,13 @@ public: using const_iterator = std::vector>::const_iterator; - iterator begin() - { - return this->lss_files.begin(); - }; + iterator begin() { return this->lss_files.begin(); } - iterator end() - { - return this->lss_files.end(); - }; + iterator end() { return this->lss_files.end(); } - const_iterator cbegin() const - { - return this->lss_files.begin(); - }; + const_iterator cbegin() const { return this->lss_files.begin(); } - const_iterator cend() const - { - return this->lss_files.end(); - }; + const_iterator cend() const { return this->lss_files.end(); } iterator find_data(content_line_t& line) { @@ -621,7 +590,7 @@ public: line = content_line_t(line % MAX_LINES_PER_FILE); return retval; - }; + } iterator find_data(content_line_t line, uint64_t& offset_out) { @@ -630,7 +599,7 @@ public: offset_out = line % MAX_LINES_PER_FILE; return retval; - }; + } nonstd::optional find_data( const std::shared_ptr& lf) @@ -659,7 +628,7 @@ public: ssize_t index = std::distance(this->begin(), iter); return content_line_t(index * MAX_LINES_PER_FILE); - }; + } void set_index_delegate(index_delegate* id) { @@ -667,12 +636,12 @@ public: this->lss_index_delegate = id; this->reload_index_delegate(); } - }; + } index_delegate* get_index_delegate() const { return this->lss_index_delegate; - }; + } void reload_index_delegate(); @@ -712,7 +681,7 @@ public: nonstd::optional get_location_history() { return &this->lss_location_history; - }; + } Result eval_sql_filter(sqlite3_stmt* stmt, iterator ld, @@ -757,19 +726,14 @@ private: }; struct __attribute__((__packed__)) indexed_content { - indexed_content(){ - - }; - - indexed_content(content_line_t cl) - : ic_value(cl){ + indexed_content() {} - }; + indexed_content(content_line_t cl) : ic_value(cl) {} operator content_line_t() const { return content_line_t(this->ic_value); - }; + } uint64_t ic_value : 40; }; @@ -783,7 +747,7 @@ private: logline* ll_rhs = this->llss_controller.find_line(rhs); return (*ll_lhs) < (*ll_rhs); - }; + } bool operator()(const uint32_t& lhs, const uint32_t& rhs) const { @@ -795,7 +759,7 @@ private: logline* ll_rhs = this->llss_controller.find_line(cl_rhs); return (*ll_lhs) < (*ll_rhs); - }; + } #if 0 bool operator()(const indexed_content &lhs, const indexed_content &rhs) { @@ -803,21 +767,23 @@ private: logline *ll_rhs = this->llss_controller.find_line(rhs.ic_value); return (*ll_lhs) < (*ll_rhs); - }; + } #endif + bool operator()(const content_line_t& lhs, const time_t& rhs) const { logline* ll_lhs = this->llss_controller.find_line(lhs); return *ll_lhs < rhs; - }; + } + bool operator()(const content_line_t& lhs, const struct timeval& rhs) const { logline* ll_lhs = this->llss_controller.find_line(lhs); return *ll_lhs < rhs; - }; + } logfile_sub_source& llss_controller; }; @@ -835,7 +801,7 @@ private: logline* ll_rhs = this->llss_controller.find_line(cl_rhs); return (*ll_lhs) < (*ll_rhs); - }; + } bool operator()(const uint32_t& lhs, const struct timeval& rhs) const { @@ -844,7 +810,7 @@ private: logline* ll_lhs = this->llss_controller.find_line(cl_lhs); return (*ll_lhs) < rhs; - }; + } const logfile_sub_source& llss_controller; }; diff --git a/src/papertrail_proc.hh b/src/papertrail_proc.hh index 184f97f6..c973a780 100644 --- a/src/papertrail_proc.hh +++ b/src/papertrail_proc.hh @@ -55,11 +55,8 @@ class papertrail_proc : public curl_request { public: papertrail_proc(const std::string& search, time_t min_time, time_t max_time) : curl_request("papertrailapp.com"), - ptp_jcontext(this->cr_name, &FORMAT_HANDLERS), ptp_jhandle(yajl_free), - ptp_gen(yajl_gen_free), ptp_search(search), - ptp_quoted_search(curl_free), ptp_header_list(curl_slist_free_all), - ptp_partial_read(false), ptp_min_time(min_time), - ptp_max_time(max_time) + ptp_jcontext(intern_string::lookup(this->cr_name), &FORMAT_HANDLERS), + ptp_search(search), ptp_min_time(min_time), ptp_max_time(max_time) { char piper_tmpname[PATH_MAX]; const char* tmpdir; @@ -111,14 +108,11 @@ public: this->cr_handle, CURLOPT_HTTPHEADER, this->ptp_header_list.in()); this->set_url(); - }; + } ~papertrail_proc() {} - auto_fd copy_fd() const - { - return this->ptp_fd.dup(); - }; + auto_fd copy_fd() const { return this->ptp_fd.dup(); } long complete(CURLcode result); @@ -149,7 +143,7 @@ public: base_url, this->ptp_quoted_search.in())); curl_easy_setopt(this->cr_handle, CURLOPT_URL, this->ptp_url.in()); - }; + } static size_t write_cb(void* contents, size_t size, @@ -164,17 +158,17 @@ public: static const char* PT_SEARCH_URL; yajlpp_parse_context ptp_jcontext; - auto_mem ptp_jhandle; - auto_mem ptp_gen; + auto_mem ptp_jhandle{yajl_free}; + auto_mem ptp_gen{yajl_gen_free}; const char* ptp_api_key; const std::string ptp_search; - auto_mem ptp_quoted_search; + auto_mem ptp_quoted_search{curl_free}; auto_mem ptp_url; auto_mem ptp_token_header; - auto_mem ptp_header_list; + auto_mem ptp_header_list{curl_slist_free_all}; auto_fd ptp_fd; std::string ptp_last_max_id; - bool ptp_partial_read; + bool ptp_partial_read{false}; std::string ptp_error; time_t ptp_min_time; time_t ptp_max_time; diff --git a/src/pcrepp/pcrepp.cc b/src/pcrepp/pcrepp.cc index 5e8f8bc1..bbd4e1a5 100644 --- a/src/pcrepp/pcrepp.cc +++ b/src/pcrepp/pcrepp.cc @@ -38,9 +38,7 @@ pcre_context::capture_t* pcre_context::operator[](const char* name) const { capture_t* retval = nullptr; - int index; - - index = this->pc_pcre->name_index(name); + auto index = this->pc_pcre->name_index(name); if (index != PCRE_ERROR_NOSUBSTRING) { retval = &this->pc_captures[index + 1]; } @@ -48,6 +46,18 @@ pcre_context::operator[](const char* name) const return retval; } +pcre_context::capture_t* +pcre_context::first_valid() const +{ + for (int lpc = 1; lpc < this->pc_count; lpc++) { + if (this->pc_captures[lpc].is_valid()) { + return &this->pc_captures[lpc]; + } + } + + return nullptr; +} + std::string pcrepp::quote(const char* unquoted) { @@ -85,7 +95,7 @@ void pcrepp::find_captures(const char* pattern) { bool in_class = false, in_escape = false, in_literal = false; - std::vector cap_in_progress; + std::vector cap_in_progress; for (int lpc = 0; pattern[lpc]; lpc++) { if (in_escape) { @@ -118,7 +128,7 @@ pcrepp::find_captures(const char* pattern) break; case ')': { if (!cap_in_progress.empty()) { - pcre_context::capture& cap = cap_in_progress.back(); + auto& cap = cap_in_progress.back(); char first = '\0', second = '\0', third = '\0'; bool is_cap = false; @@ -342,6 +352,60 @@ pcrepp::jit_stack() return retval; } +size_t +pcrepp::match_partial(pcre_input& pi) const +{ + size_t length = pi.pi_length; + int rc; + + do { + rc = pcre_exec(this->p_code, + this->p_code_extra.in(), + pi.get_string(), + length, + pi.pi_offset, + PCRE_PARTIAL, + nullptr, + 0); + switch (rc) { + case 0: + case PCRE_ERROR_PARTIAL: + return length; + } + if (length > 0) { + length -= 1; + } + } while (length > 0); + + return length; +} + +const char* +pcrepp::name_for_capture(int index) const +{ + for (pcre_named_capture::iterator iter = this->named_begin(); + iter != this->named_end(); + ++iter) + { + if (iter->index() == index) { + return iter->pnc_name; + } + } + return ""; +} + +int +pcrepp::name_index(const char* name) const +{ + int retval = pcre_get_stringnumber(this->p_code, name); + + if (retval == PCRE_ERROR_NOSUBSTRING) { + return retval; + } + + return retval - 1; +} + #else # warning "pcrejit is not available, search performance will be degraded" @@ -351,3 +415,11 @@ pcrepp::pcre_free_study(pcre_extra* extra) free(extra); } #endif + +void +pcre_context::capture_t::ltrim(const char* str) +{ + while (this->c_begin < this->c_end && isspace(str[this->c_begin])) { + this->c_begin += 1; + } +} diff --git a/src/pcrepp/pcrepp.hh b/src/pcrepp/pcrepp.hh index b99a7cc0..ce31f8a7 100644 --- a/src/pcrepp/pcrepp.hh +++ b/src/pcrepp/pcrepp.hh @@ -72,88 +72,53 @@ class pcrepp; */ class pcre_context { public: - typedef struct capture { - capture(){ - /* We don't initialize anything since it's a perf hit. */ - }; + struct capture_t { + capture_t() + { /* We don't initialize anything since it's a perf hit. */ + } - capture(int begin, int end) : c_begin(begin), c_end(end) + capture_t(int begin, int end) : c_begin(begin), c_end(end) { assert(begin <= end); - }; + } int c_begin; int c_end; - void ltrim(const char* str) - { - while (this->c_begin < this->c_end && isspace(str[this->c_begin])) { - this->c_begin += 1; - } - }; + void ltrim(const char* str); bool contains(int pos) const { return this->c_begin <= pos && pos < this->c_end; - }; + } - bool is_valid() const - { - return this->c_begin != -1; - }; + bool is_valid() const { return this->c_begin != -1; } - int length() const - { - return this->c_end - this->c_begin; - }; + int length() const { return this->c_end - this->c_begin; } - bool empty() const - { - return this->c_begin == this->c_end; - }; - } capture_t; - typedef capture_t* iterator; - typedef const capture_t* const_iterator; + bool empty() const { return this->c_begin == this->c_end; } + }; + using iterator = capture_t*; + using const_iterator = const capture_t*; /** @return The maximum number of strings this context can capture. */ - int get_max_count() const - { - return this->pc_max_count; - }; + int get_max_count() const { return this->pc_max_count; } - void set_count(int count) - { - this->pc_count = count; - }; + void set_count(int count) { this->pc_count = count; } - int get_count() const - { - return this->pc_count; - }; + int get_count() const { return this->pc_count; } - void set_pcrepp(const pcrepp* src) - { - this->pc_pcre = src; - }; + void set_pcrepp(const pcrepp* src) { this->pc_pcre = src; } /** * @return a capture_t that covers all of the text that was matched. */ - capture_t* all() const - { - return pc_captures; - }; + capture_t* all() const { return pc_captures; } /** @return An iterator to the first capture. */ - iterator begin() - { - return pc_captures + 1; - }; + iterator begin() { return pc_captures + 1; } /** @return An iterator that refers to the end of the capture array. */ - iterator end() - { - return pc_captures + pc_count; - }; + iterator end() { return pc_captures + pc_count; }; capture_t* operator[](int offset) const { @@ -161,39 +126,31 @@ public: return nullptr; } return &this->pc_captures[offset + 1]; - }; + } capture_t* operator[](const char* name) const; capture_t* operator[](const std::string& name) const { return (*this)[name.c_str()]; - }; - - capture_t* first_valid() const - { - for (int lpc = 1; lpc < this->pc_count; lpc++) { - if (this->pc_captures[lpc].is_valid()) { - return &this->pc_captures[lpc]; - } - } + } - return nullptr; - }; + capture_t* first_valid() const; protected: pcre_context(capture_t* captures, int max_count) - : pc_pcre(nullptr), pc_captures(captures), pc_max_count(max_count), - pc_count(0){}; + : pc_captures(captures), pc_max_count(max_count) + { + } - const pcrepp* pc_pcre; + const pcrepp* pc_pcre{nullptr}; capture_t* pc_captures; int pc_max_count; - int pc_count; + int pc_count{0}; }; struct capture_if_not { - capture_if_not(int begin) : cin_begin(begin){}; + capture_if_not(int begin) : cin_begin(begin) {} bool operator()(const pcre_context::capture_t& cap) const { @@ -203,19 +160,6 @@ struct capture_if_not { int cin_begin; }; -inline pcre_context::iterator -skip_invalid_captures(pcre_context::iterator iter, - pcre_context::iterator pc_end) -{ - for (; iter != pc_end; ++iter) { - if (iter->c_begin == -1) { - continue; - } - } - - return iter; -} - /** * A pcre_context that allocates storage for the capture array within the object * itself. @@ -241,7 +185,7 @@ public: if (this->pi_length == (size_t) -1) { this->pi_length = strlen(str); } - }; + } pcre_input(const string_fragment& s) : pi_offset(0), pi_next_offset(0), pi_length(s.length()), @@ -259,20 +203,17 @@ public: pcre_input(const std::string&&, size_t off = 0) = delete; - const char* get_string() const - { - return this->pi_string; - }; + const char* get_string() const { return this->pi_string; } const char* get_substr_start(pcre_context::const_iterator iter) const { return &this->pi_string[iter->c_begin]; - }; + } size_t get_substr_len(pcre_context::const_iterator iter) const { return iter->length(); - }; + } std::string get_substr(pcre_context::const_iterator iter) const { @@ -280,13 +221,13 @@ public: return ""; } return std::string(&this->pi_string[iter->c_begin], iter->length()); - }; + } intern_string_t get_substr_i(pcre_context::const_iterator iter) const { return intern_string::lookup(&this->pi_string[iter->c_begin], iter->length()); - }; + } nonstd::optional get_substr_opt( pcre_context::const_iterator iter) const @@ -302,12 +243,9 @@ public: { memcpy(dst, &this->pi_string[iter->c_begin], iter->length()); dst[iter->length()] = '\0'; - }; + } - void reset_next_offset() - { - this->pi_next_offset = this->pi_offset; - }; + void reset_next_offset() { this->pi_next_offset = this->pi_offset; } void reset(const char* str, size_t off = 0, size_t len = -1) { @@ -324,7 +262,7 @@ public: void reset(const std::string& str, size_t off = 0) { this->reset(str.c_str(), off, str.length()); - }; + } size_t pi_offset; size_t pi_next_offset; @@ -345,17 +283,17 @@ struct pcre_named_capture { const pcre_named_capture& operator*() const { return *this->i_named_capture; - }; + } const pcre_named_capture* operator->() const { return this->i_named_capture; - }; + } bool operator!=(const iterator& rhs) const { return this->i_named_capture != rhs.i_named_capture; - }; + } iterator& operator++() { @@ -364,7 +302,7 @@ struct pcre_named_capture { ptr += this->i_name_len; this->i_named_capture = (pcre_named_capture*) ptr; return *this; - }; + } private: pcre_named_capture* i_named_capture; @@ -374,7 +312,7 @@ struct pcre_named_capture { int index() const { return (this->pnc_index_msb << 8 | this->pnc_index_lsb) - 1; - }; + } char pnc_index_msb; char pnc_index_lsb; @@ -389,13 +327,13 @@ struct pcre_extractor { intern_string_t get_substr_i(T name) const { return this->pe_input.get_substr_i(this->pe_context[name]); - }; + } template std::string get_substr(T name) const { return this->pe_input.get_substr(this->pe_context[name]); - }; + } }; class pcrepp { @@ -422,8 +360,8 @@ public: } struct compile_error { - const char* ce_msg; - int ce_offset; + const char* ce_msg{nullptr}; + int ce_offset{0}; }; static Result from_str(std::string pattern, @@ -433,7 +371,7 @@ public: { pcre_refcount(this->p_code, 1); this->study(); - }; + } pcrepp(std::string pattern, pcre* code) : p_code(code), p_pattern(std::move(pattern)), @@ -442,7 +380,7 @@ public: pcre_refcount(this->p_code, 1); this->study(); this->find_captures(this->p_pattern.c_str()); - }; + } explicit pcrepp(const char* pattern, int options = 0) : p_pattern(pattern), p_code_extra(pcre_free_study) @@ -460,7 +398,7 @@ public: pcre_refcount(this->p_code, 1); this->study(); this->find_captures(pattern); - }; + } explicit pcrepp(const std::string& pattern, int options = 0) : p_pattern(pattern), p_code_extra(pcre_free_study) @@ -478,7 +416,7 @@ public: pcre_refcount(this->p_code, 1); this->study(); this->find_captures(pattern.c_str()); - }; + } pcrepp() {} @@ -488,7 +426,7 @@ public: { pcre_refcount(this->p_code, 1); this->study(); - }; + } pcrepp(pcrepp&& other) : p_code(other.p_code), p_pattern(std::move(other.p_pattern)), @@ -501,10 +439,7 @@ public: this->p_code_extra = std::move(other.p_code_extra); } - virtual ~pcrepp() - { - this->clear(); - }; + virtual ~pcrepp() { this->clear(); } pcrepp& operator=(pcrepp&& other) noexcept { @@ -555,7 +490,7 @@ public: pcre_named_capture::iterator named_begin() const { return {this->p_named_entries, static_cast(this->p_name_len)}; - }; + } pcre_named_capture::iterator named_end() const { @@ -564,56 +499,33 @@ public: ptr += this->p_named_count * this->p_name_len; return {(pcre_named_capture*) ptr, static_cast(this->p_name_len)}; - }; + } - const std::vector& captures() const + const std::vector& captures() const { return this->p_captures; - }; + } - std::vector::const_iterator cap_begin() const + std::vector::const_iterator cap_begin() const { return this->p_captures.begin(); - }; + } - std::vector::const_iterator cap_end() const + std::vector::const_iterator cap_end() const { return this->p_captures.end(); - }; + } int name_index(const std::string& name) const { return this->name_index(name.c_str()); - }; - - int name_index(const char* name) const - { - int retval = pcre_get_stringnumber(this->p_code, name); - - if (retval == PCRE_ERROR_NOSUBSTRING) { - return retval; - } + } - return retval - 1; - }; + int name_index(const char* name) const; - const char* name_for_capture(int index) const - { - for (pcre_named_capture::iterator iter = this->named_begin(); - iter != this->named_end(); - ++iter) - { - if (iter->index() == index) { - return iter->pnc_name; - } - } - return ""; - }; + const char* name_for_capture(int index) const; - int get_capture_count() const - { - return this->p_capture_count; - }; + int get_capture_count() const { return this->p_capture_count; } bool match(pcre_context& pc, pcre_input& pi, int options = 0) const; @@ -633,30 +545,7 @@ public: std::string replace(const char* str, const char* repl) const; - size_t match_partial(pcre_input& pi) const - { - size_t length = pi.pi_length; - int rc; - - do { - rc = pcre_exec(this->p_code, - this->p_code_extra.in(), - pi.get_string(), - length, - pi.pi_offset, - PCRE_PARTIAL, - nullptr, - 0); - switch (rc) { - case 0: - case PCRE_ERROR_PARTIAL: - return length; - } - length -= 1; - } while (length > 0); - - return length; - }; + size_t match_partial(pcre_input& pi) const; // #undef PCRE_STUDY_JIT_COMPILE #ifdef PCRE_STUDY_JIT_COMPILE @@ -678,7 +567,16 @@ public: int p_name_len{0}; unsigned long p_options{0}; pcre_named_capture* p_named_entries{nullptr}; - std::vector p_captures; + std::vector p_captures; +}; + +template +class pcrepp_with_options : public pcrepp { +public: + template + pcrepp_with_options(Args... args) : pcrepp(args..., options) + { + } }; #endif diff --git a/src/pcrepp/test_pcrepp.cc b/src/pcrepp/test_pcrepp.cc index 874ac334..c64d84d2 100644 --- a/src/pcrepp/test_pcrepp.cc +++ b/src/pcrepp/test_pcrepp.cc @@ -98,18 +98,20 @@ main(int argc, char* argv[]) } { - pcre_context::capture cap(1, 4); + pcre_context::capture_t cap(1, 4); pcre_input pi("\0foo", 0, 4); assert("foo" == pi.get_substr(&cap)); } - const char* empty_cap_regexes[] = {"foo (?:bar)", - "foo [(]", - "foo \\Q(bar)\\E", - "(?i)", + const char* empty_cap_regexes[] = { + "foo (?:bar)", + "foo [(]", + "foo \\Q(bar)\\E", + "(?i)", - NULL}; + nullptr, + }; for (int lpc = 0; empty_cap_regexes[lpc]; lpc++) { pcrepp re(empty_cap_regexes[lpc]); diff --git a/src/piper_proc.hh b/src/piper_proc.hh index 4c352fea..0caf29ea 100644 --- a/src/piper_proc.hh +++ b/src/piper_proc.hh @@ -48,7 +48,7 @@ class piper_proc { public: class error : public std::exception { public: - error(int err) : e_err(err){}; + error(int err) : e_err(err) {} int e_err; }; @@ -72,15 +72,9 @@ public: virtual ~piper_proc(); /** @return The file descriptor for the temporary file. */ - auto_fd get_fd() - { - return this->pp_fd.dup(); - }; + auto_fd get_fd() { return this->pp_fd.dup(); } - pid_t get_child_pid() const - { - return this->pp_child; - }; + pid_t get_child_pid() const { return this->pp_child; } private: /** A file descriptor that refers to the temporary file. */ diff --git a/src/readline_callbacks.cc b/src/readline_callbacks.cc index 3b7eaea6..852fb6e4 100644 --- a/src/readline_callbacks.cc +++ b/src/readline_callbacks.cc @@ -121,12 +121,12 @@ void rl_set_help() { switch (lnav_data.ld_mode) { - case LNM_SEARCH: { + case ln_mode_t::SEARCH: { lnav_data.ld_doc_source.replace_with(RE_HELP); lnav_data.ld_example_source.replace_with(RE_EXAMPLE); break; } - case LNM_SQL: { + case ln_mode_t::SQL: { textview_curses& log_view = lnav_data.ld_views[LNV_LOG]; auto* lss = (logfile_sub_source*) log_view.get_sub_source(); attr_line_t example_al; @@ -250,7 +250,7 @@ rl_change(readline_curses* rc) .clear(); switch (lnav_data.ld_mode) { - case LNM_COMMAND: { + case ln_mode_t::COMMAND: { static std::string last_command; static int generation = 0; @@ -302,7 +302,8 @@ rl_change(readline_curses* rc) lnav_data.ld_bottom_source.set_prompt(LNAV_CMD_PROMPT); lnav_data.ld_bottom_source.grep_error(""); } else if (args[0] == "config" && args.size() > 1) { - yajlpp_parse_context ypc("input", &lnav_config_handlers); + static const auto INPUT_SRC = intern_string::lookup("input"); + yajlpp_parse_context ypc(INPUT_SRC, &lnav_config_handlers); ypc.set_path(args[1]).with_obj(lnav_config); ypc.update_callbacks(); @@ -363,7 +364,7 @@ rl_change(readline_curses* rc) } break; } - case LNM_EXEC: { + case ln_mode_t::EXEC: { const auto line = rc->get_line_buffer(); size_t name_end = line.find(' '); const auto script_name = line.substr(0, name_end); @@ -406,18 +407,18 @@ rl_search_internal(readline_curses* rc, ln_mode_t mode, bool complete = false) lnav_data.ld_user_message_source.clear(); switch (mode) { - case LNM_SEARCH: - case LNM_SEARCH_FILTERS: - case LNM_SEARCH_FILES: + case ln_mode_t::SEARCH: + case ln_mode_t::SEARCH_FILTERS: + case ln_mode_t::SEARCH_FILES: name = "$search"; break; - case LNM_CAPTURE: + case ln_mode_t::CAPTURE: require(0); name = "$capture"; break; - case LNM_COMMAND: { + case ln_mode_t::COMMAND: { lnav_data.ld_exec_context.ec_dry_run = true; lnav_data.ld_preview_generation += 1; @@ -425,8 +426,8 @@ rl_search_internal(readline_curses* rc, ln_mode_t mode, bool complete = false) .set_cylon(false) .clear(); lnav_data.ld_preview_source.clear(); - auto result - = execute_command(lnav_data.ld_exec_context, rc->get_value()); + auto result = execute_command(lnav_data.ld_exec_context, + rc->get_value().get_string()); if (result.isOk()) { auto msg = result.unwrap(); @@ -450,8 +451,8 @@ rl_search_internal(readline_curses* rc, ln_mode_t mode, bool complete = false) return; } - case LNM_SQL: { - term_val = trim(rc->get_value() + ";"); + case ln_mode_t::SQL: { + term_val = trim(rc->get_value().get_string() + ";"); if (!term_val.empty() && term_val[0] == '.') { lnav_data.ld_bottom_source.grep_error(""); @@ -462,11 +463,12 @@ rl_search_internal(readline_curses* rc, ln_mode_t mode, bool complete = false) auto_mem stmt(sqlite3_finalize); int retcode; - retcode = sqlite3_prepare_v2(lnav_data.ld_db, - rc->get_value().c_str(), - -1, - stmt.out(), - nullptr); + retcode + = sqlite3_prepare_v2(lnav_data.ld_db, + rc->get_value().get_string().c_str(), + -1, + stmt.out(), + nullptr); if (retcode != SQLITE_OK) { const char* errmsg = sqlite3_errmsg(lnav_data.ld_db); @@ -484,18 +486,18 @@ rl_search_internal(readline_curses* rc, ln_mode_t mode, bool complete = false) return; } - case LNM_PAGING: - case LNM_FILTER: - case LNM_FILES: - case LNM_EXEC: - case LNM_USER: + case ln_mode_t::PAGING: + case ln_mode_t::FILTER: + case ln_mode_t::FILES: + case ln_mode_t::EXEC: + case ln_mode_t::USER: return; } if (!complete) { tc->set_top(lnav_data.ld_search_start_line); } - tc->execute_search(rc->get_value()); + tc->execute_search(rc->get_value().get_string()); } void @@ -529,18 +531,18 @@ lnav_rl_abort(readline_curses* rc) lnav_data.ld_bottom_source.grep_error(""); switch (lnav_data.ld_mode) { - case LNM_SEARCH: + case ln_mode_t::SEARCH: tc->set_top(lnav_data.ld_search_start_line); tc->revert_search(); break; - case LNM_SQL: + case ln_mode_t::SQL: tc->reload_data(); break; default: break; } lnav_data.ld_rl_view->set_value(""); - lnav_data.ld_mode = LNM_PAGING; + lnav_data.ld_mode = ln_mode_t::PAGING; } static void @@ -561,14 +563,14 @@ rl_callback_int(readline_curses* rc, bool is_alt) tc->get_highlights().erase({highlight_source_t::PREVIEW, "bodypreview"}); lnav_data.ld_log_source.set_preview_sql_filter(nullptr); - auto new_mode = LNM_PAGING; + auto new_mode = ln_mode_t::PAGING; switch (lnav_data.ld_mode) { - case LNM_SEARCH_FILTERS: - new_mode = LNM_FILTER; + case ln_mode_t::SEARCH_FILTERS: + new_mode = ln_mode_t::FILTER; break; - case LNM_SEARCH_FILES: - new_mode = LNM_FILES; + case ln_mode_t::SEARCH_FILES: + new_mode = ln_mode_t::FILES; break; default: break; @@ -576,17 +578,17 @@ rl_callback_int(readline_curses* rc, bool is_alt) auto old_mode = std::exchange(lnav_data.ld_mode, new_mode); switch (old_mode) { - case LNM_PAGING: - case LNM_FILTER: - case LNM_FILES: + case ln_mode_t::PAGING: + case ln_mode_t::FILTER: + case ln_mode_t::FILES: require(0); break; - case LNM_COMMAND: { + case ln_mode_t::COMMAND: { rc->set_alt_value(""); ec.ec_source.top().s_content - = fmt::format(FMT_STRING(":{}"), rc->get_value()); - auto exec_res = execute_command(ec, rc->get_value()); + = fmt::format(FMT_STRING(":{}"), rc->get_value().get_string()); + auto exec_res = execute_command(ec, rc->get_value().get_string()); if (exec_res.isOk()) { rc->set_value(exec_res.unwrap()); } else { @@ -607,16 +609,16 @@ rl_callback_int(readline_curses* rc, bool is_alt) break; } - case LNM_USER: + case ln_mode_t::USER: rc->set_alt_value(""); - ec.ec_local_vars.top()["value"] = rc->get_value(); + ec.ec_local_vars.top()["value"] = rc->get_value().get_string(); rc->set_value(""); break; - case LNM_SEARCH: - case LNM_SEARCH_FILTERS: - case LNM_SEARCH_FILES: - case LNM_CAPTURE: + case ln_mode_t::SEARCH: + case ln_mode_t::SEARCH_FILTERS: + case ln_mode_t::SEARCH_FILES: + case ln_mode_t::CAPTURE: rl_search_internal(rc, old_mode, true); if (!rc->get_value().empty()) { auto_mem pfile(pclose); @@ -627,7 +629,8 @@ rl_callback_int(readline_curses* rc, bool is_alt) pfile = sysclip::open(sysclip::type_t::FIND); if (pfile.in() != nullptr) { - fprintf(pfile, "%s", rc->get_value().c_str()); + fmt::print( + pfile, FMT_STRING("{}"), rc->get_value().get_string()); } if (vl != -1_vl) { tc->set_top(vl); @@ -660,15 +663,17 @@ rl_callback_int(readline_curses* rc, bool is_alt) return true; }); } - rc->set_value("search: " + rc->get_value()); + rc->set_attr_value( + attr_line_t("search: ").append(rc->get_value())); rc->set_alt_value(HELP_MSG_2( n, N, "to move forward/backward through search results")); } break; - case LNM_SQL: { + case ln_mode_t::SQL: { ec.ec_source.top().s_content = rc->get_value(); - auto result = execute_sql(ec, rc->get_value(), alt_msg); + auto result + = execute_sql(ec, rc->get_value().get_string(), alt_msg); db_label_source& dls = lnav_data.ld_db_row_source; std::string prompt; @@ -696,7 +701,7 @@ rl_callback_int(readline_curses* rc, bool is_alt) break; } - case LNM_EXEC: { + case ln_mode_t::EXEC: { auto_mem tmpout(fclose); tmpout = std::tmpfile(); @@ -715,7 +720,7 @@ rl_callback_int(readline_curses* rc, bool is_alt) exec_context::output_guard og( ec, "tmp", std::make_pair(tmpout.release(), fclose)); - auto result = execute_file(ec, path_and_args) + auto result = execute_file(ec, path_and_args.get_string()) .map(ok_prefix) .orElse(err_to_ok) .unwrap(); @@ -736,7 +741,7 @@ rl_callback_int(readline_curses* rc, bool is_alt) snprintf(desc, sizeof(desc), "Output of %s (%s)", - path_and_args.c_str(), + path_and_args.get_string().c_str(), timestamp); lnav_data.ld_active_files.fc_file_names[desc] .with_fd(std::move(fd_copy)) @@ -839,3 +844,22 @@ rl_completion_request(readline_curses* rc) } }); } + +void +rl_focus(readline_curses* rc) +{ + auto fos = (field_overlay_source*) lnav_data.ld_views[LNV_LOG] + .get_overlay_source(); + + fos->fos_contexts.emplace("", false, true); +} + +void +rl_blur(readline_curses* rc) +{ + auto fos = (field_overlay_source*) lnav_data.ld_views[LNV_LOG] + .get_overlay_source(); + + fos->fos_contexts.pop(); + lnav_data.ld_preview_generation += 1; +} diff --git a/src/readline_callbacks.hh b/src/readline_callbacks.hh index 7c5d046b..d89f4b18 100644 --- a/src/readline_callbacks.hh +++ b/src/readline_callbacks.hh @@ -39,6 +39,8 @@ void rl_alt_callback(readline_curses* rc); void rl_display_matches(readline_curses* rc); void rl_display_next(readline_curses* rc); void rl_completion_request(readline_curses* rc); +void rl_focus(readline_curses* rc); +void rl_blur(readline_curses* rc); extern const char* RE_HELP; extern const char* RE_EXAMPLE; diff --git a/src/readline_context.hh b/src/readline_context.hh index 86dcbf35..047cf565 100644 --- a/src/readline_context.hh +++ b/src/readline_context.hh @@ -69,10 +69,13 @@ public: help_text help = {}, prompt_func_t prompt = nullptr) noexcept : c_name(name), c_func(func), c_help(std::move(help)), - c_prompt(prompt){}; + c_prompt(prompt) + { + } - _command_t(command_func_t func) noexcept - : c_name("anon"), c_func(func){}; + _command_t(command_func_t func) noexcept : c_name("anon"), c_func(func) + { + } } command_t; typedef std::map command_map_t; @@ -80,10 +83,7 @@ public: command_map_t* commands = nullptr, bool case_sensitive = true); - const std::string& get_name() const - { - return this->rc_name; - }; + const std::string& get_name() const { return this->rc_name; } void load(); @@ -92,29 +92,26 @@ public: void add_possibility(const std::string& type, const std::string& value) { this->rc_possibilities[type].insert(value); - }; + } void rem_possibility(const std::string& type, const std::string& value) { this->rc_possibilities[type].erase(value); - }; + } void clear_possibilities(const std::string& type) { this->rc_possibilities[type].clear(); - }; + } - bool is_case_sensitive() const - { - return this->rc_case_sensitive; - }; + bool is_case_sensitive() const { return this->rc_case_sensitive; } readline_context& set_append_character(int ch) { this->rc_append_character = ch; return *this; - }; + } int get_append_character() const { @@ -125,26 +122,26 @@ public: { this->rc_highlighter = hl; return *this; - }; + } readline_context& set_quote_chars(const char* qc) { this->rc_quote_chars = qc; return *this; - }; + } readline_context& with_readline_var(char** var, const char* val) { this->rc_vars.emplace_back(var, val); return *this; - }; + } readline_highlighter_t get_highlighter() const { return this->rc_highlighter; - }; + } static int command_complete(int, int); diff --git a/src/readline_curses.cc b/src/readline_curses.cc index 5b3a86b7..161dfa6b 100644 --- a/src/readline_curses.cc +++ b/src/readline_curses.cc @@ -284,7 +284,7 @@ readline_context::completion_generator(const char* text_in, int state) for (const auto& poss : (*arg_possibilities)) { std::string poss_str = tolower(poss); - int score; + int score = 0; if (fts::fuzzy_match( text_str.c_str(), poss_str.c_str(), score) @@ -1106,8 +1106,6 @@ readline_curses::check_poll_set(const std::vector& pollfds) rc = recvstring( this->rc_command_pipe[RCF_MASTER], msg, sizeof(msg) - 1); if (rc >= 0) { - std::string old_value = this->rc_value; - msg[rc] = '\0'; if (this->rc_matches_remaining > 0) { this->rc_matches.emplace_back(msg); @@ -1377,7 +1375,7 @@ readline_curses::do_update() if (this->rc_active_context == -1) { int alt_start = -1; struct line_range lr(0, 0); - attr_line_t al, alt_al; + attr_line_t alt_al; view_colors& vc = view_colors::singleton(); wmove(this->vc_window, this->get_actual_y(), this->vc_left); @@ -1388,9 +1386,6 @@ readline_curses::do_update() this->rc_value.clear(); } - al.get_string() = this->rc_value; - scrub_ansi_string(al.get_string(), al.get_attrs()); - if (!this->rc_alt_value.empty()) { alt_al.get_string() = this->rc_alt_value; scrub_ansi_string(alt_al.get_string(), alt_al.get_attrs()); @@ -1398,15 +1393,18 @@ readline_curses::do_update() alt_start = getmaxx(this->vc_window) - alt_al.get_string().size(); } - if (alt_start >= (int) (al.get_string().length() + 5)) { + if (alt_start >= (int) (this->rc_value.length() + 5)) { lr.lr_end = alt_al.get_string().length(); view_curses::mvwattrline( this->vc_window, this->get_actual_y(), alt_start, alt_al, lr); } - lr.lr_end = al.get_string().length(); - view_curses::mvwattrline( - this->vc_window, this->get_actual_y(), this->vc_left, al, lr); + lr.lr_end = this->rc_value.length(); + view_curses::mvwattrline(this->vc_window, + this->get_actual_y(), + this->vc_left, + this->rc_value, + lr); this->set_x(0); } @@ -1453,3 +1451,20 @@ readline_curses::get_match_string() const return this->rc_line_buffer.substr(this->rc_match_start, len); } + +void +readline_curses::set_value(const std::string& value) +{ + this->set_attr_value(attr_line_t::from_ansi_str(value.c_str())); +} + +void +readline_curses::set_attr_value(const attr_line_t& value) +{ + this->rc_value = value; + if (this->rc_value.length() > 1024) { + this->rc_value = this->rc_value.subline(0, 1024); + } + this->rc_value_expiration = time(nullptr) + VALUE_EXPIRATION; + this->set_needs_update(); +} diff --git a/src/readline_curses.hh b/src/readline_curses.hh index 85698e90..d8a7d0ac 100644 --- a/src/readline_curses.hh +++ b/src/readline_curses.hh @@ -53,6 +53,7 @@ #include #include "base/auto_fd.hh" +#include "base/enum_util.hh" #include "base/func_util.hh" #include "base/result.h" #include "help_text_formatter.hh" @@ -75,7 +76,7 @@ public: class error : public std::exception { public: - error(int err) : e_err(err){}; + error(int err) : e_err(err) {} int e_err; }; @@ -92,15 +93,17 @@ public: this->rc_contexts[id] = &rc; } - void set_focus_action(const action& va) + template::value, bool> = true> + void add_context(T context, Args&... args) { - this->rc_focus = va; + this->add_context(lnav::enums::to_underlying(context), args...); } - void set_change_action(const action& va) - { - this->rc_change = va; - } + void set_focus_action(const action& va) { this->rc_focus = va; } + + void set_change_action(const action& va) { this->rc_change = va; } void set_perform_action(const action& va) { @@ -132,45 +135,26 @@ public: this->rc_display_next = va; } - void set_blur_action(const action& va) - { - this->rc_blur = va; - } + void set_blur_action(const action& va) { this->rc_blur = va; } void set_completion_request_action(const action& va) { this->rc_completion_request = va; } - void set_value(const std::string& value) - { - this->rc_value = value; - if (this->rc_value.length() > 1024) { - this->rc_value = this->rc_value.substr(0, 1024); - } - this->rc_value_expiration = time(nullptr) + VALUE_EXPIRATION; - this->set_needs_update(); - } + void set_value(const std::string& value); - std::string get_value() const - { - return this->rc_value; - } + void set_attr_value(const attr_line_t& al); - std::string get_line_buffer() const - { - return this->rc_line_buffer; - } + void clear_value() { this->rc_value.clear(); } - void set_alt_value(const std::string& value) - { - this->rc_alt_value = value; - } + const attr_line_t& get_value() const { return this->rc_value; } - std::string get_alt_value() const - { - return this->rc_alt_value; - } + std::string get_line_buffer() const { return this->rc_line_buffer; } + + void set_alt_value(const std::string& value) { this->rc_alt_value = value; } + + std::string get_alt_value() const { return this->rc_alt_value; } void update_poll_set(std::vector& pollfds) { @@ -187,11 +171,16 @@ public: const std::string& prompt, const std::string& initial = ""); - void set_alt_focus(bool alt_focus) + template::value, bool> = true> + void focus(T context, const Args&... args) { - this->rc_is_alt_focus = alt_focus; + this->focus(lnav::enums::to_underlying(context), args...); } + void set_alt_focus(bool alt_focus) { this->rc_is_alt_focus = alt_focus; } + void rewrite_line(int pos, const std::string& value); readline_context* get_active_context() const @@ -229,6 +218,20 @@ public: void clear_prefixes(int context); + template::value, bool> = true> + void add_prefix(T context, const Args&... args) + { + this->add_prefix(lnav::enums::to_underlying(context), args...); + } + + template::value, bool> = true> + void clear_prefixes(T context) + { + this->clear_prefixes(lnav::enums::to_underlying(context)); + } + void add_possibility(int context, const std::string& type, const std::string& value); @@ -267,16 +270,37 @@ public: const std::string& value); void clear_possibilities(int context, std::string type); - const std::vector& get_matches() const + template::value, bool> = true> + void add_possibility(T context, Args... args) { - return this->rc_matches; + this->add_possibility(lnav::enums::to_underlying(context), args...); } - int get_match_start() const + template::value, bool> = true> + void rem_possibility(T context, const Args&... args) { - return this->rc_match_start; + this->rem_possibility(lnav::enums::to_underlying(context), args...); } + template::value, bool> = true> + void clear_possibilities(T context, Args... args) + { + this->clear_possibilities(lnav::enums::to_underlying(context), args...); + } + + const std::vector& get_matches() const + { + return this->rc_matches; + } + + int get_match_start() const { return this->rc_match_start; } + std::string get_match_string() const; int get_max_match_length() const @@ -314,7 +338,7 @@ private: auto_fd rc_pty[2]; auto_fd rc_command_pipe[2]; std::map rc_contexts; - std::string rc_value; + attr_line_t rc_value; std::string rc_line_buffer; time_t rc_value_expiration{0}; std::string rc_alt_value; diff --git a/src/readline_possibilities.cc b/src/readline_possibilities.cc index e20889eb..fe94e702 100644 --- a/src/readline_possibilities.cc +++ b/src/readline_possibilities.cc @@ -51,7 +51,8 @@ static int handle_collation_list(void* ptr, int ncols, char** colvalues, char** colnames) { if (lnav_data.ld_rl_view != nullptr) { - lnav_data.ld_rl_view->add_possibility(LNM_SQL, "*", colvalues[1]); + lnav_data.ld_rl_view->add_possibility( + ln_mode_t::SQL, "*", colvalues[1]); } return 0; @@ -61,7 +62,8 @@ static int handle_db_list(void* ptr, int ncols, char** colvalues, char** colnames) { if (lnav_data.ld_rl_view != nullptr) { - lnav_data.ld_rl_view->add_possibility(LNM_SQL, "*", colvalues[1]); + lnav_data.ld_rl_view->add_possibility( + ln_mode_t::SQL, "*", colvalues[1]); } return 0; @@ -74,7 +76,8 @@ handle_table_list(void* ptr, int ncols, char** colvalues, char** colnames) std::string table_name = colvalues[0]; if (sqlite_function_help.count(table_name) == 0) { - lnav_data.ld_rl_view->add_possibility(LNM_SQL, "*", colvalues[0]); + lnav_data.ld_rl_view->add_possibility( + ln_mode_t::SQL, "*", colvalues[0]); } lnav_data.ld_table_ddl[colvalues[0]] = colvalues[1]; @@ -91,7 +94,7 @@ handle_table_info(void* ptr, int ncols, char** colvalues, char** colnames) quoted_name = sql_quote_ident(colvalues[1]); lnav_data.ld_rl_view->add_possibility( - LNM_SQL, "*", std::string(quoted_name)); + ln_mode_t::SQL, "*", std::string(quoted_name)); } if (strcmp(colvalues[5], "1") == 0) { lnav_data.ld_db_key_names.emplace_back(colvalues[1]); @@ -119,7 +122,8 @@ static void add_text_possibilities(readline_curses* rlc, int context, const std::string& type, - const std::string& str) + const std::string& str, + text_quoting tq) { static const std::regex re_escape(R"(([.\^$*+?()\[\]{}\\|]))"); static const std::regex re_escape_no_dot(R"(([\^$*+?()\[\]{}\\|]))"); @@ -142,8 +146,8 @@ add_text_possibilities(readline_curses* rlc, break; } - switch (context) { - case LNM_SQL: { + switch (tq) { + case text_quoting::sql: { auto token_value = ds.get_input().get_substr(pc.all()); auto_mem quoted_token; @@ -171,7 +175,7 @@ add_text_possibilities(readline_curses* rlc, switch (dt) { case DT_QUOTED_STRING: add_text_possibilities( - rlc, context, type, ds.get_input().get_substr(pc[0])); + rlc, context, type, ds.get_input().get_substr(pc[0]), tq); break; default: break; @@ -183,7 +187,8 @@ void add_view_text_possibilities(readline_curses* rlc, int context, const std::string& type, - textview_curses* tc) + textview_curses* tc, + text_quoting tq) { text_sub_source* tss = tc->get_sub_source(); @@ -196,7 +201,7 @@ add_view_text_possibilities(readline_curses* rlc, tss->text_value_for_line(*tc, curr_line, line, text_sub_source::RF_RAW); - add_text_possibilities(rlc, context, type, line); + add_text_possibilities(rlc, context, type, line, tq); } rlc->add_possibility(context, type, bookmark_metadata::KNOWN_TAGS); @@ -301,7 +306,7 @@ add_filter_expr_possibilities(readline_curses* rlc, } void -add_env_possibilities(int context) +add_env_possibilities(ln_mode_t context) { extern char** environ; readline_curses* rlc = lnav_data.ld_rl_view; @@ -336,15 +341,17 @@ add_filter_possibilities(textview_curses* tc) text_sub_source* tss = tc->get_sub_source(); filter_stack& fs = tss->get_filters(); - rc->clear_possibilities(LNM_COMMAND, "all-filters"); - rc->clear_possibilities(LNM_COMMAND, "disabled-filter"); - rc->clear_possibilities(LNM_COMMAND, "enabled-filter"); + rc->clear_possibilities(ln_mode_t::COMMAND, "all-filters"); + rc->clear_possibilities(ln_mode_t::COMMAND, "disabled-filter"); + rc->clear_possibilities(ln_mode_t::COMMAND, "enabled-filter"); for (const auto& tf : fs) { - rc->add_possibility(LNM_COMMAND, "all-filters", tf->get_id()); + rc->add_possibility(ln_mode_t::COMMAND, "all-filters", tf->get_id()); if (tf->is_enabled()) { - rc->add_possibility(LNM_COMMAND, "enabled-filter", tf->get_id()); + rc->add_possibility( + ln_mode_t::COMMAND, "enabled-filter", tf->get_id()); } else { - rc->add_possibility(LNM_COMMAND, "disabled-filter", tf->get_id()); + rc->add_possibility( + ln_mode_t::COMMAND, "disabled-filter", tf->get_id()); } } } @@ -356,8 +363,8 @@ add_file_possibilities() readline_curses* rc = lnav_data.ld_rl_view; - rc->clear_possibilities(LNM_COMMAND, "visible-files"); - rc->clear_possibilities(LNM_COMMAND, "hidden-files"); + rc->clear_possibilities(ln_mode_t::COMMAND, "visible-files"); + rc->clear_possibilities(ln_mode_t::COMMAND, "hidden-files"); for (const auto& lf : lnav_data.ld_active_files.fc_files) { if (lf.get() == nullptr) { continue; @@ -368,7 +375,7 @@ add_file_possibilities() = std::regex_replace(lf->get_filename(), sh_escape, R"(\\\1)"); rc->add_possibility( - LNM_COMMAND, + ln_mode_t::COMMAND, ld->is_visible() ? "visible-files" : "hidden-files", escaped_fn); }; @@ -380,7 +387,7 @@ add_mark_possibilities() { readline_curses* rc = lnav_data.ld_rl_view; - rc->clear_possibilities(LNM_COMMAND, "mark-type"); + rc->clear_possibilities(ln_mode_t::COMMAND, "mark-type"); for (auto iter = bookmark_type_t::type_begin(); iter != bookmark_type_t::type_end(); ++iter) @@ -390,7 +397,7 @@ add_mark_possibilities() if (bt->get_name().empty()) { continue; } - rc->add_possibility(LNM_COMMAND, "mark-type", bt->get_name()); + rc->add_possibility(ln_mode_t::COMMAND, "mark-type", bt->get_name()); } } @@ -404,30 +411,32 @@ add_config_possibilities() void* mem) { if (jph.jph_children) { if (!jph.jph_regex.p_named_count) { - rc->add_possibility(LNM_COMMAND, "config-option", path); + rc->add_possibility(ln_mode_t::COMMAND, "config-option", path); } for (auto named_iter = jph.jph_regex.named_begin(); named_iter != jph.jph_regex.named_end(); ++named_iter) { if (visited.count(named_iter->pnc_name) == 0) { - rc->clear_possibilities(LNM_COMMAND, named_iter->pnc_name); + rc->clear_possibilities(ln_mode_t::COMMAND, + named_iter->pnc_name); visited.insert(named_iter->pnc_name); } - rc->add_possibility(LNM_COMMAND, named_iter->pnc_name, path); + rc->add_possibility( + ln_mode_t::COMMAND, named_iter->pnc_name, path); } } else { - rc->add_possibility(LNM_COMMAND, "config-option", path); + rc->add_possibility(ln_mode_t::COMMAND, "config-option", path); if (jph.jph_synopsis) { - rc->add_prefix(LNM_COMMAND, + rc->add_prefix(ln_mode_t::COMMAND, std::vector{"config", path}, jph.jph_synopsis); } } }; - rc->clear_possibilities(LNM_COMMAND, "config-option"); + rc->clear_possibilities(ln_mode_t::COMMAND, "config-option"); for (const auto& jph : lnav_config_handlers.jpc_children) { jph.walk(cb, &lnav_config); } @@ -438,9 +447,10 @@ add_tag_possibilities() { readline_curses* rc = lnav_data.ld_rl_view; - rc->clear_possibilities(LNM_COMMAND, "tag"); - rc->clear_possibilities(LNM_COMMAND, "line-tags"); - rc->add_possibility(LNM_COMMAND, "tag", bookmark_metadata::KNOWN_TAGS); + rc->clear_possibilities(ln_mode_t::COMMAND, "tag"); + rc->clear_possibilities(ln_mode_t::COMMAND, "line-tags"); + rc->add_possibility( + ln_mode_t::COMMAND, "tag", bookmark_metadata::KNOWN_TAGS); if (lnav_data.ld_view_stack.top().value_or(nullptr) == &lnav_data.ld_views[LNV_LOG]) { @@ -452,8 +462,23 @@ add_tag_possibilities() if (meta_iter != user_meta.end()) { rc->add_possibility( - LNM_COMMAND, "line-tags", meta_iter->second.bm_tags); + ln_mode_t::COMMAND, "line-tags", meta_iter->second.bm_tags); } } } } + +void +add_recent_netlocs_possibilities() +{ + readline_curses* rc = lnav_data.ld_rl_view; + + rc->clear_possibilities(ln_mode_t::COMMAND, "recent-netlocs"); + std::set netlocs; + + isc::to().send_and_wait( + [&netlocs](auto& tlooper) { netlocs = tlooper.active_netlocs(); }); + netlocs.insert(session_data.sd_recent_netlocs.begin(), + session_data.sd_recent_netlocs.end()); + rc->add_possibility(ln_mode_t::COMMAND, "recent-netlocs", netlocs); +} diff --git a/src/readline_possibilities.hh b/src/readline_possibilities.hh index e59ce60e..df5255b7 100644 --- a/src/readline_possibilities.hh +++ b/src/readline_possibilities.hh @@ -34,15 +34,45 @@ #include "readline_curses.hh" #include "textview_curses.hh" +#include "view_helpers.hh" + +enum class text_quoting { + none, + sql, + regex, +}; void add_view_text_possibilities(readline_curses* rlc, int context, const std::string& type, - textview_curses* tc); + textview_curses* tc, + text_quoting tq); + +template::value, bool> = true> +void +add_view_text_possibilities(readline_curses* rlc, T context, Args... args) +{ + add_view_text_possibilities( + rlc, lnav::enums::to_underlying(context), args...); +} + void add_filter_expr_possibilities(readline_curses* rlc, int context, const std::string& type); -void add_env_possibilities(int context); + +template::value, bool> = true> +void +add_filter_expr_possibilities(readline_curses* rlc, T context, Args... args) +{ + add_filter_expr_possibilities( + rlc, lnav::enums::to_underlying(context), args...); +} + +void add_env_possibilities(ln_mode_t context); void add_filter_possibilities(textview_curses* tc); void add_mark_possibilities(); void add_config_possibilities(); diff --git a/src/regex101.client.cc b/src/regex101.client.cc new file mode 100644 index 00000000..73871cc4 --- /dev/null +++ b/src/regex101.client.cc @@ -0,0 +1,331 @@ +/** + * Copyright (c) 2022, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "regex101.client.hh" + +#include + +#include "config.h" +#include "curl_looper.hh" +#include "ghc/filesystem.hpp" +#include "yajlpp/yajlpp_def.hh" + +namespace regex101 { +namespace client { + +static const json_path_handler_base::enum_value_t CRITERIA_ENUM[] = { + {"DOES_MATCH", unit_test::criteria::DOES_MATCH}, + {"DOES_NOT_MATCH", unit_test::criteria::DOES_NOT_MATCH}, + + json_path_handler_base::ENUM_TERMINATOR, +}; + +static const json_path_container UNIT_TEST_HANDLERS = { + yajlpp::property_handler("description") + .for_field(&unit_test::ut_description), + yajlpp::property_handler("testString") + .for_field(&unit_test::ut_test_string), + yajlpp::property_handler("target").for_field(&unit_test::ut_target), + yajlpp::property_handler("criteria") + .with_enum_values(CRITERIA_ENUM) + .for_field(&unit_test::ut_criteria), +}; + +static const typed_json_path_container ENTRY_HANDLERS = { + yajlpp::property_handler("regex").for_field(&entry::e_regex), + yajlpp::property_handler("testString").for_field(&entry::e_test_string), + yajlpp::property_handler("flags").for_field(&entry::e_flags), + yajlpp::property_handler("delimiter").for_field(&entry::e_delimiter), + yajlpp::property_handler("flavor").for_field(&entry::e_flavor), + yajlpp::property_handler("unitTests#") + .for_field(&entry::e_unit_tests) + .with_children(UNIT_TEST_HANDLERS), + yajlpp::property_handler("permalinkFragment") + .for_field(&entry::e_permalink_fragment), +}; + +static const typed_json_path_container RESPONSE_HANDLERS = { + yajlpp::property_handler("deleteCode") + .for_field(&upsert_response::cr_delete_code), + yajlpp::property_handler("permalinkFragment") + .for_field(&upsert_response::cr_permalink_fragment), + yajlpp::property_handler("version").for_field(&upsert_response::cr_version), +}; + +static const ghc::filesystem::path REGEX101_BASE_URL + = "https://regex101.com/api/regex"; +static const char* USER_AGENT = "lnav/" PACKAGE_VERSION; + +Result +upsert(entry& en) +{ + auto entry_json = ENTRY_HANDLERS.to_string(en); + + curl_request cr(REGEX101_BASE_URL.string()); + + curl_easy_setopt(cr, CURLOPT_URL, cr.get_name().c_str()); + curl_easy_setopt(cr, CURLOPT_POST, 1); + curl_easy_setopt(cr, CURLOPT_POSTFIELDS, entry_json.c_str()); + curl_easy_setopt(cr, CURLOPT_POSTFIELDSIZE, entry_json.size()); + curl_easy_setopt(cr, CURLOPT_USERAGENT, USER_AGENT); + + auto_mem list(curl_slist_free_all); + + list = curl_slist_append(list, "Content-Type: application/json"); + + curl_easy_setopt(cr, CURLOPT_HTTPHEADER, list.in()); + + auto perform_res = cr.perform(); + if (perform_res.isErr()) { + return Err( + lnav::console::user_message::error( + "unable to create entry on regex101.com") + .with_reason(curl_easy_strerror(perform_res.unwrapErr()))); + } + + auto response = perform_res.unwrap(); + auto resp_code = cr.get_response_code(); + if (resp_code != 200) { + return Err(lnav::console::user_message::error( + "unable to create entry on regex101.com") + .with_reason(attr_line_t() + .append("received response code ") + .append(lnav::roles::number( + fmt::to_string(resp_code))) + .append(" content ") + .append_quoted(response))); + } + + auto parse_res + = RESPONSE_HANDLERS.parser_for(intern_string::lookup(cr.get_name())) + .of(response); + if (parse_res.isOk()) { + return Ok(parse_res.unwrap()); + } + + auto errors = parse_res.unwrapErr(); + return Err(lnav::console::user_message::error( + "unable to create entry on regex101.com") + .with_reason(errors[0].to_attr_line({}))); +} + +struct retrieve_entity { + std::string re_permalink_fragment; + int32_t re_versions{1}; +}; + +static const typed_json_path_container RETRIEVE_ENTITY_HANDLERS + = { + yajlpp::property_handler("permalinkFragment") + .for_field(&retrieve_entity::re_permalink_fragment), + yajlpp::property_handler("versions") + .for_field(&retrieve_entity::re_versions), +}; + +retrieve_result_t +retrieve(const std::string& permalink) +{ + auto entry_url = REGEX101_BASE_URL / permalink; + curl_request entry_req(entry_url.string()); + + curl_easy_setopt(entry_req, CURLOPT_URL, entry_req.get_name().c_str()); + curl_easy_setopt(entry_req, CURLOPT_USERAGENT, USER_AGENT); + + auto perform_res = entry_req.perform(); + if (perform_res.isErr()) { + return lnav::console::user_message::error( + attr_line_t("unable to get entry ") + .append_quoted(lnav::roles::symbol(permalink)) + .append(" on regex101.com")) + .with_reason(curl_easy_strerror(perform_res.unwrapErr())); + } + + auto response = perform_res.unwrap(); + auto resp_code = entry_req.get_response_code(); + if (resp_code == 404) { + return no_entry{}; + } + if (resp_code != 200) { + return lnav::console::user_message::error( + attr_line_t("unable to get entry ") + .append_quoted(lnav::roles::symbol(permalink)) + .append(" on regex101.com")) + .with_reason( + attr_line_t() + .append("received response code ") + .append(lnav::roles::number(fmt::to_string(resp_code))) + .append(" content ") + .append_quoted(response)); + } + + auto parse_res + = RETRIEVE_ENTITY_HANDLERS + .parser_for(intern_string::lookup(entry_req.get_name())) + .of(response); + + if (parse_res.isErr()) { + auto parse_errors = parse_res.unwrapErr(); + + return lnav::console::user_message::error( + attr_line_t("unable to get entry ") + .append_quoted(lnav::roles::symbol(permalink)) + .append(" on regex101.com")) + .with_reason(parse_errors[0].to_attr_line({})); + } + + auto entry_value = parse_res.unwrap(); + + if (entry_value.re_versions == 0) { + return no_entry{}; + } + + auto version_url = entry_url / fmt::to_string(entry_value.re_versions); + curl_request version_req(version_url.string()); + + curl_easy_setopt(version_req, CURLOPT_URL, version_req.get_name().c_str()); + curl_easy_setopt(version_req, CURLOPT_USERAGENT, USER_AGENT); + + auto version_perform_res = version_req.perform(); + if (version_perform_res.isErr()) { + return lnav::console::user_message::error( + attr_line_t("unable to get entry version ") + .append_quoted(lnav::roles::symbol(version_url.string())) + .append(" on regex101.com")) + .with_reason(curl_easy_strerror(version_perform_res.unwrapErr())); + } + + auto version_response = version_perform_res.unwrap(); + auto version_parse_res + = ENTRY_HANDLERS + .parser_for(intern_string::lookup(version_req.get_name())) + .of(version_response); + + if (version_parse_res.isErr()) { + auto parse_errors = version_parse_res.unwrapErr(); + return lnav::console::user_message::error( + attr_line_t("unable to get entry version ") + .append_quoted(lnav::roles::symbol(version_url.string())) + .append(" on regex101.com")) + .with_reason(parse_errors[0].to_attr_line({})); + } + + auto retval = version_parse_res.unwrap(); + + retval.e_permalink_fragment = permalink; + + return retval; +} + +struct delete_entity { + std::string de_delete_code; +}; + +static const typed_json_path_container DELETE_ENTITY_HANDLERS = { + yajlpp::property_handler("deleteCode") + .for_field(&delete_entity::de_delete_code), +}; + +Result +delete_entry(const std::string& delete_code) +{ + curl_request cr(REGEX101_BASE_URL.string()); + delete_entity entity{delete_code}; + auto entity_json = DELETE_ENTITY_HANDLERS.to_string(entity); + + curl_easy_setopt(cr, CURLOPT_URL, cr.get_name().c_str()); + curl_easy_setopt(cr, CURLOPT_CUSTOMREQUEST, "DELETE"); + curl_easy_setopt(cr, CURLOPT_USERAGENT, USER_AGENT); + curl_easy_setopt(cr, CURLOPT_POSTFIELDS, entity_json.c_str()); + curl_easy_setopt(cr, CURLOPT_POSTFIELDSIZE, entity_json.size()); + + auto_mem list(curl_slist_free_all); + + list = curl_slist_append(list, "Content-Type: application/json"); + + curl_easy_setopt(cr, CURLOPT_HTTPHEADER, list.in()); + + auto perform_res = cr.perform(); + if (perform_res.isErr()) { + return Err( + lnav::console::user_message::error( + "unable to delete entry on regex101.com") + .with_reason(curl_easy_strerror(perform_res.unwrapErr()))); + } + + auto response = perform_res.unwrap(); + auto resp_code = cr.get_response_code(); + if (resp_code != 200) { + return Err(lnav::console::user_message::error( + "unable to delete entry on regex101.com") + .with_reason(attr_line_t() + .append("received response code ") + .append(lnav::roles::number( + fmt::to_string(resp_code))) + .append(" content ") + .append_quoted(response))); + } + + return Ok(); +} + +std::string +to_edit_url(const std::string& permalink) +{ + return fmt::format(FMT_STRING("https://regex101.com/r/{}"), permalink); +} + +bool +unit_test::operator==(const unit_test& rhs) const +{ + return ut_description == rhs.ut_description + && ut_test_string == rhs.ut_test_string && ut_target == rhs.ut_target + && ut_criteria == rhs.ut_criteria; +} + +bool +unit_test::operator!=(const unit_test& rhs) const +{ + return !(rhs == *this); +} + +bool +entry::operator==(const entry& rhs) const +{ + return e_regex == rhs.e_regex && e_test_string == rhs.e_test_string + && e_flavor == rhs.e_flavor && e_unit_tests == rhs.e_unit_tests; +} + +bool +entry::operator!=(const entry& rhs) const +{ + return !(rhs == *this); +} + +} // namespace client +} // namespace regex101 diff --git a/src/regex101.client.hh b/src/regex101.client.hh new file mode 100644 index 00000000..14c1ac9a --- /dev/null +++ b/src/regex101.client.hh @@ -0,0 +1,94 @@ +/** + * Copyright (c) 2022, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef regex101_client_hh +#define regex101_client_hh + +#include +#include + +#include "base/lnav.console.hh" +#include "base/result.h" +#include "mapbox/variant.hpp" + +namespace regex101 { +namespace client { + +struct unit_test { + enum class criteria { + DOES_MATCH, + DOES_NOT_MATCH, + }; + + std::string ut_description; + std::string ut_test_string; + std::string ut_target{"REGEX"}; + criteria ut_criteria{criteria::DOES_MATCH}; + + bool operator==(const unit_test& rhs) const; + bool operator!=(const unit_test& rhs) const; +}; + +struct entry { + std::string e_regex; + std::string e_test_string; + std::string e_flags{"gs"}; + std::string e_delimiter{"/"}; + std::string e_flavor{"pcre"}; + std::vector e_unit_tests; + nonstd::optional e_permalink_fragment; + + bool operator==(const entry& rhs) const; + bool operator!=(const entry& rhs) const; +}; + +struct upsert_response { + std::string cr_delete_code; + std::string cr_permalink_fragment; + int32_t cr_version; +}; + +Result upsert(entry& en); + +struct no_entry {}; + +using retrieve_result_t + = mapbox::util::variant; + +retrieve_result_t retrieve(const std::string& permalink); + +Result delete_entry( + const std::string& delete_code); + +std::string to_edit_url(const std::string& permalink); + +} // namespace client +} // namespace regex101 + +#endif diff --git a/src/regex101.import.cc b/src/regex101.import.cc new file mode 100644 index 00000000..76565f2e --- /dev/null +++ b/src/regex101.import.cc @@ -0,0 +1,386 @@ +/** + * Copyright (c) 2022, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "regex101.import.hh" + +#include "base/fs_util.hh" +#include "base/itertools.hh" +#include "base/paths.hh" +#include "lnav_config.hh" +#include "log_format.hh" +#include "log_format_ext.hh" +#include "pcrepp/pcrepp.hh" +#include "regex101.client.hh" +#include "session_data.hh" +#include "yajlpp/yajlpp.hh" + +using namespace lnav::roles::literals; + +Result +regex101::import(const std::string& url, + const std::string& name, + const std::string& pat_name) +{ + static const pcrepp USER_URL{R"(^https://regex101.com/r/(\w+)(?:/(\d+))?)"}; + static const pcrepp NAME_RE{R"(^\w+$)"}; + + if (url.empty()) { + return Err(lnav::console::user_message::error( + "expecting a regex101.com URL to import")); + } + if (name.empty()) { + return Err(lnav::console::user_message::error( + "expecting a name for the new format")); + } + + auto lformat = log_format::find_root_format(name.c_str()); + bool existing_format = false; + + if (lformat != nullptr) { + auto* ext_format = dynamic_cast(lformat.get()); + + if (ext_format) { + auto found = ext_format->elf_pattern_order + | lnav::itertools::find_if([&pat_name](const auto& elem) { + return elem->p_name == pat_name; + }); + if (!found) { + existing_format = true; + } + } + } + + pcre_context_static<30> pc_name; + pcre_input pi_name{name}; + + if (!NAME_RE.match(pc_name, pi_name)) { + auto partial_len = NAME_RE.match_partial(pi_name); + return Err( + lnav::console::user_message::error( + attr_line_t("unable to import: ") + .append(lnav::roles::file(url))) + .with_reason(attr_line_t("expecting a format name that matches " + "the regular expression ") + .append_quoted(NAME_RE.get_pattern())) + .with_note(attr_line_t(" ") + .append_quoted(name) + .append("\n ") + .append(partial_len, ' ') + .append("^ matched up to here"_comment))); + } + + pcre_context_static<30> pc; + pcre_input pi{url}; + + if (!USER_URL.match(pc, pi)) { + auto partial_len = USER_URL.match_partial(pi); + return Err(lnav::console::user_message::error( + attr_line_t("unrecognized regex101.com URL: ") + .append(lnav::roles::file(url))) + .with_reason(attr_line_t("expecting a URL that matches ") + .append_quoted(USER_URL.get_pattern())) + .with_note(attr_line_t(" ") + .append_quoted(url) + .append("\n ") + .append(partial_len, ' ') + .append("^ matched up to here"_comment))); + } + + auto permalink = pi.get_substr(pc[0]); + + auto format_filename = existing_format + ? fmt::format(FMT_STRING("{}.regex101-{}.json"), name, permalink) + : fmt::format(FMT_STRING("{}.json"), name); + auto format_path + = lnav::paths::dotlnav() / "formats" / "installed" / format_filename; + + if (ghc::filesystem::exists(format_path)) { + return Err(lnav::console::user_message::error( + attr_line_t("unable to import: ") + .append(lnav::roles::file(url))) + .with_reason( + attr_line_t("format file already exists: ") + .append(lnav::roles::file(format_path.string()))) + .with_help("delete the existing file to continue")); + } + + auto retrieve_res = regex101::client::retrieve(permalink); + if (retrieve_res.is()) { + return Err(retrieve_res.get()); + } + + if (retrieve_res.is()) { + return Err(lnav::console::user_message::error( + attr_line_t("unknown regex101.com entry: ") + .append(lnav::roles::symbol(url)))); + } + + auto entry = retrieve_res.get(); + + if (entry.e_flavor != "pcre") { + return Err( + lnav::console::user_message::error( + attr_line_t("invalid regex ") + .append_quoted(lnav::roles::symbol(entry.e_regex)) + .append(" from ") + .append_quoted(lnav::roles::symbol(url))) + .with_reason(attr_line_t("only the ") + .append_quoted("pcre"_symbol) + .append(" flavor of regexes are supported"))); + } + + auto regex_res = pcrepp::from_str(entry.e_regex); + if (regex_res.isErr()) { + auto parse_error = regex_res.unwrapErr(); + return Err(lnav::console::user_message::error( + attr_line_t("invalid regex ") + .append_quoted(lnav::roles::symbol(entry.e_regex)) + .append(" from ") + .append_quoted(lnav::roles::symbol(url))) + .with_reason(parse_error.ce_msg) + .with_help("fix the regex and try the import again")); + } + + auto regex = regex_res.unwrap(); + yajlpp_gen gen; + + yajl_gen_config(gen, yajl_gen_beautify, true); + { + yajlpp_map root_map(gen); + + root_map.gen("$schema"); + root_map.gen(DEFAULT_FORMAT_SCHEMA); + + root_map.gen(name); + { + yajlpp_map format_map(gen); + + if (!existing_format) { + format_map.gen("description"); + format_map.gen(fmt::format( + FMT_STRING( + "Format file generated from regex101 entry -- {}"), + url)); + } + format_map.gen("regex"); + { + yajlpp_map regex_map(gen); + + regex_map.gen(pat_name); + { + yajlpp_map std_map(gen); + + std_map.gen("pattern"); + std_map.gen(entry.e_regex); + } + } + if (!existing_format) { + format_map.gen("value"); + { + yajlpp_map value_map(gen); + + for (auto named_iter = regex.named_begin(); + named_iter != regex.named_end(); + ++named_iter) + { + value_map.gen(named_iter->pnc_name); + { + yajlpp_map cap_map(gen); + + cap_map.gen("kind"); + cap_map.gen("string"); + } + } + } + } + format_map.gen("sample"); + { + yajlpp_array sample_array(gen); + + if (!entry.e_test_string.empty()) { + yajlpp_map elem_map(gen); + + elem_map.gen("line"); + elem_map.gen(rtrim(entry.e_test_string)); + } + for (const auto& ut : entry.e_unit_tests) { + if (ut.ut_test_string.empty()) { + continue; + } + + yajlpp_map elem_map(gen); + + if (!ut.ut_description.empty()) { + elem_map.gen("description"); + elem_map.gen(ut.ut_description); + } + elem_map.gen("line"); + elem_map.gen(rtrim(ut.ut_test_string)); + } + } + } + } + + auto format_json = gen.to_string_fragment(); + auto write_res = lnav::filesystem::write_file(format_path, format_json); + if (write_res.isErr()) { + return Err(lnav::console::user_message::error( + attr_line_t("unable to create format file: ") + .append(lnav::roles::file(format_path))) + .with_reason(write_res.unwrapErr())); + } + + lnav::session::regex101::insert_entry({name, pat_name, permalink, ""}); + + return Ok(format_path); +} + +ghc::filesystem::path +regex101::patch_path(const external_log_format* format, + const std::string& permalink) +{ + if (format->elf_format_source_order.empty()) { + return lnav::paths::dotlnav() / "formats" / "installed" + / fmt::format(FMT_STRING("{}.regex101-{}.json"), + format->get_name(), + permalink); + } + + auto first_path = format->elf_format_source_order.front(); + + return first_path.replace_extension( + fmt::format(FMT_STRING("regex101-{}.json"), permalink)); +} + +Result +regex101::patch(const external_log_format* format, + const std::string& pat_name, + const regex101::client::entry& entry) +{ + yajlpp_gen gen; + + yajl_gen_config(gen, yajl_gen_beautify, true); + { + yajlpp_map root_map(gen); + + root_map.gen("$schema"); + root_map.gen(DEFAULT_FORMAT_SCHEMA); + + root_map.gen(format->get_name()); + { + yajlpp_map format_map(gen); + + format_map.gen("regex"); + { + yajlpp_map regex_map(gen); + + regex_map.gen(pat_name); + { + yajlpp_map pat_map(gen); + + pat_map.gen("pattern"); + pat_map.gen(entry.e_regex); + } + } + + auto new_samples + = entry.e_unit_tests + | lnav::itertools::prepend(regex101::client::unit_test{ + "", + entry.e_test_string, + }) + | lnav::itertools::filter_out([&format](const auto& ut) { + if (ut.ut_test_string.empty()) { + return true; + } + return (format->elf_samples + | lnav::itertools::find_if( + [&ut](const auto& samp) { + return samp.s_line.pp_value + == rtrim(ut.ut_test_string); + })) + .has_value(); + }); + + if (!new_samples.empty()) { + format_map.gen("sample"); + { + yajlpp_array sample_array(gen); + + for (const auto& ut : entry.e_unit_tests) { + yajlpp_map elem_map(gen); + + if (!ut.ut_description.empty()) { + elem_map.gen("description"); + elem_map.gen(ut.ut_description); + } + elem_map.gen("line"); + elem_map.gen(rtrim(ut.ut_test_string)); + } + } + } + } + } + + auto retval + = regex101::patch_path(format, entry.e_permalink_fragment.value()); + auto write_res + = lnav::filesystem::write_file(retval, gen.to_string_fragment()); + if (write_res.isErr()) { + return Err(lnav::console::user_message::error( + attr_line_t("unable to write format patch file: ") + .append(lnav::roles::file(retval.string()))) + .with_reason(write_res.unwrapErr())); + } + + return Ok(retval); +} + +regex101::client::entry +regex101::convert_format_pattern( + const external_log_format* format, + std::shared_ptr pattern) +{ + regex101::client::entry en; + + en.e_regex = pattern->p_pcre->get_pattern(); + for (const auto& sample : format->elf_samples) { + if (en.e_test_string.empty()) { + en.e_test_string = sample.s_line.pp_value; + } else { + regex101::client::unit_test ut; + + ut.ut_test_string = sample.s_line.pp_value; + ut.ut_description = sample.s_description; + en.e_unit_tests.emplace_back(ut); + } + } + + return en; +} diff --git a/src/regex101.import.hh b/src/regex101.import.hh new file mode 100644 index 00000000..cabaa0b5 --- /dev/null +++ b/src/regex101.import.hh @@ -0,0 +1,61 @@ +/** + * Copyright (c) 2022, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef regex101_import_hh +#define regex101_import_hh + +#include + +#include "base/lnav.console.hh" +#include "ghc/filesystem.hpp" +#include "log_format_ext.hh" +#include "regex101.client.hh" + +namespace regex101 { + +Result import( + const std::string& url, + const std::string& name, + const std::string& pat_name); + +ghc::filesystem::path patch_path(const external_log_format* format, + const std::string& permalink); + +Result patch( + const external_log_format* format, + const std::string& pat_name, + const regex101::client::entry& entry); + +regex101::client::entry convert_format_pattern( + const external_log_format* format, + std::shared_ptr pat); + +} // namespace regex101 + +#endif diff --git a/src/relative_time.hh b/src/relative_time.hh index 200a84ab..305f2e41 100644 --- a/src/relative_time.hh +++ b/src/relative_time.hh @@ -121,10 +121,7 @@ public: static relative_time from_usecs(std::chrono::microseconds usecs); - relative_time() - { - this->clear(); - }; + relative_time() { this->clear(); } void clear() { @@ -132,7 +129,7 @@ public: this->rt_next = false; this->rt_previous = false; this->rt_absolute_field_end = 0; - }; + } void negate() { @@ -151,7 +148,7 @@ public: this->rt_field[lpc].value = -this->rt_field[lpc].value; } } - }; + } bool is_negative() const { @@ -164,18 +161,18 @@ public: } } return false; - }; + } bool is_absolute() const { return !this->rt_included_days.empty() || this->rt_absolute_field_end > 0; - }; + } bool is_absolute(rt_field_type rft) const { return rft < this->rt_absolute_field_end; - }; + } bool is_relative() const { @@ -193,7 +190,7 @@ public: } } return true; - }; + } struct exttm adjust_now() const { @@ -203,7 +200,7 @@ public: time(&now); tm.et_tm = *gmtime(&now); return this->adjust(tm); - }; + } struct exttm adjust(const struct timeval& tv) const { @@ -226,7 +223,7 @@ public: tv_out.tv_sec = us / (1000 * 1000); tv_out.tv_usec = us % (1000 * 1000); - }; + } struct timeval to_timeval() const { @@ -236,7 +233,7 @@ public: retval.tv_sec = us / (1000 * 1000); retval.tv_usec = us % (1000 * 1000); return retval; - }; + } std::string to_string() const; @@ -245,9 +242,9 @@ public: static const char FIELD_CHARS[RTF__MAX]; struct _rt_field { - _rt_field(int64_t value) : value(value), is_set(true){}; + _rt_field(int64_t value) : value(value), is_set(true) {} - _rt_field() : value(0), is_set(false){}; + _rt_field() : value(0), is_set(false) {} void clear() { diff --git a/src/session_data.cc b/src/session_data.cc index ddfadc24..70cb0417 100644 --- a/src/session_data.cc +++ b/src/session_data.cc @@ -53,6 +53,7 @@ #include "service_tags.hh" #include "sql_util.hh" #include "tailer/tailer.looper.hh" +#include "vtab_module.hh" #include "yajlpp/yajlpp.hh" #include "yajlpp/yajlpp_def.hh" @@ -60,7 +61,7 @@ struct session_data_t session_data; static const char* LOG_METADATA_NAME = "log_metadata.db"; -static const char* BOOKMARK_TABLE_DEF = R"( +static const char* META_TABLE_DEF = R"( CREATE TABLE IF NOT EXISTS bookmarks ( log_time datetime, log_format varchar(64), @@ -93,6 +94,20 @@ CREATE TABLE IF NOT EXISTS recent_netlocs ( PRIMARY KEY (netloc) ); + +CREATE TABLE IF NOT EXISTS regex101_entries ( + format_name text NOT NULL, + regex_name text NOT NULL, + permalink text NOT NULL, + delete_code text NOT NULL, + + PRIMARY KEY (format_name, regex_name), + + CHECK( + format_name <> '' AND + regex_name <> '' AND + permalink <> '') +); )"; static const char* BOOKMARK_LRU_STMT @@ -180,6 +195,79 @@ bind_values(sqlite3_stmt* stmt, Args... args) stmt, std::make_index_sequence(), args...); } +struct prepared_stmt { + prepared_stmt(auto_mem stmt) : ps_stmt(std::move(stmt)) {} + + Result execute() + { + auto rc = sqlite3_step(this->ps_stmt.in()); + if (rc == SQLITE_OK && rc == SQLITE_DONE) { + return Ok(); + } + + auto msg = std::string( + sqlite3_errmsg(sqlite3_db_handle(this->ps_stmt.in()))); + return Err(msg); + } + + struct end_of_rows {}; + struct fetch_error { + std::string fe_msg; + }; + + template + using fetch_result = mapbox::util::variant; + + template + fetch_result fetch_row() + { + auto rc = sqlite3_step(this->ps_stmt.in()); + if (rc == SQLITE_OK || rc == SQLITE_DONE) { + return end_of_rows{}; + } + + if (rc == SQLITE_ROW) { + const auto argc = sqlite3_column_count(this->ps_stmt.in()); + sqlite3_value* argv[argc]; + + for (int lpc = 0; lpc < argc; lpc++) { + argv[lpc] = sqlite3_column_value(this->ps_stmt.in(), lpc); + } + + return from_sqlite()(argc, argv, 0); + } + + return fetch_error{ + sqlite3_errmsg(sqlite3_db_handle(this->ps_stmt.in())), + }; + } + + auto_mem ps_stmt; +}; + +template +static Result +prepare_stmt(sqlite3* db, const char* sql, Args... args) +{ + auto_mem retval(sqlite3_finalize); + + if (sqlite3_prepare_v2(db, sql, -1, retval.out(), nullptr) != SQLITE_OK) { + return Err( + fmt::format(FMT_STRING("unable to prepare SQL statement: {}"), + sqlite3_errmsg(db))); + } + + if (bind_values(retval.in(), args...) != SQLITE_OK) { + return Err( + fmt::format(FMT_STRING("unable to prepare SQL statement: {}"), + sqlite3_errmsg(db))); + } + + return Ok(prepared_stmt{ + std::move(retval), + }); +} + static bool bind_line(sqlite3* db, sqlite3_stmt* stmt, @@ -455,7 +543,7 @@ load_time_bookmarks() while (!done) { switch (sqlite3_step(stmt.in())) { case SQLITE_ROW: { - auto netloc = sqlite3_column_text(stmt.in(), 0); + const auto* netloc = sqlite3_column_text(stmt.in(), 0); session_data.sd_recent_netlocs.insert((const char*) netloc); break; @@ -948,7 +1036,8 @@ load_session() session_data.sd_save_time = pair.first.second; const auto& view_info_path = pair.second; - yajlpp_parse_context ypc(view_info_path, &view_info_handlers); + yajlpp_parse_context ypc(intern_string::lookup(view_info_path.string()), + &view_info_handlers); ypc.with_obj(session_data); handle = yajl_alloc(&ypc.ypc_callbacks, nullptr, &ypc); @@ -1115,8 +1204,7 @@ save_time_bookmarks() return; } - if (sqlite3_exec( - db.in(), BOOKMARK_TABLE_DEF, nullptr, nullptr, errmsg.out()) + if (sqlite3_exec(db.in(), META_TABLE_DEF, nullptr, nullptr, errmsg.out()) != SQLITE_OK) { log_error("unable to make bookmark table -- %s", errmsg.in()); @@ -1689,3 +1777,166 @@ reset_session() } } } + +void +lnav::session::regex101::insert_entry(const lnav::session::regex101::entry& ei) +{ + constexpr const char* STMT = R"( + INSERT INTO regex101_entries + (format_name, regex_name, permalink, delete_code) + VALUES (?, ?, ?, ?); +)"; + + auto db_path = lnav::paths::dotlnav() / LOG_METADATA_NAME; + auto_mem db; + + if (sqlite3_open(db_path.c_str(), db.out()) != SQLITE_OK) { + return; + } + + auto_mem errmsg; + if (sqlite3_exec(db.in(), META_TABLE_DEF, nullptr, nullptr, errmsg.out()) + != SQLITE_OK) + { + log_error("unable to make bookmark table -- %s", errmsg.in()); + return; + } + + auto prep_res = prepare_stmt(db.in(), + STMT, + ei.re_format_name, + ei.re_regex_name, + ei.re_permalink, + ei.re_delete_code); + + if (prep_res.isErr()) { + return; + } + + auto ps = prep_res.unwrap(); + + ps.execute(); +} + +template<> +struct from_sqlite { + inline lnav::session::regex101::entry operator()(int argc, + sqlite3_value** argv, + int argi) + { + return { + from_sqlite()(argc, argv, argi + 0), + from_sqlite()(argc, argv, argi + 1), + from_sqlite()(argc, argv, argi + 2), + from_sqlite()(argc, argv, argi + 3), + }; + } +}; + +Result, std::string> +lnav::session::regex101::get_entries() +{ + constexpr const char* STMT = R"( + SELECT * FROM regex101_entries; +)"; + + auto db_path = lnav::paths::dotlnav() / LOG_METADATA_NAME; + auto_mem db; + + if (sqlite3_open(db_path.c_str(), db.out()) != SQLITE_OK) { + return Err(std::string()); + } + + auto_mem errmsg; + if (sqlite3_exec(db.in(), META_TABLE_DEF, nullptr, nullptr, errmsg.out()) + != SQLITE_OK) + { + log_error("unable to make bookmark table -- %s", errmsg.in()); + return Err(std::string(errmsg)); + } + + auto ps = TRY(prepare_stmt(db.in(), STMT)); + bool done = false; + std::vector retval; + + while (!done) { + auto fetch_res = ps.fetch_row(); + + if (fetch_res.is()) { + return Err(fetch_res.get().fe_msg); + } + + fetch_res.match( + [&done](const prepared_stmt::end_of_rows&) { done = true; }, + [](const prepared_stmt::fetch_error&) {}, + [&retval](entry en) { retval.emplace_back(en); }); + } + return Ok(retval); +} + +void +lnav::session::regex101::delete_entry(const std::string& format_name, + const std::string& regex_name) +{ + constexpr const char* STMT = R"( + DELETE FROM regex101_entries WHERE + format_name = ? AND regex_name = ?; +)"; + + auto db_path = lnav::paths::dotlnav() / LOG_METADATA_NAME; + auto_mem db; + + if (sqlite3_open(db_path.c_str(), db.out()) != SQLITE_OK) { + return; + } + + auto prep_res = prepare_stmt(db.in(), STMT, format_name, regex_name); + + if (prep_res.isErr()) { + return; + } + + auto ps = prep_res.unwrap(); + + ps.execute(); +} + +lnav::session::regex101::get_result_t +lnav::session::regex101::get_entry(const std::string& format_name, + const std::string& regex_name) +{ + constexpr const char* STMT = R"( + SELECT * FROM regex101_entries WHERE + format_name = ? AND regex_name = ?; + )"; + + auto db_path = lnav::paths::dotlnav() / LOG_METADATA_NAME; + auto_mem db; + + if (sqlite3_open(db_path.c_str(), db.out()) != SQLITE_OK) { + return error{std::string()}; + } + + auto_mem errmsg; + if (sqlite3_exec(db.in(), META_TABLE_DEF, nullptr, nullptr, errmsg.out()) + != SQLITE_OK) + { + log_error("unable to make bookmark table -- %s", errmsg.in()); + return error{std::string(errmsg)}; + } + + auto prep_res = prepare_stmt(db.in(), STMT, format_name, regex_name); + if (prep_res.isErr()) { + return error{prep_res.unwrapErr()}; + } + + auto ps = prep_res.unwrap(); + return ps.fetch_row().match( + [](const prepared_stmt::fetch_error& fe) -> get_result_t { + return error{fe.fe_msg}; + }, + [](const prepared_stmt::end_of_rows&) -> get_result_t { + return no_entry{}; + }, + [](const entry& en) -> get_result_t { return en; }); +} diff --git a/src/session_data.hh b/src/session_data.hh index 04398251..894afa9b 100644 --- a/src/session_data.hh +++ b/src/session_data.hh @@ -35,7 +35,10 @@ #include #include #include +#include +#include "mapbox/variant.hpp" +#include "optional.hpp" #include "view_helpers.hh" struct file_state { @@ -62,4 +65,35 @@ void load_time_bookmarks(); void save_session(); void reset_session(); +namespace lnav { +namespace session { +namespace regex101 { + +struct entry { + std::string re_format_name; + std::string re_regex_name; + std::string re_permalink; + std::string re_delete_code; +}; + +void insert_entry(const entry& ei); + +struct no_entry {}; + +struct error { + std::string e_msg; +}; + +using get_result_t = mapbox::util::variant; + +get_result_t get_entry(const std::string& format_name, + const std::string& regex_name); +void delete_entry(const std::string& format_name, + const std::string& regex_name); +Result, std::string> get_entries(); + +} // namespace regex101 +} // namespace session +} // namespace lnav + #endif diff --git a/src/shlex.hh b/src/shlex.hh index 27e74dce..16f4c585 100644 --- a/src/shlex.hh +++ b/src/shlex.hh @@ -131,7 +131,7 @@ public: result.append(&this->s_str[last_index], this->s_len - last_index); return true; - }; + } template bool split(std::vector& result, const Resolver& vars) @@ -202,7 +202,7 @@ public: { this->s_index = 0; this->s_state = state_t::STATE_NORMAL; - }; + } void scan_variable_ref(pcre_context::capture_t& cap_out, shlex_token_t& token_out); diff --git a/src/spectro_source.hh b/src/spectro_source.hh index 1057777c..aecd42b1 100644 --- a/src/spectro_source.hh +++ b/src/spectro_source.hh @@ -54,7 +54,7 @@ struct spectrogram_thresholds { }; struct spectrogram_request { - explicit spectrogram_request(spectrogram_bounds& sb) : sr_bounds(sb){}; + explicit spectrogram_request(spectrogram_bounds& sb) : sr_bounds(sb) {} spectrogram_bounds& sr_bounds; unsigned long sr_width{0}; @@ -70,10 +70,8 @@ struct spectrogram_row { } struct row_bucket { - row_bucket() : rb_counter(0), rb_marks(0){}; - - int rb_counter; - int rb_marks; + int rb_counter{0}; + int rb_marks{0}; }; row_bucket* sr_values{nullptr}; @@ -89,7 +87,7 @@ struct spectrogram_row { if (marked) { this->sr_values[index].rb_marks += 1; } - }; + } }; class spectrogram_value_source { @@ -122,7 +120,7 @@ public: this->ss_cached_bounds.sb_count = 0; this->ss_row_cache.clear(); this->ss_cursor_column = -1; - }; + } bool list_input_handle_key(listview_curses& lv, int ch) override; @@ -141,7 +139,7 @@ public: line_flags_t flags) override { return 0; - }; + } nonstd::optional time_for_row(vis_line_t row) override; diff --git a/src/sql_util.cc b/src/sql_util.cc index e06dc6d5..d2e73657 100644 --- a/src/sql_util.cc +++ b/src/sql_util.cc @@ -685,12 +685,14 @@ sql_compile_script(sqlite3* db, } else { sql_content.append(script); } - errors.emplace_back(lnav::console::user_message::error( - "failed to compile SQL statement") - .with_reason(errmsg) - .with_snippet(lnav::console::snippet::from( - src_name, sql_content) - .with_line(line_number))); + errors.emplace_back( + lnav::console::user_message::error( + "failed to compile SQL statement") + .with_reason(errmsg) + .with_snippet( + lnav::console::snippet::from( + intern_string::lookup(src_name), sql_content) + .with_line(line_number))); break; } else if (script == tail) { break; @@ -772,7 +774,8 @@ sql_execute_script(sqlite3* db, "failed to execute SQL statement") .with_reason(errmsg) .with_snippet(lnav::console::snippet::from( - src_name, sqlite3_sql(stmt)))); + intern_string::lookup(src_name), + sqlite3_sql(stmt)))); done = true; break; } diff --git a/src/sqlite-extension-func.hh b/src/sqlite-extension-func.hh index 0e140282..a6433bf3 100644 --- a/src/sqlite-extension-func.hh +++ b/src/sqlite-extension-func.hh @@ -52,7 +52,7 @@ struct FuncDef { { this->eTextRep = flags; return *this; - }; + } }; struct FuncDefAgg { diff --git a/src/statusview_curses.hh b/src/statusview_curses.hh index 8107845b..73f8f309 100644 --- a/src/statusview_curses.hh +++ b/src/statusview_curses.hh @@ -46,9 +46,10 @@ public: * @param width The maximum width of the field in characters. * @param role The color role for this field, defaults to VCR_STATUS. */ - status_field(int width = 1, - role_t role = role_t::VCR_STATUS) - : sf_width(width), sf_role(role){}; + status_field(int width = 1, role_t role = role_t::VCR_STATUS) + : sf_width(width), sf_role(role) + { + } virtual ~status_field() = default; @@ -72,98 +73,53 @@ public: va_end(args); return *this; - }; + } void set_stitch_value(role_t left, role_t right); - void set_left_pad(size_t val) - { - this->sf_left_pad = val; - }; - size_t get_left_pad() const - { - return this->sf_left_pad; - }; + void set_left_pad(size_t val) { this->sf_left_pad = val; } + + size_t get_left_pad() const { return this->sf_left_pad; } /** @return The string value for this field. */ - attr_line_t& get_value() - { - return this->sf_value; - }; + attr_line_t& get_value() { return this->sf_value; } - void right_justify(bool yes) - { - this->sf_right_justify = yes; - }; - bool is_right_justified() const - { - return this->sf_right_justify; - }; + void right_justify(bool yes) { this->sf_right_justify = yes; } + bool is_right_justified() const { return this->sf_right_justify; } status_field& set_cylon(bool yes) { this->sf_cylon = yes; return *this; - }; - bool is_cylon() const - { - return this->sf_cylon; - }; + } + + bool is_cylon() const { return this->sf_cylon; } void do_cylon(); /** @return True if this field's value is an empty string. */ - bool empty() const - { - return this->sf_value.get_string().empty(); - }; + bool empty() const { return this->sf_value.get_string().empty(); } - void clear() - { - this->sf_value.clear(); - }; + void clear() { this->sf_value.clear(); } /** @param role The color role for this field. */ - void set_role(role_t role) - { - this->sf_role = role; - }; + void set_role(role_t role) { this->sf_role = role; } /** @return The color role for this field. */ - role_t get_role() const - { - return this->sf_role; - }; + role_t get_role() const { return this->sf_role; } /** @param width The maximum display width, in characters. */ - void set_width(ssize_t width) - { - this->sf_width = width; - }; + void set_width(ssize_t width) { this->sf_width = width; } /** @param width The maximum display width, in characters. */ - ssize_t get_width() const - { - return this->sf_width; - }; + ssize_t get_width() const { return this->sf_width; } /** @param width The maximum display width, in characters. */ - void set_min_width(int width) - { - this->sf_min_width = width; - }; + void set_min_width(int width) { this->sf_min_width = width; } /** @param width The maximum display width, in characters. */ - size_t get_min_width() const - { - return this->sf_min_width; - }; + size_t get_min_width() const { return this->sf_min_width; } - void set_share(int share) - { - this->sf_share = share; - }; - int get_share() const - { - return this->sf_share; - }; + void set_share(int share) { this->sf_share = share; } + + int get_share() const { return this->sf_share; } protected: ssize_t sf_width; /*< The maximum display width, in chars. */ @@ -203,41 +159,17 @@ public: */ class statusview_curses : public view_curses { public: - void set_data_source(status_data_source* src) - { - this->sc_source = src; - }; - status_data_source* get_data_source() - { - return this->sc_source; - }; + void set_data_source(status_data_source* src) { this->sc_source = src; } + status_data_source* get_data_source() { return this->sc_source; } - void set_top(int top) - { - this->sc_top = top; - }; - int get_top() const - { - return this->sc_top; - }; + void set_top(int top) { this->sc_top = top; } + int get_top() const { return this->sc_top; } - void set_window(WINDOW* win) - { - this->sc_window = win; - }; - WINDOW* get_window() - { - return this->sc_window; - }; + void set_window(WINDOW* win) { this->sc_window = win; } + WINDOW* get_window() { return this->sc_window; } - void set_enabled(bool value) - { - this->sc_enabled = value; - }; - bool get_enabled() const - { - return this->sc_enabled; - }; + void set_enabled(bool value) { this->sc_enabled = value; } + bool get_enabled() const { return this->sc_enabled; } void window_change(); diff --git a/src/styling.cc b/src/styling.cc index 9684bf11..9f288233 100644 --- a/src/styling.cc +++ b/src/styling.cc @@ -68,7 +68,8 @@ static const struct json_path_container root_color_handler = { term_color_palette* xterm_colors() { - static term_color_palette retval(xterm_palette_json.to_string_fragment()); + static term_color_palette retval(xterm_palette_json.get_name(), + xterm_palette_json.to_string_fragment()); return &retval; } @@ -76,7 +77,8 @@ xterm_colors() term_color_palette* ansi_colors() { - static term_color_palette retval(ansi_palette_json.to_string_fragment()); + static term_color_palette retval(ansi_palette_json.get_name(), + ansi_palette_json.to_string_fragment()); return &retval; } @@ -180,9 +182,11 @@ rgb_color::operator!=(const rgb_color& rhs) const return !(rhs == *this); } -term_color_palette::term_color_palette(const string_fragment& json) +term_color_palette::term_color_palette(const char* name, + const string_fragment& json) { - yajlpp_parse_context ypc_xterm("palette.json", &root_color_handler); + yajlpp_parse_context ypc_xterm(intern_string::lookup(name), + &root_color_handler); yajl_handle handle; handle = yajl_alloc(&ypc_xterm.ypc_callbacks, nullptr, &ypc_xterm); diff --git a/src/styling.hh b/src/styling.hh index ea11a48e..16d8c700 100644 --- a/src/styling.hh +++ b/src/styling.hh @@ -71,7 +71,7 @@ struct rgb_color { }; struct lab_color { - lab_color() : lc_l(0), lc_a(0), lc_b(0){}; + lab_color() : lc_l(0), lc_a(0), lc_b(0) {} explicit lab_color(const rgb_color& rgb); @@ -84,7 +84,7 @@ struct lab_color { this->lc_b = other.lc_b; return *this; - }; + } bool operator==(const lab_color& rhs) const; @@ -112,7 +112,7 @@ struct term_color { }; struct term_color_palette { - explicit term_color_palette(const string_fragment& json); + term_color_palette(const char* name, const string_fragment& json); short match_color(const lab_color& to_match); @@ -207,6 +207,7 @@ struct lnav_theme { style_config lt_style_inactive_alert_status; style_config lt_style_file; style_config lt_style_header[6]; + style_config lt_style_list_glyph; std::map lt_level_styles; std::map lt_highlights; }; diff --git a/src/tailer/tailer.looper.cc b/src/tailer/tailer.looper.cc index f8865b68..4d678969 100644 --- a/src/tailer/tailer.looper.cc +++ b/src/tailer/tailer.looper.cc @@ -37,6 +37,7 @@ #include "config.h" #include "line_buffer.hh" #include "lnav.hh" +#include "lnav.indexing.hh" #include "service_tags.hh" #include "tailer.h" #include "tailer.looper.cfg.hh" @@ -969,7 +970,7 @@ tailer::looper::host_tailer::loop_body() isc::to().send( [full_path](auto& mlooper) { lnav_data.ld_rl_view->add_possibility( - LNM_COMMAND, "remote-path", full_path); + ln_mode_t::COMMAND, "remote-path", full_path); }); return std::move(this->ht_state); }); diff --git a/src/term_extra.hh b/src/term_extra.hh index 61b9a8c9..97403155 100644 --- a/src/term_extra.hh +++ b/src/term_extra.hh @@ -65,7 +65,7 @@ public: } this->te_prefix += ":"; } - }; + } void update_title(listview_curses* lc) { @@ -108,7 +108,7 @@ public: this->te_last_title = view_title; } - }; + } private: bool te_enabled; diff --git a/src/termios_guard.hh b/src/termios_guard.hh index 46d36ea9..64fc7df8 100644 --- a/src/termios_guard.hh +++ b/src/termios_guard.hh @@ -55,7 +55,7 @@ public: && tcgetattr(this->gt_fd, &this->gt_termios) == -1) { perror("tcgetattr"); } - }; + } /** * Restore the TTY termios settings that were captured when this object was @@ -68,12 +68,9 @@ public: { perror("tcsetattr"); } - }; + } - const struct termios* get_termios() const - { - return &this->gt_termios; - }; + const struct termios* get_termios() const { return &this->gt_termios; } private: const int gt_fd; diff --git a/src/textfile_sub_source.hh b/src/textfile_sub_source.hh index 8391a61a..26b05ef3 100644 --- a/src/textfile_sub_source.hh +++ b/src/textfile_sub_source.hh @@ -42,22 +42,13 @@ class textfile_sub_source public: typedef std::deque>::iterator file_iterator; - textfile_sub_source() - { - this->tss_supports_filtering = true; - }; + textfile_sub_source() { this->tss_supports_filtering = true; } ~textfile_sub_source() = default; - bool empty() const - { - return this->tss_files.empty(); - } + bool empty() const { return this->tss_files.empty(); } - size_t size() const - { - return this->tss_files.size(); - } + size_t size() const { return this->tss_files.size(); } size_t text_line_count(); @@ -66,7 +57,7 @@ public: return this->tss_files.empty() ? 0 : this->current_file()->get_longest_line_length(); - }; + } void text_value_for_line(textview_curses& tc, int line, @@ -88,7 +79,7 @@ public: } return this->tss_files.front(); - }; + } std::string text_source_name(const textview_curses& tv) { @@ -97,7 +88,7 @@ public: } return this->tss_files.front()->get_filename(); - }; + } void to_front(const std::shared_ptr& lf); @@ -186,7 +177,7 @@ public: } return retval; - }; + } void text_filters_changed(); @@ -207,7 +198,7 @@ private: auto* lfo = (line_filter_observer*) lf->get_logline_observer(); lf->set_logline_observer(nullptr); delete lfo; - }; + } std::deque> tss_files; std::deque> tss_hidden_files; diff --git a/src/textview_curses.cc b/src/textview_curses.cc index 27446267..524c3755 100644 --- a/src/textview_curses.cc +++ b/src/textview_curses.cc @@ -758,6 +758,24 @@ textview_curses::toggle_user_mark(const bookmark_type_t* bm, this->search_new_data(); } +void +textview_curses::redo_search() +{ + if (this->tc_search_child) { + grep_proc* gp = this->tc_search_child->get_grep_proc(); + + gp->invalidate(); + this->match_reset(); + gp->queue_request(0_vl).start(); + + if (this->tc_source_search_child) { + this->tc_source_search_child->invalidate() + .queue_request(0_vl) + .start(); + } + } +} + void text_time_translator::scroll_invoked(textview_curses* tc) { @@ -800,3 +818,157 @@ empty_filter::to_command() const { return ""; } + +nonstd::optional +filter_stack::next_index() +{ + bool used[32]; + + memset(used, 0, sizeof(used)); + for (auto& iter : *this) { + if (iter->lf_deleted) { + continue; + } + + size_t index = iter->get_index(); + + require(used[index] == false); + + used[index] = true; + } + for (size_t lpc = this->fs_reserved; + lpc < logfile_filter_state::MAX_FILTERS; + lpc++) + { + if (!used[lpc]) { + return lpc; + } + } + return nonstd::nullopt; +} + +std::shared_ptr +filter_stack::get_filter(const std::string& id) +{ + auto iter = this->fs_filters.begin(); + std::shared_ptr retval; + + for (; iter != this->fs_filters.end() && (*iter)->get_id() != id; iter++) { + } + if (iter != this->fs_filters.end()) { + retval = *iter; + } + + return retval; +} + +bool +filter_stack::delete_filter(const std::string& id) +{ + auto iter = this->fs_filters.begin(); + + for (; iter != this->fs_filters.end() && (*iter)->get_id() != id; iter++) { + } + if (iter != this->fs_filters.end()) { + this->fs_filters.erase(iter); + return true; + } + + return false; +} + +void +filter_stack::get_mask(uint32_t& filter_mask) +{ + filter_mask = 0; + for (auto& iter : *this) { + std::shared_ptr tf = iter; + + if (tf->lf_deleted) { + continue; + } + if (tf->is_enabled()) { + uint32_t bit = (1UL << tf->get_index()); + + switch (tf->get_type()) { + case text_filter::EXCLUDE: + case text_filter::INCLUDE: + filter_mask |= bit; + break; + default: + ensure(0); + break; + } + } + } +} + +void +filter_stack::get_enabled_mask(uint32_t& filter_in_mask, + uint32_t& filter_out_mask) +{ + filter_in_mask = filter_out_mask = 0; + for (auto& iter : *this) { + std::shared_ptr tf = iter; + + if (tf->lf_deleted) { + continue; + } + if (tf->is_enabled()) { + uint32_t bit = (1UL << tf->get_index()); + + switch (tf->get_type()) { + case text_filter::EXCLUDE: + filter_out_mask |= bit; + break; + case text_filter::INCLUDE: + filter_in_mask |= bit; + break; + default: + ensure(0); + break; + } + } + } +} + +void +vis_location_history::loc_history_append(vis_line_t top) +{ + auto iter = this->vlh_history.begin(); + iter += this->vlh_history.size() - this->lh_history_position; + this->vlh_history.erase_from(iter); + this->lh_history_position = 0; + this->vlh_history.push_back(top); +} + +nonstd::optional +vis_location_history::loc_history_back(vis_line_t current_top) +{ + if (this->lh_history_position == 0) { + vis_line_t history_top = this->current_position(); + if (history_top != current_top) { + return history_top; + } + } + + if (this->lh_history_position + 1 >= this->vlh_history.size()) { + return nonstd::nullopt; + } + + this->lh_history_position += 1; + + return this->current_position(); +} + +nonstd::optional +vis_location_history::loc_history_forward(vis_line_t current_top) +{ + if (this->lh_history_position == 0) { + return nonstd::nullopt; + } + + this->lh_history_position -= 1; + + return this->current_position(); +} diff --git a/src/textview_curses.hh b/src/textview_curses.hh index cb8681b7..fa0d625a 100644 --- a/src/textview_curses.hh +++ b/src/textview_curses.hh @@ -69,7 +69,7 @@ public: 0, sizeof(this->tfs_last_lines_for_message)); this->tfs_mask.reserve(64 * 1024); - }; + } void clear() { @@ -88,7 +88,7 @@ public: sizeof(this->tfs_last_lines_for_message)); this->tfs_mask.clear(); this->tfs_index.clear(); - }; + } void clear_filter_state(size_t index) { @@ -98,7 +98,7 @@ public: this->tfs_lines_for_message[index] = 0; this->tfs_last_message_matched[index] = false; this->tfs_last_lines_for_message[index] = 0; - }; + } void clear_deleted_filter_state(uint32_t used_mask) { @@ -122,7 +122,7 @@ public: 0, sizeof(uint32_t) * (newsize - old_mask_size)); } - }; + } const static int MAX_FILTERS = 32; @@ -159,47 +159,17 @@ public: : lf_type(type), lf_lang(lang), lf_id(std::move(id)), lf_index(index){}; virtual ~text_filter() = default; - type_t get_type() const - { - return this->lf_type; - } - filter_lang_t get_lang() const - { - return this->lf_lang; - } - void set_type(type_t t) - { - this->lf_type = t; - }; - std::string get_id() const - { - return this->lf_id; - }; - void set_id(std::string id) - { - this->lf_id = std::move(id); - } - size_t get_index() const - { - return this->lf_index; - }; + type_t get_type() const { return this->lf_type; } + filter_lang_t get_lang() const { return this->lf_lang; } + void set_type(type_t t) { this->lf_type = t; } + std::string get_id() const { return this->lf_id; } + void set_id(std::string id) { this->lf_id = std::move(id); } + size_t get_index() const { return this->lf_index; } - bool is_enabled() const - { - return this->lf_enabled; - }; - void enable() - { - this->lf_enabled = true; - }; - void disable() - { - this->lf_enabled = false; - }; - void set_enabled(bool value) - { - this->lf_enabled = value; - } + bool is_enabled() const { return this->lf_enabled; } + void enable() { this->lf_enabled = true; } + void disable() { this->lf_enabled = false; } + void set_enabled(bool value) { this->lf_enabled = value; } void revert_to_last(logfile_filter_state& lfs, size_t rollback_size); @@ -216,10 +186,7 @@ public: virtual std::string to_command() const = 0; - bool operator==(const std::string& rhs) - { - return this->lf_id == rhs; - }; + bool operator==(const std::string& rhs) const { return this->lf_id == rhs; } bool lf_deleted{false}; @@ -248,28 +215,23 @@ public: class filter_stack { public: using iterator = std::vector>::iterator; + using const_iterator + = std::vector>::const_iterator; + using value_type = std::shared_ptr; explicit filter_stack(size_t reserved = 0) : fs_reserved(reserved) {} - iterator begin() - { - return this->fs_filters.begin(); - } + iterator begin() { return this->fs_filters.begin(); } - iterator end() - { - return this->fs_filters.end(); - } + iterator end() { return this->fs_filters.end(); } - size_t size() const - { - return this->fs_filters.size(); - } + const_iterator begin() const { return this->fs_filters.begin(); } - bool empty() const - { - return this->fs_filters.empty(); - }; + const_iterator end() const { return this->fs_filters.end(); } + + size_t size() const { return this->fs_filters.size(); } + + bool empty() const { return this->fs_filters.empty(); }; bool full() const { @@ -277,44 +239,19 @@ public: == logfile_filter_state::MAX_FILTERS; } - nonstd::optional next_index() - { - bool used[32]; - - memset(used, 0, sizeof(used)); - for (auto& iter : *this) { - if (iter->lf_deleted) { - continue; - } - - size_t index = iter->get_index(); - - require(used[index] == false); - - used[index] = true; - } - for (size_t lpc = this->fs_reserved; - lpc < logfile_filter_state::MAX_FILTERS; - lpc++) - { - if (!used[lpc]) { - return lpc; - } - } - return nonstd::nullopt; - }; + nonstd::optional next_index(); void add_filter(const std::shared_ptr& filter) { this->fs_filters.push_back(filter); - }; + } void clear_filters() { while (!this->fs_filters.empty()) { this->fs_filters.pop_back(); } - }; + } void set_filter_enabled(const std::shared_ptr& filter, bool enabled) @@ -326,87 +263,13 @@ public: } } - std::shared_ptr get_filter(const std::string& id) - { - auto iter = this->fs_filters.begin(); - std::shared_ptr retval; - - for (; iter != this->fs_filters.end() && (*iter)->get_id() != id; - iter++) { - } - if (iter != this->fs_filters.end()) { - retval = *iter; - } - - return retval; - }; - - bool delete_filter(const std::string& id) - { - auto iter = this->fs_filters.begin(); + std::shared_ptr get_filter(const std::string& id); - for (; iter != this->fs_filters.end() && (*iter)->get_id() != id; - iter++) { - } - if (iter != this->fs_filters.end()) { - this->fs_filters.erase(iter); - return true; - } + bool delete_filter(const std::string& id); - return false; - }; + void get_mask(uint32_t& filter_mask); - void get_mask(uint32_t& filter_mask) - { - filter_mask = 0; - for (auto& iter : *this) { - std::shared_ptr tf = iter; - - if (tf->lf_deleted) { - continue; - } - if (tf->is_enabled()) { - uint32_t bit = (1UL << tf->get_index()); - - switch (tf->get_type()) { - case text_filter::EXCLUDE: - case text_filter::INCLUDE: - filter_mask |= bit; - break; - default: - ensure(0); - break; - } - } - } - } - - void get_enabled_mask(uint32_t& filter_in_mask, uint32_t& filter_out_mask) - { - filter_in_mask = filter_out_mask = 0; - for (auto& iter : *this) { - std::shared_ptr tf = iter; - - if (tf->lf_deleted) { - continue; - } - if (tf->is_enabled()) { - uint32_t bit = (1UL << tf->get_index()); - - switch (tf->get_type()) { - case text_filter::EXCLUDE: - filter_out_mask |= bit; - break; - case text_filter::INCLUDE: - filter_in_mask |= bit; - break; - default: - ensure(0); - break; - } - } - } - }; + void get_enabled_mask(uint32_t& filter_in_mask, uint32_t& filter_out_mask); private: const size_t fs_reserved; @@ -478,20 +341,14 @@ public: { } - void register_view(textview_curses* tc) - { - this->tss_view = tc; - }; + void register_view(textview_curses* tc) { this->tss_view = tc; } /** * @return The total number of lines available from the source. */ virtual size_t text_line_count() = 0; - virtual size_t text_line_width(textview_curses& curses) - { - return INT_MAX; - }; + virtual size_t text_line_width(textview_curses& curses) { return INT_MAX; } /** * Get the value for a line. @@ -560,31 +417,20 @@ public: virtual std::string text_source_name(const textview_curses& tv) { return ""; - }; + } - filter_stack& get_filters() - { - return this->tss_filters; - }; + filter_stack& get_filters() { return this->tss_filters; } - virtual void text_filters_changed(){ + virtual void text_filters_changed() {} - }; + virtual int get_filtered_count() const { return 0; } - virtual int get_filtered_count() const - { - return 0; - }; - - virtual int get_filtered_count_for(size_t filter_index) const - { - return 0; - } + virtual int get_filtered_count_for(size_t filter_index) const { return 0; } virtual text_format_t get_text_format() const { return text_format_t::TF_UNKNOWN; - }; + } virtual nonstd::optional< std::pair*, grep_proc_sink*>> @@ -620,45 +466,13 @@ public: { } - void loc_history_append(vis_line_t top) override - { - auto iter = this->vlh_history.begin(); - iter += this->vlh_history.size() - this->lh_history_position; - this->vlh_history.erase_from(iter); - this->lh_history_position = 0; - this->vlh_history.push_back(top); - } + void loc_history_append(vis_line_t top) override; nonstd::optional loc_history_back( - vis_line_t current_top) override - { - if (this->lh_history_position == 0) { - vis_line_t history_top = this->current_position(); - if (history_top != current_top) { - return history_top; - } - } - - if (this->lh_history_position + 1 >= this->vlh_history.size()) { - return nonstd::nullopt; - } - - this->lh_history_position += 1; - - return this->current_position(); - } + vis_line_t current_top) override; nonstd::optional loc_history_forward( - vis_line_t current_top) override - { - if (this->lh_history_position == 0) { - return nonstd::nullopt; - } - - this->lh_history_position -= 1; - - return this->current_position(); - } + vis_line_t current_top) override; nonstd::ring_span vlh_history; @@ -679,12 +493,12 @@ class text_delegate { public: virtual ~text_delegate() = default; - virtual void text_overlay(textview_curses& tc){}; + virtual void text_overlay(textview_curses& tc) {} virtual bool text_handle_mouse(textview_curses& tc, mouse_event& me) { return false; - }; + } }; /** @@ -719,20 +533,11 @@ public: } } - bool is_paused() const - { - return this->tc_paused; - } + bool is_paused() const { return this->tc_paused; } - vis_bookmarks& get_bookmarks() - { - return this->tc_bookmarks; - }; + vis_bookmarks& get_bookmarks() { return this->tc_bookmarks; } - const vis_bookmarks& get_bookmarks() const - { - return this->tc_bookmarks; - }; + const vis_bookmarks& get_bookmarks() const { return this->tc_bookmarks; } void toggle_user_mark(const bookmark_type_t* bm, vis_line_t start_line, @@ -748,24 +553,21 @@ public: } this->reload_data(); return *this; - }; + } - text_sub_source* get_sub_source() const - { - return this->tc_sub_source; - }; + text_sub_source* get_sub_source() const { return this->tc_sub_source; } textview_curses& set_delegate(std::shared_ptr del) { this->tc_delegate = del; return *this; - }; + } std::shared_ptr get_delegate() const { return this->tc_delegate; - }; + } void horiz_shift(vis_line_t start, vis_line_t end, @@ -785,14 +587,14 @@ public: return this->tc_sub_source == nullptr ? 0 : this->tc_sub_source->text_line_count(); - }; + } size_t listview_width(const listview_curses& lv) { return this->tc_sub_source == nullptr ? 0 : this->tc_sub_source->text_line_width(*this); - }; + } void listview_value_for_rows(const listview_curses& lv, vis_line_t line, @@ -803,14 +605,14 @@ public: size_t listview_size_for_row(const listview_curses& lv, vis_line_t row) { return this->tc_sub_source->text_size_for_line(*this, row); - }; + } std::string listview_source_name(const listview_curses& lv) { return this->tc_sub_source == nullptr ? "" : this->tc_sub_source->text_source_name(*this); - }; + } bool grep_value_for_line(vis_line_t line, std::string& value_out) { @@ -824,7 +626,7 @@ public: } return retval; - }; + } void grep_begin(grep_proc& gp, vis_line_t start, @@ -834,10 +636,7 @@ public: int start, int end); - bool is_searching() const - { - return this->tc_searching > 0; - }; + bool is_searching() const { return this->tc_searching > 0; } void set_follow_search_for(int64_t ms_to_deadline, std::function func) @@ -850,12 +649,9 @@ public: timeradd(&now, &tv, &this->tc_follow_deadline); this->tc_follow_top = this->get_top(); this->tc_follow_func = func; - }; + } - size_t get_match_count() - { - return this->tc_bookmarks[&BM_SEARCH].size(); - }; + size_t get_match_count() { return this->tc_bookmarks[&BM_SEARCH].size(); } void match_reset() { @@ -863,17 +659,14 @@ public: if (this->tc_sub_source != nullptr) { this->tc_sub_source->text_clear_marks(&BM_SEARCH); } - }; + } - highlight_map_t& get_highlights() - { - return this->tc_highlights; - }; + highlight_map_t& get_highlights() { return this->tc_highlights; } const highlight_map_t& get_highlights() const { return this->tc_highlights; - }; + } std::set& get_disabled_highlights() { @@ -890,7 +683,7 @@ public: if (this->tc_delegate != nullptr) { this->tc_delegate->text_overlay(*this); } - }; + } bool toggle_hide_fields() { @@ -899,7 +692,7 @@ public: this->tc_hide_fields = !this->tc_hide_fields; return retval; - }; + } bool get_hide_fields() const { @@ -908,22 +701,7 @@ public: void execute_search(const std::string& regex_orig); - void redo_search() - { - if (this->tc_search_child) { - grep_proc* gp = this->tc_search_child->get_grep_proc(); - - gp->invalidate(); - this->match_reset(); - gp->queue_request(0_vl).start(); - - if (this->tc_source_search_child) { - this->tc_source_search_child->invalidate() - .queue_request(0_vl) - .start(); - } - } - }; + void redo_search(); void search_range(vis_line_t start, vis_line_t stop = -1_vl) { @@ -999,18 +777,20 @@ protected: std::string hl_name, highlight_map_t& hl_map) : gh_grep_proc(std::move(gp)), gh_hl_source(source), - gh_hl_name(std::move(hl_name)), gh_hl_map(hl_map){}; + gh_hl_name(std::move(hl_name)), gh_hl_map(hl_map) + { + } ~grep_highlighter() { this->gh_hl_map.erase( this->gh_hl_map.find({this->gh_hl_source, this->gh_hl_name})); - }; + } grep_proc* get_grep_proc() { return this->gh_grep_proc.get(); - }; + } private: std::unique_ptr> gh_grep_proc; diff --git a/src/themes/default-theme.json b/src/themes/default-theme.json index e56849f8..94e388d4 100644 --- a/src/themes/default-theme.json +++ b/src/themes/default-theme.json @@ -76,6 +76,9 @@ }, "h6": { "underline": true + }, + "list-glyph": { + "color": "Yellow" } }, "syntax-styles": { diff --git a/src/themes/monocai.json b/src/themes/monocai.json index a67a8186..37a65c00 100644 --- a/src/themes/monocai.json +++ b/src/themes/monocai.json @@ -89,6 +89,9 @@ }, "h6": { "underline": true + }, + "list-glyph": { + "color": "$yellow" } }, "syntax-styles": { diff --git a/src/third-party/CLI/App.hpp b/src/third-party/CLI/App.hpp new file mode 100644 index 00000000..0a70bac6 --- /dev/null +++ b/src/third-party/CLI/App.hpp @@ -0,0 +1,3246 @@ +// Copyright (c) 2017-2022, University of Cincinnati, developed by Henry Schreiner +// under NSF AWARD 1414736 and by the respective contributors. +// All rights reserved. +// +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +// [CLI11:public_includes:set] +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +// [CLI11:public_includes:end] + +// CLI Library includes +#include "ConfigFwd.hpp" +#include "Error.hpp" +#include "FormatterFwd.hpp" +#include "Macros.hpp" +#include "Option.hpp" +#include "Split.hpp" +#include "StringTools.hpp" +#include "TypeTools.hpp" + +namespace CLI { +// [CLI11:app_hpp:verbatim] + +#ifndef CLI11_PARSE +#define CLI11_PARSE(app, argc, argv) \ + try { \ + (app).parse((argc), (argv)); \ + } catch(const CLI::ParseError &e) { \ + return (app).exit(e); \ + } +#endif + +namespace detail { +enum class Classifier { NONE, POSITIONAL_MARK, SHORT, LONG, WINDOWS_STYLE, SUBCOMMAND, SUBCOMMAND_TERMINATOR }; +struct AppFriend; +} // namespace detail + +namespace FailureMessage { +std::string simple(const App *app, const Error &e); +std::string help(const App *app, const Error &e); +} // namespace FailureMessage + +/// enumeration of modes of how to deal with extras in config files + +enum class config_extras_mode : char { error = 0, ignore, ignore_all, capture }; + +class App; + +using App_p = std::shared_ptr; + +namespace detail { +/// helper functions for adding in appropriate flag modifiers for add_flag + +template ::value || (sizeof(T) <= 1U), detail::enabler> = detail::dummy> +Option *default_flag_modifiers(Option *opt) { + return opt->always_capture_default(); +} + +/// summing modifiers +template ::value && (sizeof(T) > 1U), detail::enabler> = detail::dummy> +Option *default_flag_modifiers(Option *opt) { + return opt->multi_option_policy(MultiOptionPolicy::Sum)->default_str("0")->force_callback(); +} + +} // namespace detail + +class Option_group; +/// Creates a command line program, with very few defaults. +/** To use, create a new `Program()` instance with `argc`, `argv`, and a help description. The templated + * add_option methods make it easy to prepare options. Remember to call `.start` before starting your + * program, so that the options can be evaluated and the help option doesn't accidentally run your program. */ +class App { + friend Option; + friend detail::AppFriend; + + protected: + // This library follows the Google style guide for member names ending in underscores + + /// @name Basics + ///@{ + + /// Subcommand name or program name (from parser if name is empty) + std::string name_{}; + + /// Description of the current program/subcommand + std::string description_{}; + + /// If true, allow extra arguments (ie, don't throw an error). INHERITABLE + bool allow_extras_{false}; + + /// If ignore, allow extra arguments in the ini file (ie, don't throw an error). INHERITABLE + /// if error error on an extra argument, and if capture feed it to the app + config_extras_mode allow_config_extras_{config_extras_mode::ignore}; + + /// If true, return immediately on an unrecognized option (implies allow_extras) INHERITABLE + bool prefix_command_{false}; + + /// If set to true the name was automatically generated from the command line vs a user set name + bool has_automatic_name_{false}; + + /// If set to true the subcommand is required to be processed and used, ignored for main app + bool required_{false}; + + /// If set to true the subcommand is disabled and cannot be used, ignored for main app + bool disabled_{false}; + + /// Flag indicating that the pre_parse_callback has been triggered + bool pre_parse_called_{false}; + + /// Flag indicating that the callback for the subcommand should be executed immediately on parse completion which is + /// before help or ini files are processed. INHERITABLE + bool immediate_callback_{false}; + + /// This is a function that runs prior to the start of parsing + std::function pre_parse_callback_{}; + + /// This is a function that runs when parsing has finished. + std::function parse_complete_callback_{}; + + /// This is a function that runs when all processing has completed + std::function final_callback_{}; + + ///@} + /// @name Options + ///@{ + + /// The default values for options, customizable and changeable INHERITABLE + OptionDefaults option_defaults_{}; + + /// The list of options, stored locally + std::vector options_{}; + + ///@} + /// @name Help + ///@{ + + /// Footer to put after all options in the help output INHERITABLE + std::string footer_{}; + + /// This is a function that generates a footer to put after all other options in help output + std::function footer_callback_{}; + + /// A pointer to the help flag if there is one INHERITABLE + Option *help_ptr_{nullptr}; + + /// A pointer to the help all flag if there is one INHERITABLE + Option *help_all_ptr_{nullptr}; + + /// A pointer to a version flag if there is one + Option *version_ptr_{nullptr}; + + /// This is the formatter for help printing. Default provided. INHERITABLE (same pointer) + std::shared_ptr formatter_{new Formatter()}; + + /// The error message printing function INHERITABLE + std::function failure_message_{FailureMessage::simple}; + + ///@} + /// @name Parsing + ///@{ + + using missing_t = std::vector>; + + /// Pair of classifier, string for missing options. (extra detail is removed on returning from parse) + /// + /// This is faster and cleaner than storing just a list of strings and reparsing. This may contain the -- separator. + missing_t missing_{}; + + /// This is a list of pointers to options with the original parse order + std::vector