From 8df83d562a1811dda5ef6b5541eaa37f47974e18 Mon Sep 17 00:00:00 2001 From: Tim Stack Date: Mon, 10 Jul 2023 10:35:18 -0700 Subject: [PATCH] [config] simplify file format detection by moving it into the log format definition --- .gitignore | 1 + docs/schemas/config-v1.schema.json | 53 ---- docs/schemas/format-v1.schema.json | 52 +++- docs/source/config.rst | 4 - docs/source/formats.rst | 52 ++-- src/CMakeLists.txt | 4 +- src/Makefile.am | 1 - src/file_collection.cc | 2 +- src/file_converter_manager.cc | 10 +- src/file_format.cc | 239 +----------------- src/file_format.cfg.hh | 63 ----- src/file_format.hh | 21 +- src/formats/pcap_log.json | 13 +- src/lnav.cc | 10 +- src/lnav_config.cc | 57 +---- src/lnav_config.hh | 2 - src/log_format.cc | 212 +++++++++++++++- src/log_format.hh | 8 - src/log_format_ext.hh | 20 +- src/log_format_loader.cc | 52 +++- src/logfile.cc | 6 +- src/logfile_fwd.hh | 2 +- src/root-config.json | 13 - ...cap-converter.sh => pcap_log-converter.sh} | 0 src/scripts/scripts.am | 2 +- src/view_curses.cc | 7 +- src/view_curses.hh | 3 +- src/view_helpers.cc | 4 +- test/Makefile.am | 3 +- .../formats/invalid-file-format/format.json | 27 ++ .../configs/invalid-file-format/config.json | 19 -- .../configs/invalid-theme/config.json | 15 ++ ...907769aba112d628e7ebe39c4ec252e5e0bc69.err | 47 ++-- ...92e2695b7e6cf1f3520dbb87af8fc2b8f27088.err | 7 + 34 files changed, 450 insertions(+), 581 deletions(-) delete mode 100644 src/file_format.cfg.hh rename src/scripts/{pcap-converter.sh => pcap_log-converter.sh} (100%) create mode 100644 test/bad-config/formats/invalid-file-format/format.json delete mode 100644 test/bad-config2/configs/invalid-file-format/config.json create mode 100644 test/bad-config2/configs/invalid-theme/config.json diff --git a/.gitignore b/.gitignore index 100682ca..dbfb3d94 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .deps .lnav +.DS_Store *.dat *.diff test/*.err diff --git a/docs/schemas/config-v1.schema.json b/docs/schemas/config-v1.schema.json index 0c7001b8..1019772d 100644 --- a/docs/schemas/config-v1.schema.json +++ b/docs/schemas/config-v1.schema.json @@ -73,59 +73,6 @@ }, "additionalProperties": false }, - "file-format": { - "description": "sdfjdls", - "title": "/tuning/file-format", - "type": "object", - "patternProperties": { - "(\\w+~1[\\w\\.\\-]+)": { - "description": "File format definitions, keyed by their MIME type", - "title": "/tuning/file-format/", - "type": "object", - "properties": { - "title": { - "title": "/tuning/file-format//title", - "description": "The display name for this file format", - "type": "string" - }, - "header": { - "description": "File header detection definitions", - "title": "/tuning/file-format//header", - "type": "object", - "properties": { - "expr": { - "description": "The expressions used to check if a file header matches this file format", - "title": "/tuning/file-format//header/expr", - "type": "object", - "patternProperties": { - "(\\w+)": { - "title": "/tuning/file-format//header/expr/", - "description": "SQLite expression", - "type": "string" - } - }, - "additionalProperties": false - }, - "size": { - "title": "/tuning/file-format//header/size", - "description": "The minimum size required for this header type", - "type": "integer" - } - }, - "additionalProperties": false - }, - "converter": { - "title": "/tuning/file-format//converter", - "description": "The script used to convert the file", - "type": "string", - "pattern": "[\\w\\.\\-]+" - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false - }, "logfile": { "description": "Settings related to log files", "title": "/tuning/logfile", diff --git a/docs/schemas/format-v1.schema.json b/docs/schemas/format-v1.schema.json index ab449515..9e75876a 100644 --- a/docs/schemas/format-v1.schema.json +++ b/docs/schemas/format-v1.schema.json @@ -75,14 +75,50 @@ "description": "A regular expression that restricts this format to log files with a matching name", "type": "string" }, - "mime-types": { - "title": "//mime-types", - "description": "A list of mime-types this format should be used for", - "type": "array", - "items": { - "type": "string", - "pattern": "^\\w/[\\w\\.]+" - } + "converter": { + "description": "Describes how the file format can be detected and converted to a log that can be understood by lnav", + "title": "//converter", + "type": "object", + "properties": { + "type": { + "title": "//converter/type", + "description": "The MIME type", + "type": "string" + }, + "header": { + "description": "File header detection definitions", + "title": "//converter/header", + "type": "object", + "properties": { + "expr": { + "description": "The expressions used to check if a file header matches this file format", + "title": "//converter/header/expr", + "type": "object", + "patternProperties": { + "(\\w+)": { + "title": "//converter/header/expr/", + "description": "SQLite expression", + "type": "string" + } + }, + "additionalProperties": false + }, + "size": { + "title": "//converter/header/size", + "description": "The minimum size required for this header type", + "type": "integer" + } + }, + "additionalProperties": false + }, + "command": { + "title": "//converter/command", + "description": "The script used to convert the file", + "type": "string", + "pattern": "[\\w\\.\\-]+" + } + }, + "additionalProperties": false }, "level-field": { "title": "//level-field", diff --git a/docs/source/config.rst b/docs/source/config.rst index 8c0a0027..771c5f36 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -260,10 +260,6 @@ command. .. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/clipboard -.. _tuning_file_format: - -.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/file-format - .. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/piper .. jsonschema:: ../schemas/config-v1.schema.json#/definitions/clip-commands diff --git a/docs/source/formats.rst b/docs/source/formats.rst index c069c91c..d30ba8c3 100644 --- a/docs/source/formats.rst +++ b/docs/source/formats.rst @@ -133,11 +133,29 @@ object with the following fields: :json: True if each log line is JSON-encoded. -:mime-types: An array of MIME types that this log format should only be - used with. These MIME types refer to file formats that are defined - using the `Automatic File Conversion`_ feature. This property should - not be defined for log files that do not require conversion and can be - naturally parsed using regexes or are JSON-lines. +:converter: An object that describes how an input file can be detected and + then converted to a form that can be interpreted by **lnav**. For + example, a PCAP file is in a binary format that cannot be handled natively + by **lnav**. However, a PCAP file can be converted by :file:`tshark` + into JSON-lines that can be handled by **lnav**. So, this configuration + describes how the input file format can be detected and converted. See + `Automatic File Conversion`_ for more information. + + :header: An object that describes how to match the header of the input + file. + + :expr: An object that contains SQLite expressions that can be used to + check if the input file's header is of this type. The property + name is the name of the expression and the value is the expression. + The expression is evaluated with the following variables: + + :\:header: The hex-encoded version of the header content. + + :\:filepath: The path to the input file. + + :size: The minimum size of header that is needed to do the match. + + :command: The command to execute to convert the input file. :line-format: An array that specifies the text format for JSON-encoded log messages. Log files that are JSON-encoded will have each message @@ -571,23 +589,25 @@ Automatic File Conversion File formats that are not naturally understood by **lnav** can be automatically detected and converted to a usable form using the -:ref:`tuning_file_format` configuration options. For example, -PCAP files can be detected and converted to a JSON-lines form -using :code:`tshark`. The conversion process works as follows: +:code:`converter` property. For example, PCAP files can be +detected and converted to a JSON-lines form using :code:`tshark`. +The conversion process works as follows: #. The first 1024 bytes of the file are read, if available. #. This header is converted into a hex string. -#. For each file-format, every "header expression" is evaluated - to see if there is a match. The header expressions are - SQLite expressions where the following variables are defined: +#. For each log format that has defined a :code:`converter`, + every "header expression" is evaluated to see if there is a + match. The header expressions are SQLite expressions where + the following variables are defined: :\:header: A string containing the header as a hex string. + :\:filepath: The path to the file. + #. If a match is found, the converter script defined in the - file format will be invoked and passed the format MIME type - and path to the file as arguments. The script should write + log format will be invoked and passed the format name and + path to the file as arguments. The script should write the converted form of the input file on its standard output. Any errors should be written to the standard error. -#. The MIME type will be associated with the original file and - only log formats that have the corresponding type will be - used to interpret the file. +#. The log format will be associated with the original file will + be used to interpret the converted file. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 83aafb5a..d112b0a0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -204,8 +204,7 @@ add_custom_command( DEPENDS bin2c ${BUILTIN_LNAV_SCRIPTS}) list(APPEND GEN_SRCS builtin-scripts.h builtin-scripts.cc) -set(BUILTIN_SH_SCRIPTS scripts/dhclient-summary.lnav scripts/lnav-pop-view.lnav - scripts/partition-by-boot.lnav scripts/search-for.lnav) +set(BUILTIN_SH_SCRIPTS scripts/dump-pid.sh scripts/pcap_log-converter.sh) set(BUILTIN_SH_SCRIPT_PATHS ${BUILTIN_SH_SCRIPTS}) @@ -465,7 +464,6 @@ add_library( file_collection.hh file_converter_manager.hh file_format.hh - file_format.cfg.hh files_sub_source.hh filter_observer.hh filter_status_source.hh diff --git a/src/Makefile.am b/src/Makefile.am index a97ac930..c95cb59f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -203,7 +203,6 @@ noinst_HEADERS = \ file_collection.hh \ file_converter_manager.hh \ file_format.hh \ - file_format.cfg.hh \ file_vtab.cfg.hh \ files_sub_source.hh \ filter_observer.hh \ diff --git a/src/file_collection.cc b/src/file_collection.cc index 34745457..202f708f 100644 --- a/src/file_collection.cc +++ b/src/file_collection.cc @@ -454,7 +454,7 @@ file_collection::watch_logfile(const std::string& filename, }); loo.with_filename(filename); loo.with_stat_for_temp(st); - loo.loo_mime_type = eff->eff_mime_type; + loo.loo_format_name = eff->eff_format_name; filename_to_open = convert_res.cr_destination; } } diff --git a/src/file_converter_manager.cc b/src/file_converter_manager.cc index d34de0ce..ee5d7820 100644 --- a/src/file_converter_manager.cc +++ b/src/file_converter_manager.cc @@ -69,11 +69,11 @@ convert(const external_file_format& eff, const std::string& filename) log_info("invoking converter: %s (PATH=%s)", eff.eff_converter.c_str(), new_path.c_str()); - auto mime_str = eff.eff_mime_type.to_string(); + auto format_str = eff.eff_format_name; const char* args[] = { eff.eff_converter.c_str(), - mime_str.c_str(), + format_str.c_str(), filename.c_str(), nullptr, }; @@ -95,6 +95,7 @@ convert(const external_file_format& eff, const std::string& filename) auto error_queue = std::make_shared>(); std::thread err_reader([err = std::move(err_pipe.read_end()), + converter = eff.eff_converter, error_queue, child_pid = child.in()]() mutable { line_buffer lb; @@ -115,7 +116,7 @@ convert(const external_file_format& eff, const std::string& filename) done = true; } else { lb.read_range(li.li_file_range) - .then([error_queue, child_pid](auto sbr) { + .then([converter, error_queue, child_pid](auto sbr) { auto line_str = string_fragment( sbr.get_data(), 0, sbr.length()) .trim("\n"); @@ -123,7 +124,8 @@ convert(const external_file_format& eff, const std::string& filename) error_queue->emplace_back(line_str.to_string()); } - log_debug("pcap[%d]: %.*s", + log_debug("%s[%d]: %.*s", + converter.c_str(), child_pid, line_str.length(), line_str.data()); diff --git a/src/file_format.cc b/src/file_format.cc index 3fa72130..12582629 100644 --- a/src/file_format.cc +++ b/src/file_format.cc @@ -31,20 +31,11 @@ #include "file_format.hh" -#include - #include "archive_manager.hh" #include "base/auto_fd.hh" #include "base/fs_util.hh" -#include "base/injector.hh" #include "base/intern_string.hh" -#include "base/lnav_log.hh" #include "config.h" -#include "lnav_config.hh" -#include "readline_highlighters.hh" -#include "safe/safe.h" -#include "sql_util.hh" -#include "sqlite-extension-func.hh" file_format_t detect_file_format(const ghc::filesystem::path& filename) @@ -62,7 +53,7 @@ detect_file_format(const ghc::filesystem::path& filename) if ((rc = read(fd, buffer, sizeof(buffer))) > 0) { static auto SQLITE3_HEADER = "SQLite format 3"; - auto header_frag = string_fragment(buffer, 0, rc); + auto header_frag = string_fragment::from_bytes(buffer, rc); if (header_frag.startswith(SQLITE3_HEADER)) { retval = file_format_t::SQLITE_DB; @@ -72,231 +63,3 @@ detect_file_format(const ghc::filesystem::path& filename) return retval; } - -mime_type -mime_type::from_str(const std::string& str) -{ - auto slash_index = str.find('/'); - - if (slash_index == std::string::npos) { - return {"application", str}; - } - - return {str.substr(0, slash_index), str.substr(slash_index + 1)}; -} - -struct compiled_header_expr { - auto_mem che_stmt{sqlite3_finalize}; - bool che_enabled{true}; -}; - -struct file_format_expressions : public lnav_config_listener { - void reload_config(error_reporter& reporter) override - { - log_debug("reloading file-format header expressions"); - - safe::WriteAccess> in(instance); - - if (in->e_db.in() == nullptr) { - if (sqlite3_open(":memory:", in->e_db.out()) != SQLITE_OK) { - log_error("unable to open memory DB"); - return; - } - register_sqlite_funcs(in->e_db.in(), sqlite_registration_funcs); - } - - in->e_header_exprs.clear(); - const auto& cfg = injector::get(); - for (const auto& fpair : cfg.c_defs) { - for (const auto& hpair : fpair.second.fd_header.h_exprs.he_exprs) { - auto stmt_str = fmt::format(FMT_STRING("SELECT 1 WHERE {}"), - hpair.second); - compiled_header_expr che; - - log_info("preparing file-format header expression: %s", - stmt_str.c_str()); - auto retcode = sqlite3_prepare_v2(in->e_db.in(), - stmt_str.c_str(), - stmt_str.size(), - che.che_stmt.out(), - nullptr); - if (retcode != SQLITE_OK) { - auto sql_al - = attr_line_t(hpair.second) - .with_attr_for_all(SA_PREFORMATTED.value()) - .with_attr_for_all( - VC_ROLE.value(role_t::VCR_QUOTED_CODE)); - readline_sqlite_highlighter(sql_al, -1); - intern_string_t watch_expr_path - = intern_string::lookup(fmt::format( - FMT_STRING( - "/tuning/file-formats/{}/header/expr/{}"), - json_ptr::encode_str(fpair.first.c_str()), - hpair.first)); - auto snippet = lnav::console::snippet::from( - source_location(watch_expr_path), sql_al); - - auto um = lnav::console::user_message::error( - "SQL expression is invalid") - .with_reason(sqlite3_errmsg(in->e_db.in())) - .with_snippet(snippet); - - reporter(&hpair.second, um); - continue; - } - - in->e_header_exprs[fpair.first][hpair.first] = std::move(che); - } - - if (fpair.second.fd_header.h_exprs.he_exprs.empty()) { - auto um - = lnav::console::user_message::error( - "At least one header expression is required for " - "a file format") - .with_reason( - "Header expressions are used to detect a format"); - reporter(&fpair.second.fd_header.h_exprs, um); - } - if (fpair.second.fd_converter.pp_value.empty()) { - auto um = lnav::console::user_message::error( - "A converter is required for a file format") - .with_reason( - "The converter script transforms the file " - "into a format that can be consumed by lnav"); - reporter(&fpair.second.fd_converter, um); - } - } - } - - void unload_config() override - { - safe::WriteAccess> in(instance); - - in->e_header_exprs.clear(); - } - - struct inner { - auto_sqlite3 e_db; - std::map> - e_header_exprs; - }; - - safe::Safe instance; -}; - -static file_format_expressions format_exprs; - -nonstd::optional -detect_mime_type(const ghc::filesystem::path& filename) -{ - uint8_t buffer[1024]; - size_t buffer_size = 0; - - { - auto_fd fd; - - if ((fd = lnav::filesystem::openp(filename, O_RDONLY)) == -1) { - return nonstd::nullopt; - } - - ssize_t rc; - - if ((rc = read(fd, buffer, sizeof(buffer))) == -1) { - return nonstd::nullopt; - } - buffer_size = rc; - } - - auto hexbuf = auto_buffer::alloc(buffer_size * 2); - - for (int lpc = 0; lpc < buffer_size; lpc++) { - fmt::format_to( - std::back_inserter(hexbuf), FMT_STRING("{:02x}"), buffer[lpc]); - } - - safe::WriteAccess> in( - format_exprs.instance); - - const auto& cfg = injector::get(); - for (const auto& fpair : cfg.c_defs) { - if (buffer_size < fpair.second.fd_header.h_size) { - log_debug("file content too small (%d) for header detection: %s", - buffer_size, - fpair.first.c_str()); - continue; - } - for (const auto& hpair : fpair.second.fd_header.h_exprs.he_exprs) { - auto& he = in->e_header_exprs[fpair.first][hpair.first]; - - if (!he.che_enabled) { - continue; - } - - auto* stmt = he.che_stmt.in(); - - if (stmt == nullptr) { - continue; - } - sqlite3_reset(stmt); - auto count = sqlite3_bind_parameter_count(stmt); - for (int lpc = 0; lpc < count; lpc++) { - const auto* name = sqlite3_bind_parameter_name(stmt, lpc + 1); - - if (name[0] == '$') { - const char* env_value; - - if ((env_value = getenv(&name[1])) != nullptr) { - sqlite3_bind_text( - stmt, lpc + 1, env_value, -1, SQLITE_STATIC); - } - continue; - } - if (strcmp(name, ":header") == 0) { - sqlite3_bind_text(stmt, - lpc + 1, - hexbuf.in(), - hexbuf.size(), - SQLITE_STATIC); - continue; - } - if (strcmp(name, ":filepath") == 0) { - sqlite3_bind_text( - stmt, lpc + 1, filename.c_str(), -1, SQLITE_STATIC); - continue; - } - } - - auto step_res = sqlite3_step(stmt); - - switch (step_res) { - case SQLITE_OK: - case SQLITE_DONE: - continue; - case SQLITE_ROW: - break; - default: { - log_error( - "failed to execute file-format header expression: " - "%s:%s -- %s", - fpair.first.c_str(), - hpair.first.c_str(), - sqlite3_errmsg(in->e_db)); - he.che_enabled = false; - continue; - } - } - - log_info("detected MIME type for: %s -- %s (header-expr: %s)", - filename.c_str(), - fpair.first.c_str(), - hpair.first.c_str()); - return external_file_format{ - mime_type::from_str(fpair.first), - fpair.second.fd_converter.pp_value, - fpair.second.fd_converter.pp_location.sl_source.to_string(), - }; - } - } - - return nonstd::nullopt; -} diff --git a/src/file_format.cfg.hh b/src/file_format.cfg.hh deleted file mode 100644 index 2f15e28e..00000000 --- a/src/file_format.cfg.hh +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright (c) 2023, Timothy Stack - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * * Neither the name of Timothy Stack nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef lnav_file_format_cfg_hh -#define lnav_file_format_cfg_hh - -#include -#include - -#include "yajlpp/yajlpp.hh" - -namespace lnav { -namespace file_formats { - -struct header_exprs { - std::map he_exprs; -}; - -struct header { - header_exprs h_exprs; - size_t h_size{32}; -}; - -struct format_def { - std::string fd_title; - header fd_header; - positioned_property fd_converter; -}; - -struct config { - std::map c_defs; -}; - -} // namespace file_formats -} // namespace lnav - -#endif diff --git a/src/file_format.hh b/src/file_format.hh index 283a7fa6..d74347be 100644 --- a/src/file_format.hh +++ b/src/file_format.hh @@ -43,27 +43,8 @@ enum class file_format_t : int { REMOTE, }; -struct mime_type { - static mime_type from_str(const std::string& str); - - std::string mt_type; - std::string mt_subtype; - - bool operator<(const mime_type& other) const - { - return this->mt_type < other.mt_type - && this->mt_subtype < other.mt_subtype; - } - - std::string to_string() const - { - return fmt::format( - FMT_STRING("{}/{}"), this->mt_type, this->mt_subtype); - } -}; - struct external_file_format { - mime_type eff_mime_type; + std::string eff_format_name; std::string eff_converter; ghc::filesystem::path eff_source_path; }; diff --git a/src/formats/pcap_log.json b/src/formats/pcap_log.json index 8ae73e2e..93b110ec 100644 --- a/src/formats/pcap_log.json +++ b/src/formats/pcap_log.json @@ -4,11 +4,18 @@ "json": true, "title": "Packet Capture", "description": "Internal format for pcap files", - "mime-types": [ - "application/vnd.tcpdump.pcap" - ], "multiline": false, "convert-to-local-time": true, + "converter": { + "header": { + "expr": { + "pcapng": ":header REGEXP '^0a0d0d0a.{8}(?:1a2b3c4d|4d3c2b1a).*'", + "pcap": ":header REGEXP '^(?:a1b2c3d4|d4c3b2a1|a1b23c4d|4d3cb2a1).*'" + }, + "size": 24 + }, + "command": "pcap_log-converter.sh" + }, "line-format": [ { "field": "time" diff --git a/src/lnav.cc b/src/lnav.cc index 9f4fc542..bc6437c7 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -1002,7 +1002,13 @@ wait_for_pipers() log_debug("all pipers finished"); break; } - std::this_thread::sleep_for(sleep_time); + // Use usleep() since it is defined to be interruptable by a signal. + auto urc = usleep( + std::chrono::duration_cast(sleep_time) + .count()); + if (urc == -1 && errno == EINTR) { + log_trace("wait_for_pipers(): sleep interrupted"); + } rebuild_indexes(); log_debug("%d pipers and %d children are still active", @@ -1201,7 +1207,6 @@ looper() (void) signal(SIGINT, sigint); (void) signal(SIGTERM, sigint); (void) signal(SIGWINCH, sigwinch); - (void) signal(SIGCHLD, sigchld); auto create_screen_res = screen_curses::create(); @@ -2115,6 +2120,7 @@ main(int argc, char* argv[]) } (void) signal(SIGPIPE, SIG_IGN); + (void) signal(SIGCHLD, sigchld); setlocale(LC_ALL, ""); try { std::locale::global(std::locale("")); diff --git a/src/lnav_config.cc b/src/lnav_config.cc index d72b5ee8..d8144c60 100644 --- a/src/lnav_config.cc +++ b/src/lnav_config.cc @@ -81,9 +81,6 @@ lnav_config_listener* lnav_config_listener::LISTENER_LIST; static auto a = injector::bind::to_instance( +[]() { return &lnav_config.lc_archive_manager; }); -static auto ff = injector::bind::to_instance( - +[]() { return &lnav_config.lc_file_formats; }); - static auto fvc = injector::bind::to_instance( +[]() { return &lnav_config.lc_file_vtab; }); @@ -1043,55 +1040,6 @@ static const struct json_path_container ui_handlers = { .with_children(keymap_defs_handlers), }; -static const struct json_path_container header_expr_handlers = { - yajlpp::pattern_property_handler("(?\\w+)") - .with_description("SQLite expression") - .for_field(&lnav::file_formats::header_exprs::he_exprs), -}; - -static const struct json_path_container header_handlers = { - yajlpp::property_handler("expr") - .with_description("The expressions used to check if a file header " - "matches this file format") - .for_child(&lnav::file_formats::header::h_exprs) - .with_children(header_expr_handlers), - yajlpp::property_handler("size") - .with_description("The minimum size required for this header type") - .for_field(&lnav::file_formats::header::h_size), -}; - -static const struct json_path_container format_def_handlers = { - yajlpp::property_handler("title") - .with_description("The display name for this file format") - .for_field(&lnav::file_formats::format_def::fd_title), - yajlpp::property_handler("header") - .with_description("File header detection definitions") - .for_child(&lnav::file_formats::format_def::fd_header) - .with_children(header_handlers), - yajlpp::property_handler("converter") - .with_description("The script used to convert the file") - .with_pattern(R"([\w\.\-]+)") - .for_field(&lnav::file_formats::format_def::fd_converter), -}; - -static const struct json_path_container format_defs_handlers = { - yajlpp::pattern_property_handler(R"((?\w+~1[\w\.\-]+))") - .with_description("File format definitions, keyed by their MIME type") - .with_obj_provider( - [](const yajlpp_provider_context& ypc, _lnav_config* root) { - auto& retval - = root->lc_file_formats.c_defs[ypc.get_substr("mime_type")]; - return &retval; - }) - .with_path_provider<_lnav_config>( - [](struct _lnav_config* cfg, std::vector& paths_out) { - for (const auto& iter : cfg->lc_file_formats.c_defs) { - paths_out.emplace_back(iter.first); - } - }) - .with_children(format_def_handlers), -}; - static const struct json_path_container archive_handlers = { yajlpp::property_handler("min-free-space") .with_synopsis("") @@ -1319,9 +1267,6 @@ static const struct json_path_container tuning_handlers = { yajlpp::property_handler("file-vtab") .with_description("Settings related to the lnav_file virtual-table") .with_children(file_vtab_handlers), - yajlpp::property_handler("file-format") - .with_description("sdfjdls") - .with_children(format_defs_handlers), yajlpp::property_handler("logfile") .with_description("Settings related to log files") .with_children(logfile_handlers), @@ -1706,6 +1651,8 @@ reload_config(std::vector& errors) while (curr != nullptr) { auto reporter = [&errors](const void* cfg_value, const lnav::console::user_message& errmsg) { + log_error("configuration error: %s", + errmsg.to_attr_line().get_string().c_str()); auto cb = [&cfg_value, &errors, &errmsg]( const json_path_handler_base& jph, const std::string& path, diff --git a/src/lnav_config.hh b/src/lnav_config.hh index 714bc53e..e38f8483 100644 --- a/src/lnav_config.hh +++ b/src/lnav_config.hh @@ -43,7 +43,6 @@ #include "base/file_range.hh" #include "base/lnav.console.hh" #include "base/result.h" -#include "file_format.cfg.hh" #include "file_vtab.cfg.hh" #include "ghc/filesystem.hpp" #include "lnav_config_fwd.hh" @@ -112,7 +111,6 @@ struct _lnav_config { archive_manager::config lc_archive_manager; lnav::piper::config lc_piper; - lnav::file_formats::config lc_file_formats; file_vtab::config lc_file_vtab; lnav::logfile::config lc_logfile; tailer::config lc_tailer; diff --git a/src/log_format.cc b/src/log_format.cc index 81db82ff..2bbd108f 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -33,6 +33,7 @@ #include #include +#include "base/fs_util.hh" #include "base/is_utf8.hh" #include "base/snippet_highlighters.hh" #include "base/string_util.hh" @@ -44,8 +45,10 @@ #include "log_search_table.hh" #include "log_vtab_impl.hh" #include "ptimec.hh" +#include "readline_highlighters.hh" #include "scn/scn.h" #include "sql_util.hh" +#include "sqlite-extension-func.hh" #include "yajlpp/yajlpp.hh" #include "yajlpp/yajlpp_def.hh" @@ -1836,6 +1839,139 @@ external_log_format::get_subline(const logline& ll, this->jlf_line_values.lvv_sbr = sbr; } +struct compiled_header_expr { + auto_mem che_stmt{sqlite3_finalize}; + bool che_enabled{true}; +}; + +struct format_header_expressions : public lnav_config_listener { + auto_sqlite3 e_db; + std::map> + e_header_exprs; +}; + +using safe_format_header_expressions = safe::Safe; + +static safe_format_header_expressions format_header_exprs; + +nonstd::optional +detect_mime_type(const ghc::filesystem::path& filename) +{ + uint8_t buffer[1024]; + size_t buffer_size = 0; + + { + auto_fd fd; + + if ((fd = lnav::filesystem::openp(filename, O_RDONLY)) == -1) { + return nonstd::nullopt; + } + + ssize_t rc; + + if ((rc = read(fd, buffer, sizeof(buffer))) == -1) { + return nonstd::nullopt; + } + buffer_size = rc; + } + + auto hexbuf = auto_buffer::alloc(buffer_size * 2); + + for (int lpc = 0; lpc < buffer_size; lpc++) { + fmt::format_to( + std::back_inserter(hexbuf), FMT_STRING("{:02x}"), buffer[lpc]); + } + + safe::WriteAccess in(format_header_exprs); + + for (const auto& format : log_format::get_root_formats()) { + auto elf = std::dynamic_pointer_cast(format); + if (elf == nullptr) { + continue; + } + + if (buffer_size < elf->elf_converter.c_header.h_size) { + log_debug("file content too small (%d) for header detection: %s", + buffer_size, + elf->get_name().get()); + continue; + } + for (const auto& hpair : elf->elf_converter.c_header.h_exprs.he_exprs) { + auto& he = in->e_header_exprs[elf->get_name()][hpair.first]; + + if (!he.che_enabled) { + continue; + } + + auto* stmt = he.che_stmt.in(); + + if (stmt == nullptr) { + continue; + } + sqlite3_reset(stmt); + auto count = sqlite3_bind_parameter_count(stmt); + for (int lpc = 0; lpc < count; lpc++) { + const auto* name = sqlite3_bind_parameter_name(stmt, lpc + 1); + + if (name[0] == '$') { + const char* env_value; + + if ((env_value = getenv(&name[1])) != nullptr) { + sqlite3_bind_text( + stmt, lpc + 1, env_value, -1, SQLITE_STATIC); + } + continue; + } + if (strcmp(name, ":header") == 0) { + sqlite3_bind_text(stmt, + lpc + 1, + hexbuf.in(), + hexbuf.size(), + SQLITE_STATIC); + continue; + } + if (strcmp(name, ":filepath") == 0) { + sqlite3_bind_text( + stmt, lpc + 1, filename.c_str(), -1, SQLITE_STATIC); + continue; + } + } + + auto step_res = sqlite3_step(stmt); + + switch (step_res) { + case SQLITE_OK: + case SQLITE_DONE: + continue; + case SQLITE_ROW: + break; + default: { + log_error( + "failed to execute file-format header expression: " + "%s:%s -- %s", + elf->get_name().get(), + hpair.first.c_str(), + sqlite3_errmsg(in->e_db)); + he.che_enabled = false; + continue; + } + } + + log_info("detected format for: %s -- %s (header-expr: %s)", + filename.c_str(), + elf->get_name().get(), + hpair.first.c_str()); + return external_file_format{ + elf->get_name().to_string(), + elf->elf_converter.c_command.pp_value, + elf->elf_converter.c_command.pp_location.sl_source.to_string(), + }; + } + } + + return nonstd::nullopt; +} + void external_log_format::build(std::vector& errors) { @@ -2013,6 +2149,70 @@ external_log_format::build(std::vector& errors) stable_sort(this->elf_level_pairs.begin(), this->elf_level_pairs.end()); + { + safe::WriteAccess hexprs( + format_header_exprs); + + if (hexprs->e_db.in() == nullptr) { + if (sqlite3_open(":memory:", hexprs->e_db.out()) != SQLITE_OK) { + log_error("unable to open memory DB"); + return; + } + register_sqlite_funcs(hexprs->e_db.in(), sqlite_registration_funcs); + } + + for (const auto& hpair : this->elf_converter.c_header.h_exprs.he_exprs) + { + auto stmt_str + = fmt::format(FMT_STRING("SELECT 1 WHERE {}"), hpair.second); + compiled_header_expr che; + + log_info("preparing file-format header expression: %s", + stmt_str.c_str()); + auto retcode = sqlite3_prepare_v2(hexprs->e_db.in(), + stmt_str.c_str(), + stmt_str.size(), + che.che_stmt.out(), + nullptr); + if (retcode != SQLITE_OK) { + auto sql_al = attr_line_t(hpair.second) + .with_attr_for_all(SA_PREFORMATTED.value()) + .with_attr_for_all( + VC_ROLE.value(role_t::VCR_QUOTED_CODE)); + readline_sqlite_highlighter(sql_al, -1); + intern_string_t watch_expr_path = intern_string::lookup( + fmt::format(FMT_STRING("/{}/converter/header/expr/{}"), + this->elf_name, + hpair.first)); + auto snippet = lnav::console::snippet::from( + source_location(watch_expr_path), sql_al); + + auto um = lnav::console::user_message::error( + "SQL expression is invalid") + .with_reason(sqlite3_errmsg(hexprs->e_db.in())) + .with_snippet(snippet); + + errors.emplace_back(um); + continue; + } + + hexprs->e_header_exprs[this->elf_name][hpair.first] + = std::move(che); + } + + if (!this->elf_converter.c_header.h_exprs.he_exprs.empty() + && this->elf_converter.c_command.pp_value.empty()) + { + auto um = lnav::console::user_message::error( + "A command is required when a converter is defined") + .with_help( + "The converter command transforms the file " + "into a format that can be consumed by lnav") + .with_snippets(this->get_snippets()); + errors.emplace_back(um); + } + } + for (auto& vd : this->elf_value_def_order) { std::vector::iterator act_iter; @@ -2949,18 +3149,6 @@ external_log_format::match_name(const std::string& filename) .has_value(); } -bool -external_log_format::match_mime_type(const mime_type& mt) const -{ - if (mt.mt_type == "text" && mt.mt_subtype == "plain" - && this->elf_mime_types.empty()) - { - return true; - } - - return this->elf_mime_types.count(mt) == 1; -} - auto external_log_format::value_line_count(const intern_string_t ist, bool top_level, diff --git a/src/log_format.hh b/src/log_format.hh index d379ca7f..f4cf88da 100644 --- a/src/log_format.hh +++ b/src/log_format.hh @@ -352,14 +352,6 @@ public: virtual bool match_name(const std::string& filename) { return true; } - virtual bool match_mime_type(const mime_type& mt) const - { - if (mt.mt_type == "text" && mt.mt_subtype == "plain") { - return true; - } - return false; - } - struct scan_match { uint32_t sm_quality; }; diff --git a/src/log_format_ext.hh b/src/log_format_ext.hh index e0803e01..a72ca1ec 100644 --- a/src/log_format_ext.hh +++ b/src/log_format_ext.hh @@ -141,8 +141,6 @@ public: bool match_name(const std::string& filename) override; - bool match_mime_type(const mime_type& mt) const override; - scan_result_t scan(logfile& lf, std::vector& dst, const line_info& offset, @@ -313,7 +311,6 @@ public: std::vector elf_format_source_order; std::map elf_format_sources; std::list elf_collision; - std::set elf_mime_types; factory_container elf_filename_pcre; std::map> elf_patterns; std::vector> elf_pattern_order; @@ -344,6 +341,23 @@ public: bool elf_has_module_format{false}; bool elf_builtin_format{false}; + struct header_exprs { + std::map he_exprs; + }; + + struct header { + header_exprs h_exprs; + size_t h_size{32}; + }; + + struct converter { + std::string c_type; + header c_header; + positioned_property c_command; + }; + + converter elf_converter; + using search_table_pcre2pp = factory_container::with_default_args< log_search_table_ns::PATTERN_OPTIONS>; diff --git a/src/log_format_loader.cc b/src/log_format_loader.cc index d35c2173..7ff102db 100644 --- a/src/log_format_loader.cc +++ b/src/log_format_loader.cc @@ -281,8 +281,6 @@ read_format_field(yajlpp_parse_context* ypc, } else if (field_name == "module-field") { elf->elf_module_id_field = intern_string::lookup(value); elf->elf_container = true; - } else if (field_name == "mime-types") { - elf->elf_mime_types.insert(mime_type::from_str(value)); } return 1; @@ -817,6 +815,37 @@ static const struct json_path_container search_table_handlers = { .with_children(search_table_def_handlers), }; +static const struct json_path_container header_expr_handlers = { + yajlpp::pattern_property_handler(R"((?\w+))") + .with_description("SQLite expression") + .for_field(&external_log_format::header_exprs::he_exprs), +}; + +static const struct json_path_container header_handlers = { + yajlpp::property_handler("expr") + .with_description("The expressions used to check if a file header " + "matches this file format") + .for_child(&external_log_format::header::h_exprs) + .with_children(header_expr_handlers), + yajlpp::property_handler("size") + .with_description("The minimum size required for this header type") + .for_field(&external_log_format::header::h_size), +}; + +static const struct json_path_container converter_handlers = { + yajlpp::property_handler("type") + .with_description("The MIME type") + .for_field(&external_log_format::converter::c_type), + yajlpp::property_handler("header") + .with_description("File header detection definitions") + .for_child(&external_log_format::converter::c_header) + .with_children(header_handlers), + yajlpp::property_handler("command") + .with_description("The script used to convert the file") + .with_pattern(R"([\w\.\-]+)") + .for_field(&external_log_format::converter::c_command), +}; + const struct json_path_container format_handlers = { yajlpp::property_handler("regex") .with_description( @@ -826,14 +855,16 @@ const struct json_path_container format_handlers = { json_path_handler("json", read_format_bool) .with_description( R"(Indicates that log files are JSON-encoded (deprecated, use "file-type": "json"))"), - json_path_handler("convert-to-local-time", read_format_bool) + json_path_handler("convert-to-local-time") .with_description("Indicates that displayed timestamps should " - "automatically be converted to local time"), - json_path_handler("hide-extra", read_format_bool) + "automatically be converted to local time") + .for_field(&external_log_format::lf_date_time, + &date_time_scanner::dts_local_time), + json_path_handler("hide-extra") .with_description( "Specifies whether extra values in JSON logs should be displayed") .for_field(&external_log_format::jlf_hide_extra), - json_path_handler("multiline", read_format_bool) + json_path_handler("multiline") .with_description("Indicates that log messages can span multiple lines") .for_field(&log_format::lf_multiline), json_path_handler("timestamp-divisor", read_format_double) @@ -845,10 +876,11 @@ const struct json_path_container format_handlers = { .with_description("A regular expression that restricts this format to " "log files with a matching name") .for_field(&external_log_format::elf_filename_pcre), - json_path_handler("mime-types#", read_format_field) - .with_pattern(R"(^\w/[\w\.]+)") - .with_description( - "A list of mime-types this format should be used for"), + json_path_handler("converter") + .with_description("Describes how the file format can be detected and " + "converted to a log that can be understood by lnav") + .for_child(&external_log_format::elf_converter) + .with_children(converter_handlers), json_path_handler("level-field") .with_description( "The name of the level field in the log message pattern") diff --git a/src/logfile.cc b/src/logfile.cc index e327bea1..c3c9b1ab 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -259,9 +259,9 @@ logfile::process_prefix(shared_buffer_ref& sbr, this->lf_mismatched_formats.insert(curr->get_name()); continue; } - if (this->lf_options.loo_mime_type - && !curr->match_mime_type( - this->lf_options.loo_mime_type.value())) + if (this->lf_options.loo_format_name + && !(curr->get_name() + == this->lf_options.loo_format_name.value())) { if (li.li_file_range.fr_offset == 0) { log_debug("(%s) does not match file format: %s", diff --git a/src/logfile_fwd.hh b/src/logfile_fwd.hh index 5ffa0071..6c00949b 100644 --- a/src/logfile_fwd.hh +++ b/src/logfile_fwd.hh @@ -66,7 +66,7 @@ struct logfile_open_options_base { ssize_t loo_visible_size_limit{-1}; bool loo_tail{true}; file_format_t loo_file_format{file_format_t::UNKNOWN}; - nonstd::optional loo_mime_type; + nonstd::optional loo_format_name; nonstd::optional loo_piper; }; diff --git a/src/root-config.json b/src/root-config.json index 577e337d..b8a2ea6c 100644 --- a/src/root-config.json +++ b/src/root-config.json @@ -26,19 +26,6 @@ "transfer-command": "cat > {0:} && chmod ugo+rx ./{0:}" } }, - "file-format": { - "application/vnd.tcpdump.pcap": { - "title": "\ud83d\udda5 Pcap", - "header": { - "expr": { - "pcapng": ":header REGEXP '^0a0d0d0a.{8}(?:1a2b3c4d|4d3c2b1a).*'", - "pcap": ":header REGEXP '^(?:a1b2c3d4|d4c3b2a1|a1b23c4d|4d3cb2a1).*'" - }, - "size": 24 - }, - "converter": "pcap-converter.sh" - } - }, "piper": { "max-size": 10485760, "rotations": 4 diff --git a/src/scripts/pcap-converter.sh b/src/scripts/pcap_log-converter.sh similarity index 100% rename from src/scripts/pcap-converter.sh rename to src/scripts/pcap_log-converter.sh diff --git a/src/scripts/scripts.am b/src/scripts/scripts.am index 23908402..b180f967 100644 --- a/src/scripts/scripts.am +++ b/src/scripts/scripts.am @@ -9,5 +9,5 @@ BUILTIN_LNAVSCRIPTS = \ BUILTIN_SHSCRIPTS = \ $(srcdir)/scripts/dump-pid.sh \ - $(srcdir)/scripts/pcap-converter.sh \ + $(srcdir)/scripts/pcap_log-converter.sh \ $() diff --git a/src/view_curses.cc b/src/view_curses.cc index 7f8f0791..3e0bcd32 100644 --- a/src/view_curses.cc +++ b/src/view_curses.cc @@ -501,7 +501,7 @@ public: void reload_config(error_reporter& reporter) override { if (!view_colors::initialized) { - return; + view_colors::vc_active_palette = ansi_colors(); } auto& vc = view_colors::singleton(); @@ -556,9 +556,8 @@ view_colors::init(bool headless) initialized = true; { - auto reporter = [](const void*, const lnav::console::user_message&) { - - }; + auto reporter + = [](const void*, const lnav::console::user_message& um) {}; _COLOR_LISTENER.reload_config(reporter); } diff --git a/src/view_curses.hh b/src/view_curses.hh index afa0be39..d74a198d 100644 --- a/src/view_curses.hh +++ b/src/view_curses.hh @@ -278,10 +278,9 @@ public: std::unordered_map vc_class_to_role; static bool initialized; - -private: static term_color_palette* vc_active_palette; +private: /** Private constructor that initializes the member fields. */ view_colors(); diff --git a/src/view_helpers.cc b/src/view_helpers.cc index 465f9e37..c11f535f 100644 --- a/src/view_helpers.cc +++ b/src/view_helpers.cc @@ -866,8 +866,8 @@ execute_examples() EXAMPLE_RESULTS[ex.he_cmd] = result; - log_debug("example: %s", ex.he_cmd); - log_debug("example result: %s", + log_trace("example: %s", ex.he_cmd); + log_trace("example result: %s", result.get_string().c_str()); break; } diff --git a/test/Makefile.am b/test/Makefile.am index f1383b4c..06ec3085 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -226,6 +226,7 @@ dist_noinst_DATA = \ expected/test_tailer.sh_12f539e535df04364316699f9edeac461aa9f9de.err \ expected/test_tailer.sh_12f539e535df04364316699f9edeac461aa9f9de.out \ ansi-colors.0.in \ + bad-config/formats/invalid-file-format/format.json \ bad-config/formats/invalid-json-format/format.json \ bad-config/formats/invalid-properties/format.json \ bad-config/formats/invalid-regex/format.json \ @@ -235,7 +236,7 @@ dist_noinst_DATA = \ bad-config/formats/invalid-sql/init2.sql \ bad-config/formats/no-regexes/format.json \ bad-config/formats/no-samples/format.json \ - bad-config2/configs/invalid-file-format/config.json \ + bad-config2/configs/invalid-theme/config.json \ bad-config2/formats/invalid-config/config.json \ bad-config2/formats/invalid-config/config.bad-schema.json \ bad-config2/formats/invalid-config/config.malformed.json \ diff --git a/test/bad-config/formats/invalid-file-format/format.json b/test/bad-config/formats/invalid-file-format/format.json new file mode 100644 index 00000000..cca2b477 --- /dev/null +++ b/test/bad-config/formats/invalid-file-format/format.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "bad_file_format1": { + "title": "bad file format", + "regex": { + "std": { + "pattern": "(?\\d+): (?.*)$" + } + }, + "converter": { + "header": { + "expr": { + "default": ":header REGEXP 'foobar" + }, + "size": 8 + } + }, + "timestamp-format": [ + "%i" + ], + "sample": [ + { + "line": "1234: abcd" + } + ] + } +} \ No newline at end of file diff --git a/test/bad-config2/configs/invalid-file-format/config.json b/test/bad-config2/configs/invalid-file-format/config.json deleted file mode 100644 index dd003ab7..00000000 --- a/test/bad-config2/configs/invalid-file-format/config.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "https://lnav.org/schemas/config-v1.schema.json", - "tuning": { - "file-format": { - "application/vnd.example.com": { - "title": "example", - "header": { - "expr": { - "default": ":header REGEXP 'foobar" - }, - "size": 8 - } - }, - "application/vnd.example2.com": { - "title": "example" - } - } - } -} \ No newline at end of file diff --git a/test/bad-config2/configs/invalid-theme/config.json b/test/bad-config2/configs/invalid-theme/config.json new file mode 100644 index 00000000..6db0b4a3 --- /dev/null +++ b/test/bad-config2/configs/invalid-theme/config.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://lnav.org/schemas/config-v1.schema.json", + "ui": { + "theme-defs": { + "invalid-theme": { + "styles": { + "text": { + "color": "InvalidColor", + "bad-property": "abc" + } + } + } + } + } +} \ No newline at end of file diff --git a/test/expected/test_config.sh_a0907769aba112d628e7ebe39c4ec252e5e0bc69.err b/test/expected/test_config.sh_a0907769aba112d628e7ebe39c4ec252e5e0bc69.err index 6b1b18f7..1197777d 100644 --- a/test/expected/test_config.sh_a0907769aba112d628e7ebe39c4ec252e5e0bc69.err +++ b/test/expected/test_config.sh_a0907769aba112d628e7ebe39c4ec252e5e0bc69.err @@ -1,3 +1,11 @@ +⚠ warning: unexpected value for property “/ui/theme-defs/invalid-theme/styles/text/bad-property” + --> {test_dir}/bad-config2/configs/invalid-theme/config.json:9 + |  "bad-property": "abc" + = help: Available Properties + color #hex|color_name + background-color #hex|color_name + underline + bold ✘ error: 'bad' is not a supported configuration $schema version  --> {test_dir}/bad-config2/formats/invalid-config/config.bad-schema.json:2  |  "$schema": "bad"  @@ -36,34 +44,15 @@ ✘ error: invalid JSON reason: parse error: premature EOF  --> {test_dir}/bad-config2/formats/invalid-config/config.truncated.json:3 -✘ error: invalid value for property “/tuning/file-format/application~1vnd.example.com/header/expr/default” - reason: SQL expression is invalid - |  reason: unrecognized token: "'foobar" - |   --> /tuning/file-formats/application~1vnd.example.com/header/expr/default - |   | :header REGEXP 'foobar  - --> {test_dir}/bad-config2/configs/invalid-file-format/config.json:9 - = help: Property Synopsis - /tuning/file-format/application~1vnd.example.com/header/expr/default - Description - SQLite expression -✘ error: missing value for property “/tuning/file-format/application~1vnd.example.com/converter” - reason: A converter is required for a file format - |  reason: The converter script transforms the file into a format that can be consumed by lnav - = help: Property Synopsis - /tuning/file-format/application~1vnd.example.com/converter - Description - The script used to convert the file -✘ error: missing value for property “/tuning/file-format/application~1vnd.example2.com/header/expr/” - reason: At least one header expression is required for a file format - |  reason: Header expressions are used to detect a format - = help: Property Synopsis - /tuning/file-format/application~1vnd.example2.com/header/expr/ - Description - SQLite expression -✘ error: missing value for property “/tuning/file-format/application~1vnd.example2.com/converter” - reason: A converter is required for a file format - |  reason: The converter script transforms the file into a format that can be consumed by lnav +✘ error: invalid value for property “/ui/theme-defs/invalid-theme/styles/text/color” + reason: invalid color -- “InvalidColor” + |  reason: Unknown color: 'InvalidColor'. See https://jonasjacek.github.io/colors/ for a list of supported color names + --> {test_dir}/bad-config2/configs/invalid-theme/config.json:8  = help: Property Synopsis - /tuning/file-format/application~1vnd.example2.com/converter + /ui/theme-defs/invalid-theme/styles/text/color #hex|color_name Description - The script used to convert the file + The foreground color value for this style. The value can be the name of an xterm color, the hexadecimal value, or a theme variable reference. + Examples + #fff + Green + $black diff --git a/test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.err b/test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.err index c3d7699f..c20de962 100644 --- a/test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.err +++ b/test/expected/test_format_loader.sh_5992e2695b7e6cf1f3520dbb87af8fc2b8f27088.err @@ -104,6 +104,13 @@ /$schema The URI of the schema for this file Description Specifies the type of this file +✘ error: SQL expression is invalid + reason: unrecognized token: "'foobar" + --> /bad_file_format1/converter/header/expr/default + | :header REGEXP 'foobar  +✘ error: A command is required when a converter is defined + --> {test_dir}/bad-config/formats/invalid-file-format/format.json:4 + = help: The converter command transforms the file into a format that can be consumed by lnav ✘ error: invalid line format element “/bad_json_log/line-format/0/field” reason: “” is not a defined value  --> {test_dir}/bad-config/formats/invalid-json-format/format.json:7