diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml index 74046909..e4a0d89d 100644 --- a/.github/workflows/c-cpp.yml +++ b/.github/workflows/c-cpp.yml @@ -65,6 +65,7 @@ jobs: libbz2-dev libcurl4-openssl-dev libreadline-dev + tshark zlib1g-dev - name: autogen run: ./autogen.sh diff --git a/NEWS b/NEWS index 5200b98f..f9d740a1 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +lnav v0.11.0: + Features: + * Add support for pcap(3) files using tshark(1). + lnav v0.10.1: Features: * Added ":show-only-this-file" command that hides all files except the diff --git a/README b/README index ebc380c9..498c09c4 100644 --- a/README +++ b/README @@ -15,7 +15,7 @@ efficiently zero in on problems. PREREQUISITES ------------- -The following software packages are required to build lnav: +The following software packages are required to build/run lnav: gcc/clang - A C++14-compatible compiler. libpcre - The Perl Compatible Regular Expression (PCRE) library. @@ -28,6 +28,7 @@ The following software packages are required to build lnav: libcurl - The cURL library for downloading files from URLs. Version 7.23.0 or higher is required. libarchive - The libarchive library for opening archive files, like zip/tgz. + wireshark - The 'tshark' program is used to interpret pcap files. INSTALLATION diff --git a/README.md b/README.md index 6aee76ea..38ee31a4 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ The following software packages are required to build lnav: - bz2 - The bzip2 compression library. - libcurl - The cURL library for downloading files from URLs. Version 7.23.0 or higher is required. - libarchive - The libarchive library for opening archive files, like zip/tgz. +- wireshark - The 'tshark' program is used to interpret pcap files. #### Build diff --git a/TESTS_ENVIRONMENT.in b/TESTS_ENVIRONMENT.in index 8f9a7e71..4eacf186 100644 --- a/TESTS_ENVIRONMENT.in +++ b/TESTS_ENVIRONMENT.in @@ -26,6 +26,9 @@ export BZIP2_CMD XZ_CMD="@XZ_CMD@" export XZ_CMD +TSHARK_CMD="@TSHARK_CMD@" +export TSHARK_CMD + LIBARCHIVE_LIBS="@LIBARCHIVE_LIBS@" export LIBARCHIVE_LIBS diff --git a/aminclude_static.am b/aminclude_static.am index 20c2e6ec..10601ce2 100644 --- a/aminclude_static.am +++ b/aminclude_static.am @@ -1,6 +1,6 @@ # aminclude_static.am generated automatically by Autoconf -# from AX_AM_MACROS_STATIC on Mon Oct 18 09:00:48 PDT 2021 +# from AX_AM_MACROS_STATIC on Wed Nov 3 22:10:14 PDT 2021 # Code coverage diff --git a/configure.ac b/configure.ac index fcfb8261..0d4cd86a 100644 --- a/configure.ac +++ b/configure.ac @@ -52,7 +52,7 @@ dnl you are building it) AS_CASE([x$srcdir], [x/*], AS_VAR_SET(abssrcdir, $srcdir), - AS_VAR_SET(abssrcdir, `pwd`/$srcdir) + AS_VAR_SET(abssrcdir, `cd $srcdir; pwd`) ) AC_SUBST(abssrcdir) @@ -83,6 +83,7 @@ AC_PATH_PROG(BZIP2_CMD, [bzip2]) AC_PATH_PROG(RE2C_CMD, [re2c]) AM_CONDITIONAL(HAVE_RE2C, test x"$RE2C_CMD" != x"") AC_PATH_PROG(XZ_CMD, [xz]) +AC_PATH_PROG(TSHARK_CMD, [tshark]) AC_CHECK_SIZEOF(off_t) AC_CHECK_SIZEOF(size_t) diff --git a/docs/schemas/format-v1.schema.json b/docs/schemas/format-v1.schema.json index 881a4f84..7c38e501 100644 --- a/docs/schemas/format-v1.schema.json +++ b/docs/schemas/format-v1.schema.json @@ -74,11 +74,27 @@ "description": "A regular expression that restricts this format to log files with a matching name", "type": "string" }, + "mime-types": { + "title": "//mime-types", + "description": "A list of mime-types this format should be used for", + "type": "array", + "items": { + "type": "string", + "enum": [ + "application/vnd.tcpdump.pcap" + ] + } + }, "level-field": { "title": "//level-field", "description": "The name of the level field in the log message pattern", "type": "string" }, + "level-pointer": { + "title": "//level-pointer", + "description": "A regular-expression that matches the JSON-pointer of the level property", + "type": "string" + }, "timestamp-field": { "title": "//timestamp-field", "description": "The name of the timestamp field in the log message pattern", diff --git a/snapcraft.yaml b/snapcraft.yaml index 8dc3d348..2405c912 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -74,4 +74,5 @@ parts: - libxml2 - locales-all - ssh + - tshark - xclip diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d8154bb7..e81b6bd5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -280,6 +280,7 @@ add_library( data_scanner_re.cc data_parser.cc papertrail_proc.cc + pcap_manager.cc ptimec_rt.cc pretty_printer.cc pugixml/pugixml.cpp @@ -387,6 +388,7 @@ add_library( logfile_stats.hh optional.hpp papertrail_proc.hh + pcap_manager.hh plain_text_source.hh pretty_printer.hh preview_status_source.hh diff --git a/src/Makefile.am b/src/Makefile.am index 532f9d68..df927f53 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -226,6 +226,7 @@ noinst_HEADERS = \ mapbox/variant_visitor.hpp \ optional.hpp \ papertrail_proc.hh \ + pcap_manager.hh \ piper_proc.hh \ plain_text_source.hh \ pretty_printer.hh \ @@ -355,6 +356,7 @@ libdiag_a_SOURCES = \ network-extension-functions.cc \ data_parser.cc \ papertrail_proc.cc \ + pcap_manager.cc \ pretty_printer.cc \ ptimec_rt.cc \ readline_callbacks.cc \ diff --git a/src/base/auto_pid.hh b/src/base/auto_pid.hh index 475825d6..7a9077c4 100644 --- a/src/base/auto_pid.hh +++ b/src/base/auto_pid.hh @@ -39,6 +39,7 @@ #include "base/result.h" #include "base/lnav_log.hh" +#include "mapbox/variant.hpp" enum class process_state { RUNNING, @@ -54,10 +55,12 @@ public: auto_pid(const auto_pid &other) = delete; - auto_pid(auto_pid &&other) noexcept: ap_child(std::move(other).release()) + auto_pid(auto_pid &&other) noexcept + : ap_child(std::move(other).release()), + ap_status(other.ap_status) {}; - ~auto_pid() + ~auto_pid() noexcept { this->reset(); }; auto_pid &operator=(auto_pid &&other) noexcept @@ -103,6 +106,24 @@ public: return WEXITSTATUS(this->ap_status); } + using poll_result = mapbox::util::variant< + auto_pid, + auto_pid + >; + + poll_result poll() && { + if (this->ap_child != -1) { + auto rc = waitpid(this->ap_child, &this->ap_status, WNOHANG); + + if (rc <= 0) { + return std::move(*this); + } + } + + return auto_pid( + std::exchange(this->ap_child, -1), this->ap_status); + } + auto_pid wait_for_child(int options = 0) && { if (this->ap_child != -1) { @@ -116,11 +137,11 @@ public: std::exchange(this->ap_child, -1), this->ap_status); } - void reset(pid_t child = -1) + void reset(pid_t child = -1) noexcept { if (this->ap_child != child) { this->ap_status = 0; - if (this->ap_child != -1) { + if (ProcState == process_state::RUNNING && this->ap_child != -1) { log_debug("sending SIGTERM to child: %d", this->ap_child); kill(this->ap_child, SIGTERM); } diff --git a/src/base/future_util.hh b/src/base/future_util.hh index a8a6280c..9bfd6cab 100644 --- a/src/base/future_util.hh +++ b/src/base/future_util.hh @@ -63,7 +63,7 @@ public: /** * @param processor The function to execute with the result of a future. */ - explicit future_queue(std::function processor) + explicit future_queue(std::function processor) : fq_processor(processor) {}; ~future_queue() { @@ -90,12 +90,13 @@ public: */ void pop_to(size_t size = 0) { while (this->fq_deque.size() > size) { - this->fq_processor(this->fq_deque.front().get()); + auto v = this->fq_deque.front().get(); + this->fq_processor(v); this->fq_deque.pop_front(); } } - std::function fq_processor; + std::function fq_processor; std::deque> fq_deque; }; diff --git a/src/file_collection.cc b/src/file_collection.cc index 88b946e8..88392a14 100644 --- a/src/file_collection.cc +++ b/src/file_collection.cc @@ -44,10 +44,31 @@ #include "tailer/tailer.looper.hh" #include "service_tags.hh" #include "lnav_util.hh" +#include "pcap_manager.hh" static std::mutex REALPATH_CACHE_MUTEX; static std::unordered_map REALPATH_CACHE; +child_poll_result_t child_poller::poll(file_collection& fc) +{ + if (!this->cp_child) { + return child_poll_result_t::FINISHED; + } + + auto poll_res = std::move(this->cp_child.value()).poll(); + this->cp_child = nonstd::nullopt; + return poll_res.match( + [this](auto_pid& alive) { + this->cp_child = std::move(alive); + return child_poll_result_t::ALIVE; + }, + [this, &fc](auto_pid& finished) { + this->cp_finalizer(fc, finished); + return child_poll_result_t::FINISHED; + } + ); +} + void file_collection::close_files(const std::vector> &files) { for (const auto& lf : files) { @@ -109,6 +130,7 @@ void file_collection::regenerate_unique_file_names() switch (pair.second.ofd_format) { case file_format_t::FF_UNKNOWN: case file_format_t::FF_ARCHIVE: + case file_format_t::FF_PCAP: case file_format_t::FF_SQLITE_DB: { auto bn = ghc::filesystem::path(pair.first).filename().string(); if (bn.length() > this->fc_largest_path_length) { @@ -126,7 +148,7 @@ void file_collection::regenerate_unique_file_names() } } -void file_collection::merge(const file_collection &other) +void file_collection::merge(file_collection &other) { this->fc_recursive = this->fc_recursive || other.fc_recursive; this->fc_rotated = this->fc_rotated || other.fc_rotated; @@ -153,6 +175,13 @@ void file_collection::merge(const file_collection &other) other.fc_closed_files.end()); this->fc_other_files.insert(other.fc_other_files.begin(), other.fc_other_files.end()); + if (!other.fc_child_pollers.empty()) { + this->fc_child_pollers.insert( + this->fc_child_pollers.begin(), + std::make_move_iterator(other.fc_child_pollers.begin()), + std::make_move_iterator(other.fc_child_pollers.end())); + other.fc_child_pollers.clear(); + } } /** @@ -195,7 +224,7 @@ file_collection::watch_logfile(const std::string &filename, int rc; if (this->fc_closed_files.count(filename)) { - return lnav::futures::make_ready_future(retval); + return lnav::futures::make_ready_future(std::move(retval)); } if (loo.loo_fd != -1) { @@ -212,14 +241,14 @@ file_collection::watch_logfile(const std::string &filename, this->fc_file_names.end()) { retval.fc_file_names.emplace(wilddir, logfile_open_options()); } - return lnav::futures::make_ready_future(retval); + return lnav::futures::make_ready_future(std::move(retval)); } if (!S_ISREG(st.st_mode)) { if (required) { rc = -1; errno = EINVAL; } else { - return lnav::futures::make_ready_future(retval); + return lnav::futures::make_ready_future(std::move(retval)); } } auto err_iter = this->fc_name_to_errors.find(filename); @@ -236,7 +265,7 @@ file_collection::watch_logfile(const std::string &filename, std::string(strerror(errno)), }); } - return lnav::futures::make_ready_future(retval); + return lnav::futures::make_ready_future(std::move(retval)); } auto stat_iter = find_if(this->fc_new_stats.begin(), @@ -248,7 +277,7 @@ file_collection::watch_logfile(const std::string &filename, if (stat_iter != this->fc_new_stats.end()) { // this file is probably a link that we have already scanned in this // pass. - return lnav::futures::make_ready_future(retval); + return lnav::futures::make_ready_future(std::move(retval)); } this->fc_new_stats.emplace_back(st); @@ -258,7 +287,7 @@ file_collection::watch_logfile(const std::string &filename, if (file_iter == this->fc_files.end()) { if (this->fc_other_files.find(filename) != this->fc_other_files.end()) { - return lnav::futures::make_ready_future(retval); + return lnav::futures::make_ready_future(std::move(retval)); } auto func = [filename, st, loo, prog = this->fc_progress, errs = this->fc_name_to_errors]() mutable { @@ -271,11 +300,52 @@ file_collection::watch_logfile(const std::string &filename, auto ff = detect_file_format(filename); + loo.loo_file_format = ff; switch (ff) { case file_format_t::FF_SQLITE_DB: retval.fc_other_files[filename].ofd_format = ff; break; + case file_format_t::FF_PCAP: { + auto res = pcap_manager::convert(filename); + + if (res.isOk()) { + auto convert_res = res.unwrap(); + + loo.loo_fd = std::move(convert_res.cr_destination); + retval.fc_child_pollers.emplace_back(child_poller{ + std::move(convert_res.cr_child), + [filename, st, error_queue = convert_res.cr_error_queue](auto& fc, auto& child) { + if (child.was_normal_exit() && child.exit_status() == EXIT_SUCCESS) { + log_info("pcap[%d] exited normally", child.in()); + return; + } + log_error("pcap[%d] exited with %d", child.in(), child.status()); + fc.fc_name_to_errors.emplace(filename, file_error_info{ + st.st_mtime, + fmt::format("{}", fmt::join(*error_queue, "\n")), + }); + }, + }); + auto open_res = logfile::open(filename, loo); + if (open_res.isOk()) { + retval.fc_files.push_back(open_res.unwrap()); + } else { + retval.fc_name_to_errors.emplace( + filename, file_error_info{ + st.st_mtime, + open_res.unwrapErr(), + }); + } + } else { + retval.fc_name_to_errors.emplace(filename, file_error_info{ + st.st_mtime, + res.unwrapErr(), + }); + } + break; + } + case file_format_t::FF_ARCHIVE: { nonstd::optional::iterator> prog_iter_opt; @@ -381,7 +451,7 @@ file_collection::watch_logfile(const std::string &filename, } } - return lnav::futures::make_ready_future(retval); + return lnav::futures::make_ready_future(std::move(retval)); } /** @@ -440,7 +510,7 @@ void file_collection::expand_filename(lnav::futures::future_queue #include #include +#include #include "safe/safe.h" @@ -71,6 +72,39 @@ struct file_error_info { const std::string fei_description; }; +struct file_collection; + +enum class child_poll_result_t { + ALIVE, + FINISHED, +}; + +class child_poller { +public: + explicit child_poller(auto_pid child, + std::function&)> finalizer) + : cp_child(std::move(child)), cp_finalizer(std::move(finalizer)) { + } + + child_poller(child_poller&& other) noexcept + : cp_child(std::move(other.cp_child)), + cp_finalizer(std::move(other.cp_finalizer)) {} + + child_poller& operator=(child_poller&& other) noexcept { + this->cp_child = std::move(other.cp_child); + this->cp_finalizer = std::move(other.cp_finalizer); + + return *this; + } + + ~child_poller() noexcept = default; + + child_poll_result_t poll(file_collection& fc); +private: + nonstd::optional> cp_child; + std::function&)> cp_finalizer; +}; + struct file_collection { bool fc_invalidate_merge{false}; @@ -88,6 +122,7 @@ struct file_collection { std::set fc_synced_files; std::shared_ptr fc_progress; std::vector fc_new_stats; + std::list fc_child_pollers; size_t fc_largest_path_length{0}; file_collection() @@ -116,7 +151,7 @@ struct file_collection { watch_logfile(const std::string &filename, logfile_open_options &loo, bool required); - void merge(const file_collection &other); + void merge(file_collection &other); void close_files(const std::vector> &files); diff --git a/src/file_format.cc b/src/file_format.cc index f12ac328..cf7078a4 100644 --- a/src/file_format.cc +++ b/src/file_format.cc @@ -31,12 +31,70 @@ #include "config.h" +#include + #include "base/intern_string.hh" +#include "base/lnav_log.hh" #include "auto_fd.hh" #include "lnav_util.hh" #include "file_format.hh" #include "archive_manager.hh" +static bool is_pcap_header(uint8_t *buffer) +{ + size_t offset = 0; + if (buffer[0] == 0x0a && + buffer[1] == 0x0d && + buffer[2] == 0x0d && + buffer[3] == 0x0a) { + offset += sizeof(uint32_t) * 2; + if (buffer[offset + 0] == 0x1a && + buffer[offset + 1] == 0x2b && + buffer[offset + 2] == 0x3c && + buffer[offset + 3] == 0x4d) { + return true; + } + + if (buffer[offset + 0] == 0x4d && + buffer[offset + 1] == 0x3c && + buffer[offset + 2] == 0x2b && + buffer[offset + 3] == 0x1a) { + return true; + } + return false; + } + + if (buffer[0] == 0xa1 && + buffer[1] == 0xb2 && + buffer[2] == 0xc3 && + buffer[3] == 0xd4) { + return true; + } + + if (buffer[0] == 0xd4 && + buffer[1] == 0xc3 && + buffer[2] == 0xb2 && + buffer[3] == 0xa1) { + return true; + } + + if (buffer[0] == 0xa1 && + buffer[1] == 0xb2 && + buffer[2] == 0x3c && + buffer[3] == 0x4d) { + return true; + } + + if (buffer[0] == 0x4d && + buffer[1] == 0x3c && + buffer[2] == 0xb2 && + buffer[3] == 0xa1) { + return true; + } + + return false; +} + file_format_t detect_file_format(const ghc::filesystem::path &filename) { if (archive_manager::is_archive(filename)) { @@ -47,7 +105,7 @@ file_format_t detect_file_format(const ghc::filesystem::path &filename) auto_fd fd; if ((fd = openp(filename, O_RDONLY)) != -1) { - char buffer[32]; + uint8_t buffer[32]; ssize_t rc; if ((rc = read(fd, buffer, sizeof(buffer))) > 0) { @@ -56,6 +114,8 @@ file_format_t detect_file_format(const ghc::filesystem::path &filename) if (header_frag.startswith(SQLITE3_HEADER)) { retval = file_format_t::FF_SQLITE_DB; + } else if (rc > 24 && is_pcap_header(buffer)) { + retval = file_format_t::FF_PCAP; } } } diff --git a/src/file_format.hh b/src/file_format.hh index 2641cc31..5360640f 100644 --- a/src/file_format.hh +++ b/src/file_format.hh @@ -35,10 +35,11 @@ #include "fmt/format.h" #include "ghc/filesystem.hpp" -enum class file_format_t { +enum class file_format_t : int { FF_UNKNOWN, FF_SQLITE_DB, FF_ARCHIVE, + FF_PCAP, FF_REMOTE, }; @@ -58,6 +59,9 @@ struct formatter : formatter { case file_format_t::FF_ARCHIVE: name = "\U0001F5C4 Archive"; break; + case file_format_t::FF_PCAP: + name = "\U0001F5A5 Pcap"; + break; case file_format_t::FF_REMOTE: name = "\U0001F5A5 Remote"; break; @@ -69,5 +73,4 @@ struct formatter : formatter { }; } - #endif diff --git a/src/formats/formats.am b/src/formats/formats.am index 74be15cb..b322e845 100644 --- a/src/formats/formats.am +++ b/src/formats/formats.am @@ -22,6 +22,7 @@ FORMAT_FILES = \ $(srcdir)/%reldir%/openstack_log.json \ $(srcdir)/%reldir%/page_log.json \ $(srcdir)/%reldir%/papertrail_log.json \ + $(srcdir)/%reldir%/pcap_log.json \ $(srcdir)/%reldir%/snaplogic_log.json \ $(srcdir)/%reldir%/sssd_log.json \ $(srcdir)/%reldir%/strace_log.json \ diff --git a/src/formats/pcap_log.json b/src/formats/pcap_log.json new file mode 100644 index 00000000..5f997b42 --- /dev/null +++ b/src/formats/pcap_log.json @@ -0,0 +1,76 @@ +{ + "$schema": "https://lnav.org/schemas/format-v1.schema.json", + "pcap_log": { + "json": true, + "description": "pcap log format", + "mime-types": [ + "application/vnd.tcpdump.pcap" + ], + "multiline": false, + "line-format": [ + { "field": "time" }, + " ", + { + "field": "source", + "min-width": 15, + "align": "right" + }, + " → ", + { + "field": "destination", + "min-width": 15, + "align": "left" + }, + " ", + { + "field": "protocol", + "min-width": 7, + "align": "left" + }, + " ", + { + "field": "length", + "min-width": 4, + "align": "right" + }, + " ", + { "field": "info" } + ], + "level": { + "warning": "^6291456$", + "error": "^8388608$" + }, + "timestamp-field": "time", + "level-pointer": "/_ws_expert__ws_expert_severity$", + "body-field": "info", + "hide-extra": true, + "value": { + "source": { + "kind": "string", + "foreign-key": true, + "collate": "ipaddress", + "identifier": true + }, + "destination": { + "kind": "string", + "foreign-key": true, + "collate": "ipaddress", + "identifier": true + }, + "protocol": { + "kind": "string", + "identifier": true + }, + "length": { + "kind": "integer" + }, + "info": { + "kind": "string" + }, + "layers": { + "kind": "json", + "hidden": true + } + } + } +} \ No newline at end of file diff --git a/src/lnav.cc b/src/lnav.cc index b5050ab9..82d39768 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -907,7 +907,7 @@ static void clear_last_user_mark(listview_curses *lv) } } -bool update_active_files(const file_collection& new_files) +bool update_active_files(file_collection& new_files) { static loading_observer obs; @@ -936,6 +936,10 @@ bool update_active_files(const file_collection& new_files) !new_files.fc_name_to_errors.empty()) { lnav_data.ld_active_files.regenerate_unique_file_names(); } + lnav_data.ld_child_pollers.insert( + lnav_data.ld_child_pollers.begin(), + std::make_move_iterator(lnav_data.ld_active_files.fc_child_pollers.begin()), + std::make_move_iterator(lnav_data.ld_active_files.fc_child_pollers.end())); return true; } @@ -1230,13 +1234,22 @@ static void gather_pipers() ++iter; } } + + for (auto iter = lnav_data.ld_child_pollers.begin(); + iter != lnav_data.ld_child_pollers.end();) { + if (iter->poll(lnav_data.ld_active_files) == child_poll_result_t::FINISHED) { + iter = lnav_data.ld_child_pollers.erase(iter); + } else { + ++iter; + } + } } static void wait_for_pipers() { for (;;) { gather_pipers(); - if (lnav_data.ld_pipers.empty()) { + if (lnav_data.ld_pipers.empty() && lnav_data.ld_child_pollers.empty()) { log_debug("all pipers finished"); break; } @@ -1244,8 +1257,9 @@ static void wait_for_pipers() usleep(10000); rebuild_indexes(); } - log_debug("%d pipers still active", - lnav_data.ld_pipers.size()); + log_debug("%d pipers and %d children still active", + lnav_data.ld_pipers.size(), + lnav_data.ld_child_pollers.size()); } } @@ -1537,7 +1551,7 @@ static void looper() future rescan_future = std::async(std::launch::async, &file_collection::rescan_files, - active_copy, + std::move(active_copy), false); bool initial_rescan_completed = false; int session_stage = 0; @@ -1617,7 +1631,7 @@ static void looper() (session_stage < 2 || ui_clock::now() >= next_rescan_time)) { rescan_future = std::async(std::launch::async, &file_collection::rescan_files, - active_copy, + std::move(active_copy), false); } @@ -1828,7 +1842,7 @@ static void looper() else { timer.start_fade(index_counter, 3); } - log_debug("initial build rebuild"); + // log_debug("initial build rebuild"); changes += rebuild_indexes(loop_deadline); if (!initial_build && lnav_data.ld_log_source.text_line_count() == 0 && @@ -1947,6 +1961,7 @@ static void looper() if (lnav_data.ld_child_terminated) { lnav_data.ld_child_terminated = false; + log_info("checking for terminated child processes"); for (auto iter = lnav_data.ld_children.begin(); iter != lnav_data.ld_children.end(); ++iter) { @@ -2352,7 +2367,7 @@ int main(int argc, char *argv[]) if (lnav_data.ld_flags & LNF_SECURE_MODE) { if ((sqlite3_set_authorizer(lnav_data.ld_db.in(), - sqlite_authorizer, NULL)) != SQLITE_OK) { + sqlite_authorizer, nullptr)) != SQLITE_OK) { fprintf(stderr, "error: unable to attach sqlite authorizer\n"); exit(EXIT_FAILURE); } @@ -2483,11 +2498,6 @@ int main(int argc, char *argv[]) load_format_extra(lnav_data.ld_db.in(), lnav_data.ld_config_paths, loader_errors); load_format_vtabs(lnav_data.ld_vtab_manager.get(), loader_errors); - if (!loader_errors.empty()) { - print_errors(loader_errors); - return EXIT_FAILURE; - } - auto _vtab_cleanup = finally([] { static const char *VIRT_TABLES = R"( SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' @@ -2533,6 +2543,11 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' } }); + if (!loader_errors.empty()) { + print_errors(loader_errors); + return EXIT_FAILURE; + } + if (!(lnav_data.ld_flags & LNF_CHECK_CONFIG)) { DEFAULT_FILES.insert(make_pair(LNF_SYSLOG, string("var/log/messages"))); DEFAULT_FILES.insert( @@ -2919,6 +2934,16 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' clooper.process_all(); }); rebuild_indexes_repeatedly(); + if (!lnav_data.ld_active_files.fc_name_to_errors.empty()) { + for (const auto& pair : lnav_data.ld_active_files.fc_name_to_errors) { + fprintf(stderr, + "error: unable to open file: %s -- %s\n", + pair.first.c_str(), + pair.second.fei_description.c_str()); + } + + return EXIT_FAILURE; + } for (auto &pair : cmd_results) { if (pair.first.isErr()) { diff --git a/src/lnav.hh b/src/lnav.hh index efe3e9aa..0f8b2841 100644 --- a/src/lnav.hh +++ b/src/lnav.hh @@ -207,6 +207,7 @@ struct lnav_data_t { bool ld_session_loaded; std::vector ld_config_paths; file_collection ld_active_files; + std::list ld_child_pollers; std::list > ld_files_to_front; std::string ld_pt_search; time_t ld_pt_min_time; @@ -314,7 +315,7 @@ void rebuild_indexes_repeatedly(); bool setup_logline_table(exec_context &ec); bool rescan_files(bool required = false); -bool update_active_files(const file_collection& new_files); +bool update_active_files(file_collection& new_files); void wait_for_children(); diff --git a/src/log_format.cc b/src/log_format.cc index 1582c41f..6ef5a35c 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -76,7 +76,7 @@ struct line_range logline_value::origin_in_full_msg(const char *msg, ssize_t len for (int lpc = 0; lpc < this->lv_sub_offset; lpc++) { const auto *next = (const char *) memchr(last, '\n', msg_end - last); - require(next != NULL); + require(next != nullptr); next += 1; int amount = (next - last); @@ -172,7 +172,7 @@ std::string logline_value::to_string() const if (this->lv_sbr.empty()) { return this->lv_intern_string.to_string(); } - return std::string(this->lv_sbr.get_data(), this->lv_sbr.length()); + return {this->lv_sbr.get_data(), this->lv_sbr.length()}; case value_kind_t::VALUE_QUOTED: case value_kind_t::VALUE_W3C_QUOTED: @@ -190,10 +190,10 @@ std::string logline_value::to_string() const unquoted_len = unquote_func(unquoted_str, this->lv_sbr.get_data(), this->lv_sbr.length()); - return std::string(unquoted_str, unquoted_len); + return {unquoted_str, unquoted_len}; } default: - return std::string(this->lv_sbr.get_data(), this->lv_sbr.length()); + return {this->lv_sbr.get_data(), this->lv_sbr.length()}; } } break; @@ -220,7 +220,7 @@ std::string logline_value::to_string() const break; } - return std::string(buffer); + return {buffer}; } vector> log_format::lf_root_formats; @@ -258,7 +258,7 @@ bool log_format::next_format(pcre_format *fmt, int &index, int &locked_index) if (locked_index == -1) { index += 1; - if (fmt[index].name == NULL) { + if (fmt[index].name == nullptr) { retval = false; } } @@ -282,7 +282,7 @@ const char *log_format::log_scanf(uint32_t line_number, ...) { int curr_fmt = -1; - const char *retval = NULL; + const char *retval = nullptr; bool done = false; pcre_input pi(line, 0, len); pcre_context_static<128> pc; @@ -294,7 +294,7 @@ const char *log_format::log_scanf(uint32_t line_number, pi.reset(line, 0, len); if (!fmt[curr_fmt].pcre.match(pc, pi, PCRE_NO_UTF8_CHECK)) { - retval = NULL; + retval = nullptr; } else { pcre_context::capture_t *ts = pc[fmt[curr_fmt].pf_timestamp_index]; @@ -307,7 +307,7 @@ const char *log_format::log_scanf(uint32_t line_number, } retval = this->lf_date_time.scan( - pi.get_substr_start(ts), ts->length(), NULL, tm_out, *tv_out); + pi.get_substr_start(ts), ts->length(), nullptr, tm_out, *tv_out); if (retval) { if (curr_fmt != pat_index) { @@ -385,20 +385,18 @@ void log_format::check_for_new_year(std::vector &dst, exttm etm, */ struct json_log_userdata { json_log_userdata(shared_buffer_ref &sbr) - : jlu_format(NULL), jlu_line(NULL), jlu_base_line(NULL), - jlu_sub_line_count(1), jlu_handle(NULL), jlu_line_value(NULL), - jlu_line_size(0), jlu_sub_start(0), jlu_shared_buffer(sbr) { + : jlu_shared_buffer(sbr) { }; - external_log_format *jlu_format; - const logline *jlu_line; - logline *jlu_base_line; - int jlu_sub_line_count; - yajl_handle jlu_handle; - const char *jlu_line_value; - size_t jlu_line_size; - size_t jlu_sub_start; + external_log_format *jlu_format{nullptr}; + const logline *jlu_line{nullptr}; + logline *jlu_base_line{nullptr}; + int jlu_sub_line_count{1}; + yajl_handle jlu_handle{nullptr}; + const char *jlu_line_value{nullptr}; + size_t jlu_line_size{0}; + size_t jlu_sub_start{0}; shared_buffer_ref &jlu_shared_buffer; }; @@ -609,7 +607,7 @@ bool external_log_format::scan_for_partial(shared_buffer_ref &sbr, size_t &len_o auto pat = this->elf_pattern_order[this->last_pattern_index()]; pcre_input pi(sbr.get_data(), 0, sbr.length()); - if (!this->elf_multiline) { + if (!this->lf_multiline) { len_out = pat->p_pcre->match_partial(pi); return true; } @@ -640,6 +638,10 @@ log_format::scan_result_t external_log_format::scan(logfile &lf, if (li.li_partial) { log_debug("skipping partial line at offset %d", li.li_file_range.fr_offset); + if (this->lf_specialized) { + ll.set_level(LEVEL_INVALID); + dst.emplace_back(ll); + } return log_format::SCAN_INCOMPLETE; } @@ -661,7 +663,14 @@ log_format::scan_result_t external_log_format::scan(logfile &lf, if (yajl_parse(handle, line_data, sbr.length()) == yajl_status_ok && yajl_complete_parse(handle) == yajl_status_ok) { if (ll.get_time() == 0) { - return log_format::SCAN_NO_MATCH; + if (this->lf_specialized) { + ll.set_ignore(true); + dst.emplace_back(ll); + return log_format::SCAN_MATCH; + } else { + log_debug("no match! %.*s", sbr.length(), line_data); + return log_format::SCAN_NO_MATCH; + } } jlu.jlu_sub_line_count += this->jlf_line_format_init_count; @@ -857,7 +866,7 @@ log_format::scan_result_t external_log_format::scan(logfile &lf, return log_format::SCAN_MATCH; } - if (this->lf_specialized && !this->elf_multiline) { + if (this->lf_specialized && !this->lf_multiline) { auto& last_line = dst.back(); dst.emplace_back(li.li_file_range.fr_offset, @@ -940,7 +949,7 @@ void external_log_format::annotate(uint64_t line_number, shared_buffer_ref &line lr.lr_start = 0; lr.lr_end = line.length(); sa.emplace_back(lr, &SA_BODY); - if (!this->elf_multiline) { + if (!this->lf_multiline) { auto len = pat.p_pcre->match_partial(pi); sa.emplace_back(line_range{(int) len, -1}, &SA_INVALID, @@ -1112,6 +1121,17 @@ static int read_json_field(yajlpp_parse_context *ypc, const unsigned char *str, jlu->jlu_format->lf_timestamp_flags = tm_out.et_flags & ~ETF_MACHINE_ORIENTED; jlu->jlu_base_line->set_time(tv_out); } + else if (!jlu->jlu_format->elf_level_pointer.empty()) { + pcre_context_static<30> pc; + pcre_input pi(field_name); + + if (jlu->jlu_format->elf_level_pointer.match(pc, pi)) { + pcre_input pi_level((const char *) str, 0, len); + pcre_context::capture_t level_cap = {0, (int) len}; + + jlu->jlu_base_line->set_level(jlu->jlu_format->convert_level(pi_level, &level_cap)); + } + } else if (jlu->jlu_format->elf_level_field == field_name) { pcre_input pi((const char *) str, 0, len); pcre_context::capture_t level_cap = {0, (int) len}; @@ -1767,7 +1787,7 @@ void external_log_format::build(std::vector &errors) { } found = true; if (ts_len == -1 || - dts.scan(ts, ts_len, custom_formats, &tm, tv) == NULL) { + dts.scan(ts, ts_len, custom_formats, &tm, tv) == nullptr) { errors.push_back("error:" + this->elf_name.to_string() + ":invalid sample -- " + @@ -1776,9 +1796,9 @@ void external_log_format::build(std::vector &errors) { this->elf_name.to_string() + ":unrecognized timestamp format -- " + ts); - if (custom_formats == NULL) { + if (custom_formats == nullptr) { for (int lpc = 0; - PTIMEC_FORMATS[lpc].pf_fmt != NULL; lpc++) { + PTIMEC_FORMATS[lpc].pf_fmt != nullptr; lpc++) { off_t off = 0; PTIMEC_FORMATS[lpc].pf_func(&tm, ts, off, ts_len); @@ -1789,7 +1809,7 @@ void external_log_format::build(std::vector &errors) { } } else { - for (int lpc = 0; custom_formats[lpc] != NULL; lpc++) { + for (int lpc = 0; custom_formats[lpc] != nullptr; lpc++) { off_t off = 0; ptime_fmt(custom_formats[lpc], &tm, ts, off, @@ -2238,6 +2258,15 @@ bool external_log_format::match_name(const string &filename) return this->elf_filename_pcre->match(pc, pi); } +bool external_log_format::match_mime_type(const file_format_t ff) const +{ + if (ff == file_format_t::FF_UNKNOWN && this->elf_mime_types.empty()) { + return true; + } + + return this->elf_mime_types.count(ff) == 1; +} + int log_format::pattern_index_for_line(uint64_t line_number) const { auto iter = lower_bound(this->lf_pattern_locks.cbegin(), diff --git a/src/log_format.hh b/src/log_format.hh index bbb79391..a6f0612e 100644 --- a/src/log_format.hh +++ b/src/log_format.hh @@ -60,6 +60,7 @@ #include "log_level.hh" #include "line_buffer.hh" #include "log_format_fwd.hh" +#include "file_format.hh" struct sqlite3; class logfile; @@ -346,6 +347,13 @@ public: virtual bool match_name(const std::string &filename) { return true; }; + virtual bool match_mime_type(const file_format_t ff) const { + if (ff == file_format_t::FF_UNKNOWN) { + return true; + } + return false; + }; + enum scan_result_t { SCAN_MATCH, SCAN_NO_MATCH, @@ -456,6 +464,7 @@ public: int pattern_index_for_line(uint64_t line_number) const; uint8_t lf_mod_index{0}; + bool lf_multiline{true}; date_time_scanner lf_date_time; std::vector lf_pattern_locks; intern_string_t lf_timestamp_field{intern_string::lookup("timestamp", -1)}; diff --git a/src/log_format_ext.hh b/src/log_format_ext.hh index 54ef63af..ea6ee263 100644 --- a/src/log_format_ext.hh +++ b/src/log_format_ext.hh @@ -116,7 +116,6 @@ public: elf_timestamp_divisor(1.0), elf_level_field(intern_string::lookup("level", -1)), elf_body_field(intern_string::lookup("body", -1)), - elf_multiline(true), elf_container(false), elf_has_module_format(false), elf_builtin_format(false), @@ -134,6 +133,8 @@ public: bool match_name(const std::string &filename); + bool match_mime_type(const file_format_t ff) const; + scan_result_t scan(logfile &lf, std::vector &dst, const line_info &offset, @@ -349,6 +350,7 @@ public: std::set elf_source_path; std::list elf_collision; std::string elf_file_pattern; + std::set elf_mime_types; std::shared_ptr elf_filename_pcre; std::map> elf_patterns; std::vector> elf_pattern_order; @@ -360,12 +362,12 @@ public: int elf_column_count; double elf_timestamp_divisor; intern_string_t elf_level_field; + pcrepp elf_level_pointer; intern_string_t elf_body_field; intern_string_t elf_module_id_field; intern_string_t elf_opid_field; std::map elf_level_patterns; std::vector > elf_level_pairs; - bool elf_multiline; bool elf_container; bool elf_has_module_format; bool elf_builtin_format; diff --git a/src/log_format_loader.cc b/src/log_format_loader.cc index 4f3b0c38..372dd087 100644 --- a/src/log_format_loader.cc +++ b/src/log_format_loader.cc @@ -41,6 +41,7 @@ #include #include "fmt/format.h" +#include "file_format.hh" #include "base/paths.hh" #include "base/string_util.hh" @@ -174,7 +175,7 @@ static int read_format_bool(yajlpp_parse_context *ypc, int val) else if (field_name == "hide-extra") elf->jlf_hide_extra = val; else if (field_name == "multiline") - elf->elf_multiline = val; + elf->lf_multiline = val; return 1; } @@ -229,6 +230,21 @@ static int read_format_field(yajlpp_parse_context *ypc, const unsigned char *str else if (field_name == "level-field") { elf->elf_level_field = intern_string::lookup(value); } + else if (field_name == "level-pointer") { + auto pcre_res = pcrepp::from_str(value); + + if (pcre_res.isErr()) { + ypc->ypc_error_reporter( + *ypc, + lnav_log_level_t::ERROR, + fmt::format("error:{}:{}:invalid regular expression for level-pointer -- {}", + ypc->ypc_source, + ypc->get_line_number(), + pcre_res.unwrapErr().ce_msg).c_str()); + } else { + elf->elf_level_pointer = pcre_res.unwrap(); + } + } else if (field_name == "timestamp-field") { elf->lf_timestamp_field = intern_string::lookup(value); } @@ -245,6 +261,12 @@ static int read_format_field(yajlpp_parse_context *ypc, const unsigned char *str else if (field_name == "opid-field") { elf->elf_opid_field = intern_string::lookup(value); } + else if (field_name == "mime-types") { + auto value_opt = ypc->ypc_current_handler->to_enum_value(value); + if (value_opt) { + elf->elf_mime_types.insert((file_format_t) *value_opt); + } + } return 1; } @@ -655,6 +677,12 @@ static struct json_path_container search_table_handlers = { .with_children(search_table_def_handlers) }; +static const json_path_handler_base::enum_value_t MIME_TYPE_ENUM[] = { + { "application/vnd.tcpdump.pcap", file_format_t::FF_PCAP, }, + + json_path_handler_base::ENUM_TERMINATOR +}; + struct json_path_container format_handlers = { yajlpp::property_handler("regex") .with_description("The set of regular expressions used to match log messages") @@ -674,8 +702,13 @@ struct json_path_container format_handlers = { .with_description("The value to divide a numeric timestamp by in a JSON log."), json_path_handler("file-pattern", read_format_field) .with_description("A regular expression that restricts this format to log files with a matching name"), + json_path_handler("mime-types#", read_format_field) + .with_description("A list of mime-types this format should be used for") + .with_enum_values(MIME_TYPE_ENUM), json_path_handler("level-field", read_format_field) .with_description("The name of the level field in the log message pattern"), + json_path_handler("level-pointer", read_format_field) + .with_description("A regular-expression that matches the JSON-pointer of the level property"), json_path_handler("timestamp-field", read_format_field) .with_description("The name of the timestamp field in the log message pattern"), json_path_handler("body-field", read_format_field) diff --git a/src/logfile.cc b/src/logfile.cc index f5add3cb..ba3f6186 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -200,6 +200,15 @@ bool logfile::process_prefix(shared_buffer_ref &sbr, const line_info &li) iter != root_formats.end() && (found != log_format::SCAN_MATCH); ++iter) { if (!(*iter)->match_name(this->lf_filename)) { + log_debug("(%s) does not match file name: %s", + (*iter)->get_name().get(), + this->lf_filename.c_str()); + continue; + } + if (!(*iter)->match_mime_type(this->lf_options.loo_file_format)) { + log_debug("(%s) does not match file format: %d", + (*iter)->get_name().get(), + this->lf_options.loo_file_format); continue; } @@ -232,8 +241,12 @@ bool logfile::process_prefix(shared_buffer_ref &sbr, const line_info &li) logline &last_line = this->lf_index[this->lf_index.size() - 1]; for (size_t lpc = 0; lpc < this->lf_index.size() - 1; lpc++) { - this->lf_index[lpc].set_time(last_line.get_time()); - this->lf_index[lpc].set_millis(last_line.get_millis()); + if (this->lf_format->lf_multiline) { + this->lf_index[lpc].set_time(last_line.get_time()); + this->lf_index[lpc].set_millis(last_line.get_millis()); + } else { + this->lf_index[lpc].set_ignore(true); + } } break; } @@ -254,7 +267,7 @@ bool logfile::process_prefix(shared_buffer_ref &sbr, const line_info &li) logline &second_to_last = this->lf_index[prescan_size - 1]; logline &latest = this->lf_index[prescan_size]; - if (latest < second_to_last) { + if (!second_to_last.is_ignored() && latest < second_to_last) { if (this->lf_format->lf_time_ordered) { this->lf_out_of_time_order_count += 1; for (size_t lpc = prescan_size; diff --git a/src/logfile_fwd.hh b/src/logfile_fwd.hh index 2fc75880..9a42f9a1 100644 --- a/src/logfile_fwd.hh +++ b/src/logfile_fwd.hh @@ -36,6 +36,7 @@ #include #include "auto_fd.hh" +#include "file_format.hh" using ui_clock = std::chrono::steady_clock; @@ -103,6 +104,12 @@ struct logfile_open_options { return *this; } + logfile_open_options &with_file_format(file_format_t ff) { + this->loo_file_format = ff; + + return *this; + } + std::string loo_filename; auto_fd loo_fd; logfile_name_source loo_source{logfile_name_source::USER}; @@ -112,6 +119,7 @@ struct logfile_open_options { bool loo_non_utf_is_visible{true}; ssize_t loo_visible_size_limit{-1}; bool loo_tail{true}; + file_format_t loo_file_format{file_format_t::FF_UNKNOWN}; }; #endif diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc index 66aa5417..2b8b9614 100644 --- a/src/logfile_sub_source.cc +++ b/src/logfile_sub_source.cc @@ -824,6 +824,10 @@ logfile_sub_source::rebuild_result logfile_sub_source::rebuild_index(nonstd::opt } for (size_t line_index = 0; line_index < lf->size(); line_index++) { + if ((*lf)[line_index].is_ignored()) { + continue; + } + content_line_t con_line(ld->ld_file_index * MAX_LINES_PER_FILE + line_index); @@ -873,13 +877,15 @@ logfile_sub_source::rebuild_result logfile_sub_source::rebuild_index(nonstd::opt break; } - int file_index = ld->ld_file_index; - int line_index = lf_iter - ld->get_file_ptr()->begin(); + if (!lf_iter->is_ignored()) { + int file_index = ld->ld_file_index; + int line_index = lf_iter - ld->get_file_ptr()->begin(); - content_line_t con_line(file_index * MAX_LINES_PER_FILE + - line_index); + content_line_t con_line(file_index * MAX_LINES_PER_FILE + + line_index); - this->lss_index.push_back(con_line); + this->lss_index.push_back(con_line); + } merge.next(); index_off += 1; diff --git a/src/pcap_manager.cc b/src/pcap_manager.cc new file mode 100644 index 00000000..bee90446 --- /dev/null +++ b/src/pcap_manager.cc @@ -0,0 +1,129 @@ +/** + * Copyright (c) 2021, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file pcap_manager.cc + */ + +#include "config.h" + +#include +#include + +#include + +#include "pcap_manager.hh" +#include "lnav_util.hh" +#include "line_buffer.hh" + +namespace pcap_manager { + +Result +convert(const std::string &filename) +{ + log_info("attempting to convert pcap file -- %s", filename.c_str()); + + auto outfile = TRY(open_temp_file( + ghc::filesystem::temp_directory_path() / "lnav.pcap.XXXXXX")); + ghc::filesystem::remove(outfile.first); + auto err_pipe = TRY(auto_pipe::for_child_fd(STDERR_FILENO)); + auto child = TRY(lnav::pid::from_fork()); + + err_pipe.after_fork(child.in()); + if (child.in_child()) { + auto dev_null = open("/dev/null", O_RDONLY); + + dup2(dev_null, STDIN_FILENO); + dup2(outfile.second, STDOUT_FILENO); + + const char *args[] = { + "tshark", + "-T", "ek", + "-P", + "-V", + "-t", "ad", + "-r", filename.c_str(), + nullptr, + }; + + execvp("tshark", (char **) args); + if (errno == ENOENT) { + fprintf(stderr, + "pcap support requires 'tshark' v3+ to be installed\n"); + } else { + fprintf(stderr, + "failed to execute 'tshark' -- %s\n", + strerror(errno)); + } + _exit(EXIT_FAILURE); + } + + auto error_queue = std::make_shared>(); + std::thread err_reader([err = std::move(err_pipe.read_end()), error_queue, child_pid=child.in()]() mutable { + line_buffer lb; + file_range pipe_range; + bool done = false; + + lb.set_fd(err); + while (!done) { + auto load_res = lb.load_next_line(pipe_range); + + if (load_res.isErr()) { + done = true; + } else { + auto li = load_res.unwrap(); + + pipe_range = li.li_file_range; + if (li.li_file_range.empty()) { + done = true; + } else { + lb.read_range(li.li_file_range).then([error_queue, child_pid](auto sbr) { + auto line_str = string_fragment(sbr.get_data(), + 0, + sbr.length()); + line_str.trim("\n"); + if (error_queue->size() < 5) { + error_queue->emplace_back(line_str.to_string()); + } + + log_debug("pcap[%d]: %.*s", + child_pid, line_str.length(), line_str.data()); + }); + } + } + } + }); + err_reader.detach(); + + log_info("started tshark %d to process file", child.in()); + + return Ok(convert_result{ + std::move(child), auto_fd(outfile.second), error_queue, + }); +} + +} diff --git a/src/pcap_manager.hh b/src/pcap_manager.hh new file mode 100644 index 00000000..55ab02ad --- /dev/null +++ b/src/pcap_manager.hh @@ -0,0 +1,55 @@ +/** + * Copyright (c) 2021, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file pcap_manager.hh + */ + +#ifndef lnav_pcap_manager_hh +#define lnav_pcap_manager_hh + +#include +#include + +#include "base/result.h" +#include "base/auto_pid.hh" +#include "auto_fd.hh" + +namespace pcap_manager { + +struct convert_result { + auto_pid cr_child; + auto_fd cr_destination; + std::shared_ptr> cr_error_queue; +}; + +Result +convert(const std::string& filename); + +} + +#endif diff --git a/src/pcrepp/pcrepp.hh b/src/pcrepp/pcrepp.hh index bb9d84f3..36522bb7 100644 --- a/src/pcrepp/pcrepp.hh +++ b/src/pcrepp/pcrepp.hh @@ -224,6 +224,12 @@ public: pi_length(s.length()), pi_string(s.data()) {}; + pcre_input(const intern_string_t& s) + : pi_offset(0), + pi_next_offset(0), + pi_length(s.size()), + pi_string(s.get()) {}; + pcre_input(const string_fragment &&) = delete; pcre_input(const std::string &str, size_t off = 0) diff --git a/src/sql_util.cc b/src/sql_util.cc index 23bc55ed..b6eb16cf 100644 --- a/src/sql_util.cc +++ b/src/sql_util.cc @@ -583,10 +583,12 @@ static void sqlite_logger(void *dummy, int code, const char *msg) break; } - log_msg(level, __FILE__, __LINE__, "%s", msg); + log_msg(level, __FILE__, __LINE__, "(%d) %s", code, msg); + + ensure(code != 21); } -void sql_install_logger(void) +void sql_install_logger() { #ifdef SQLITE_CONFIG_LOG sqlite3_config(SQLITE_CONFIG_LOG, sqlite_logger, NULL); @@ -869,7 +871,7 @@ int sqlite_authorizer(void *pUserData, int action_code, const char *detail1, return SQLITE_OK; } -string sql_keyword_re(void) +string sql_keyword_re() { string retval = "(?:"; bool first = true; diff --git a/src/tailer/tailer.looper.cc b/src/tailer/tailer.looper.cc index fd1ab310..783c50ac 100644 --- a/src/tailer/tailer.looper.cc +++ b/src/tailer/tailer.looper.cc @@ -329,7 +329,7 @@ tailer::looper::host_tailer::for_host(const std::string& netloc) args.push_back(nullptr); execvp(cfg.c_ssh_cmd.c_str(), args.data()); - exit(EXIT_FAILURE); + _exit(EXIT_FAILURE); } std::vector error_queue; @@ -406,7 +406,7 @@ tailer::looper::host_tailer::for_host(const std::string& netloc) args.push_back(nullptr); execvp(cfg.c_ssh_cmd.c_str(), args.data()); - exit(EXIT_FAILURE); + _exit(EXIT_FAILURE); } return Ok(std::make_shared( @@ -568,7 +568,7 @@ void tailer::looper::host_tailer::loop_body() if (read_res.isErr()) { log_error("read error: %s", read_res.unwrapErr().c_str()); - exit(EXIT_FAILURE); + return; } auto packet = read_res.unwrap(); diff --git a/test/Makefile.am b/test/Makefile.am index cb50109c..027cf8e8 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -240,6 +240,8 @@ dist_noinst_DATA = \ datafile_simple.19 \ datafile_simple.20 \ datafile_xml.0 \ + dhcp.pcapng \ + dhcp-trunc.pcapng \ expected_help.txt \ listview_output.0 \ listview_output.1 \ @@ -273,6 +275,7 @@ dist_noinst_DATA = \ logfile_glog.0 \ logfile_haproxy.0 \ logfile_invalid_json.json \ + logfile_invalid_json2.json \ logfile_journald.json \ logfile_json.json \ logfile_json2.json \ diff --git a/test/dhcp-trunc.pcapng b/test/dhcp-trunc.pcapng new file mode 100644 index 00000000..00734bad Binary files /dev/null and b/test/dhcp-trunc.pcapng differ diff --git a/test/dhcp.pcapng b/test/dhcp.pcapng new file mode 100644 index 00000000..530c64ce Binary files /dev/null and b/test/dhcp.pcapng differ diff --git a/test/formats/nestedjson/format.json b/test/formats/nestedjson/format.json index 3a28b4eb..eeee5295 100644 --- a/test/formats/nestedjson/format.json +++ b/test/formats/nestedjson/format.json @@ -2,7 +2,7 @@ "ntest_log" : { "title" : "Test JSON Log", "json" : true, - "file-pattern" : "logfile_(nested|invalid)_json\\.json", + "file-pattern" : "logfile_(nested|invalid)_json\\d*\\.json", "description" : "Test config", "line-format" : [ { "field" : "ts" }, diff --git a/test/logfile_invalid_json2.json b/test/logfile_invalid_json2.json new file mode 100644 index 00000000..ed97989c --- /dev/null +++ b/test/logfile_invalid_json2.json @@ -0,0 +1,3 @@ +{"ts": "2013-09-06T20:00:48.124817Z", "@fields": { "lvl": "TRACE", "msg": "trace test"}} +{"ts": "2013-09-06T20:00:49.124817Z", "@fields": { "lvl": "INFO", "msg": "Starting up service"}} +{"ts": "2013-09-06T22:00:49.124817Z", "@fields": { "lvl": "INFO", "msg": diff --git a/test/test_json_format.sh b/test/test_json_format.sh index d63b656b..7df2a831 100644 --- a/test/test_json_format.sh +++ b/test/test_json_format.sh @@ -533,3 +533,19 @@ parse error: premature EOF 2013-09-06T22:00:59.222 DEBUG4 Details... @fields: { "lvl": "DEBUG4", "msg": "Details..."} EOF + +run_test ${lnav_test} -n \ + -d /tmp/lnav.err \ + -I ${test_dir} \ + ${test_dir}/logfile_invalid_json2.json + +check_output "json log format is not working" <