/** * Copyright (c) 2020, Timothy Stack * * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * * Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * Neither the name of Timothy Stack nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * @file log_format_fwd.hh */ #ifndef lnav_log_format_fwd_hh #define lnav_log_format_fwd_hh #include #include "ArenaAlloc/arenaalloc.h" #include "base/file_range.hh" #include "base/string_attr_type.hh" #include "byte_array.hh" #include "log_level.hh" #include "pcrepp/pcrepp.hh" #include "ptimec.hh" #include "robin_hood/robin_hood.h" class log_format; struct opid_time_range { struct timeval otr_begin; struct timeval otr_end; }; using log_opid_map = robin_hood::unordered_map>; struct scan_batch_context { ArenaAlloc::Alloc& sbc_allocator; log_opid_map sbc_opids; std::string sbc_cached_level_strings[4]; log_level_t sbc_cached_level_values[4]; size_t sbc_cached_level_count{0}; }; /** * Metadata for a single line in a log file. */ class logline { public: static string_attr_type L_PREFIX; static string_attr_type L_TIMESTAMP; static string_attr_type> L_FILE; static string_attr_type L_PARTITION; static string_attr_type L_MODULE; static string_attr_type L_OPID; static string_attr_type L_META; /** * Construct a logline object with the given values. * * @param off The offset of the line in the file. * @param t The timestamp for the line. * @param millis The millisecond timestamp for the line. * @param l The logging level. */ logline(file_off_t off, time_t t, uint16_t millis, log_level_t l, uint8_t mod = 0, uint8_t opid = 0) : ll_offset(off), ll_time(t), ll_millis(millis), ll_opid(opid), ll_sub_offset(0), ll_valid_utf(1), ll_level(l), ll_module_id(mod), ll_expr_mark(0) { memset(this->ll_schema, 0, sizeof(this->ll_schema)); } logline(file_off_t off, const struct timeval& tv, log_level_t l, uint8_t mod = 0, uint8_t opid = 0) : ll_offset(off), ll_opid(opid), ll_sub_offset(0), ll_valid_utf(1), ll_level(l), ll_module_id(mod), ll_expr_mark(0) { this->set_time(tv); memset(this->ll_schema, 0, sizeof(this->ll_schema)); } /** @return The offset of the line in the file. */ file_off_t get_offset() const { return this->ll_offset; } uint16_t get_sub_offset() const { return this->ll_sub_offset; } void set_sub_offset(uint16_t suboff) { this->ll_sub_offset = suboff; } /** @return The timestamp for the line. */ time_t get_time() const { return this->ll_time; } void to_exttm(struct exttm& tm_out) const { tm_out.et_tm = *gmtime(&this->ll_time); tm_out.et_nsec = this->ll_millis * 1000 * 1000; } void set_time(time_t t) { this->ll_time = t; } /** @return The millisecond timestamp for the line. */ uint16_t get_millis() const { return this->ll_millis; } void set_millis(uint16_t m) { this->ll_millis = m; } uint64_t get_time_in_millis() const { return (this->ll_time * 1000ULL + (uint64_t) this->ll_millis); } struct timeval get_timeval() const { struct timeval retval = {this->ll_time, this->ll_millis * 1000}; return retval; } void set_time(const struct timeval& tv) { this->ll_time = tv.tv_sec; this->ll_millis = tv.tv_usec / 1000; } void set_ignore(bool val) { if (val) { this->ll_level |= LEVEL_IGNORE; } else { this->ll_level &= ~LEVEL_IGNORE; } } bool is_ignored() const { return this->ll_level & LEVEL_IGNORE; } void set_mark(bool val) { if (val) { this->ll_level |= LEVEL_MARK; } else { this->ll_level &= ~LEVEL_MARK; } } bool is_marked() const { return this->ll_level & LEVEL_MARK; } void set_expr_mark(bool val) { this->ll_expr_mark = val; } bool is_expr_marked() const { return this->ll_expr_mark; } void set_time_skew(bool val) { if (val) { this->ll_level |= LEVEL_TIME_SKEW; } else { this->ll_level &= ~LEVEL_TIME_SKEW; } } bool is_time_skewed() const { return this->ll_level & LEVEL_TIME_SKEW; } void set_valid_utf(bool v) { this->ll_valid_utf = v; } bool is_valid_utf() const { return this->ll_valid_utf; } /** @param l The logging level. */ void set_level(log_level_t l) { this->ll_level = l; }; /** @return The logging level. */ log_level_t get_level_and_flags() const { return (log_level_t) this->ll_level; } log_level_t get_msg_level() const { return (log_level_t) (this->ll_level & ~LEVEL__FLAGS); } const char* get_level_name() const { return level_names[this->ll_level & ~LEVEL__FLAGS]; } bool is_message() const { return (this->ll_level & (LEVEL_IGNORE | LEVEL_CONTINUED)) == 0; } bool is_continued() const { return this->ll_level & LEVEL_CONTINUED; } uint8_t get_module_id() const { return this->ll_module_id; } void set_opid(uint8_t opid) { this->ll_opid = opid; } uint8_t get_opid() const { return this->ll_opid; } /** * @return True if there is a schema value set for this log line. */ bool has_schema() const { return (this->ll_schema[0] != 0 || this->ll_schema[1] != 0); } /** * Set the "schema" for this log line. The schema ID is used to match log * lines that have a similar format when generating the logline table. The * schema is set lazily so that startup is faster. * * @param ba The SHA-1 hash of the constant parts of this log line. */ void set_schema(const byte_array<2, uint64_t>& ba) { memcpy(this->ll_schema, ba.in(), sizeof(this->ll_schema)); } char get_schema() const { return this->ll_schema[0]; } /** * Perform a partial match of the given schema against this log line. * Storing the full schema is not practical, so we just keep the first four * bytes. * * @param ba The SHA-1 hash of the constant parts of a log line. * @return True if the first four bytes of the given schema match the * schema stored in this log line. */ bool match_schema(const byte_array<2, uint64_t>& ba) const { return memcmp(this->ll_schema, ba.in(), sizeof(this->ll_schema)) == 0; } /** * Compare loglines based on their timestamp. */ bool operator<(const logline& rhs) const { return (this->ll_time < rhs.ll_time) || (this->ll_time == rhs.ll_time && this->ll_millis < rhs.ll_millis) || (this->ll_time == rhs.ll_time && this->ll_millis == rhs.ll_millis && this->ll_offset < rhs.ll_offset) || (this->ll_time == rhs.ll_time && this->ll_millis == rhs.ll_millis && this->ll_offset == rhs.ll_offset && this->ll_sub_offset < rhs.ll_sub_offset); } bool operator<(const time_t& rhs) const { return this->ll_time < rhs; } bool operator<(const struct timeval& rhs) const { return ((this->ll_time < rhs.tv_sec) || ((this->ll_time == rhs.tv_sec) && (this->ll_millis < (rhs.tv_usec / 1000)))); } bool operator<=(const struct timeval& rhs) const { return ((this->ll_time < rhs.tv_sec) || ((this->ll_time == rhs.tv_sec) && (this->ll_millis <= (rhs.tv_usec / 1000)))); } private: file_off_t ll_offset; time_t ll_time; unsigned int ll_millis : 10; unsigned int ll_opid : 6; unsigned int ll_sub_offset : 15; unsigned int ll_valid_utf : 1; uint8_t ll_level; uint8_t ll_module_id : 7; uint8_t ll_expr_mark : 1; char ll_schema[2]; }; struct format_tag_def { format_tag_def(std::string name) : ftd_name(name) {} struct path_restriction { std::string p_glob; bool matches(const char* fn) const; }; std::string ftd_name; std::string ftd_description; std::vector ftd_paths; std::shared_ptr> ftd_pattern; log_level_t ftd_level{LEVEL_UNKNOWN}; }; #endif