From 16d39f3fd9da3af97e14a2fc5f927b729cc5546b Mon Sep 17 00:00:00 2001 From: Timothy Stack Date: Sun, 24 Jul 2022 23:29:46 -0700 Subject: [PATCH] [db] move the shared_buffer_ref out of logline_value --- src/all_logs_vtab.cc | 26 +-- src/all_logs_vtab.hh | 6 +- src/command_executor.cc | 4 +- src/command_executor.hh | 5 +- src/field_overlay_source.cc | 10 +- src/hotkeys.cc | 22 +- src/lnav_commands.cc | 4 +- src/log.watch.cc | 23 +- src/log_actions.cc | 16 +- src/log_data_helper.cc | 34 +-- src/log_data_helper.hh | 10 +- src/log_data_table.cc | 35 ++- src/log_data_table.hh | 4 +- src/log_format.cc | 200 ++++++++++-------- src/log_format.hh | 54 +++-- src/log_format_ext.hh | 7 +- src/log_format_impls.cc | 40 ++-- src/log_search_table.cc | 39 ++-- src/log_search_table.hh | 6 +- src/log_vtab_impl.cc | 62 +++--- src/log_vtab_impl.hh | 3 +- src/logfile_sub_source.cc | 32 +-- src/logfile_sub_source.hh | 7 +- src/readline_possibilities.cc | 9 +- src/shared_buffer.cc | 15 ++ src/shared_buffer.hh | 5 + src/spectro_impls.cc | 27 ++- test/drive_data_scanner.cc | 6 +- ...aae556ecb1661602f176215e28f661d3404032.out | 6 +- ...0fd242f57a96d40f466493938cda0789a094fa.out | 40 ++-- 30 files changed, 398 insertions(+), 359 deletions(-) diff --git a/src/all_logs_vtab.cc b/src/all_logs_vtab.cc index ada48c86..87f920ab 100644 --- a/src/all_logs_vtab.cc +++ b/src/all_logs_vtab.cc @@ -62,15 +62,16 @@ all_logs_vtab::get_columns(std::vector& cols) const void all_logs_vtab::extract(logfile* lf, uint64_t line_number, - shared_buffer_ref& line, - std::vector& values) + logline_value_vector& values) { - auto format = lf->get_format(); + auto& line = values.lvv_sbr; + auto format = lf->get_format_ptr(); - std::vector sub_values; + logline_value_vector sub_values; this->vi_attrs.clear(); - format->annotate(line_number, line, this->vi_attrs, sub_values, false); + sub_values.lvv_sbr = line; + format->annotate(line_number, this->vi_attrs, sub_values, false); auto body = find_string_attr_range(this->vi_attrs, &SA_BODY); if (body.lr_start == -1) { @@ -85,18 +86,9 @@ all_logs_vtab::extract(logfile* lf, dp.dp_msg_format = &str; dp.parse(); - tmp_shared_buffer tsb(str.c_str()); - - values.emplace_back(this->alv_msg_meta, tsb.tsb_ref); - - this->alv_schema_manager.invalidate_refs(); - this->alv_schema_buffer.clear(); - dp.dp_schema_id.to_string(std::back_inserter(this->alv_schema_buffer)); - shared_buffer_ref schema_ref; - schema_ref.share(this->alv_schema_manager, - this->alv_schema_buffer.data(), - data_parser::schema_id_t::STRING_SIZE - 1); - values.emplace_back(this->alv_schema_meta, schema_ref); + values.lvv_values.emplace_back(this->alv_msg_meta, std::move(str)); + values.lvv_values.emplace_back(this->alv_schema_meta, + dp.dp_schema_id.to_string()); } bool diff --git a/src/all_logs_vtab.hh b/src/all_logs_vtab.hh index ffa3534c..cab0a4d3 100644 --- a/src/all_logs_vtab.hh +++ b/src/all_logs_vtab.hh @@ -48,17 +48,13 @@ public: void extract(logfile* lf, uint64_t line_number, - shared_buffer_ref& line, - std::vector& values) override; + logline_value_vector& values) override; bool next(log_cursor& lc, logfile_sub_source& lss) override; private: logline_value_meta alv_msg_meta; logline_value_meta alv_schema_meta; - shared_buffer alv_schema_manager; - fmt::basic_memory_buffer - alv_schema_buffer; }; #endif // LNAV_ALL_LOGS_VTAB_HH diff --git a/src/command_executor.cc b/src/command_executor.cc index 9cc26816..c49450c0 100644 --- a/src/command_executor.cc +++ b/src/command_executor.cc @@ -251,7 +251,7 @@ bind_sql_parameters(exec_context& ec, sqlite3_stmt* stmt) retval[name] = env_value; } } else if (name[0] == ':' && ec.ec_line_values != nullptr) { - for (auto& lv : *ec.ec_line_values) { + for (auto& lv : ec.ec_line_values->lvv_values) { if (lv.lv_meta.lvm_name != &name[1]) { continue; } @@ -1026,7 +1026,7 @@ exec_context::clear_output() this->ec_output_stack.back() = std::make_pair("default", nonstd::nullopt); } -exec_context::exec_context(std::vector* line_values, +exec_context::exec_context(logline_value_vector* line_values, sql_callback_t sql_callback, pipe_callback_t pipe_callback) : ec_line_values(line_values), diff --git a/src/command_executor.hh b/src/command_executor.hh index 1bae7e8f..4461d559 100644 --- a/src/command_executor.hh +++ b/src/command_executor.hh @@ -49,6 +49,7 @@ struct exec_context; class attr_line_t; class logline_value; +struct logline_value_vector; using sql_callback_t = int (*)(exec_context&, sqlite3_stmt*); int sql_callback(exec_context& ec, sqlite3_stmt* stmt); @@ -67,7 +68,7 @@ struct exec_context { using output_t = std::pair; - exec_context(std::vector* line_values = nullptr, + exec_context(logline_value_vector* line_values = nullptr, sql_callback_t sql_callback = ::sql_callback, pipe_callback_t pipe_callback = nullptr); @@ -218,7 +219,7 @@ struct exec_context { perm_t ec_perms{perm_t::READ_WRITE}; std::map ec_override; - std::vector* ec_line_values; + logline_value_vector* ec_line_values; std::stack> ec_local_vars; std::map ec_global_vars; std::vector ec_path_stack; diff --git a/src/field_overlay_source.cc b/src/field_overlay_source.cc index f7f21f8b..0aba62f7 100644 --- a/src/field_overlay_source.cc +++ b/src/field_overlay_source.cc @@ -147,7 +147,8 @@ field_overlay_source::build_field_lines(const listview_curses& lv) curr_tv = this->fos_log_helper.ldh_line->get_timeval(); if (ll->is_time_skewed() && time_range.lr_end != -1) { const char* time_src - = this->fos_log_helper.ldh_msg.get_data() + time_range.lr_start; + = this->fos_log_helper.ldh_line_values.lvv_sbr.get_data() + + time_range.lr_start; struct timeval actual_tv; date_time_scanner dts; struct exttm tm; @@ -215,7 +216,8 @@ field_overlay_source::build_field_lines(const listview_curses& lv) this->fos_known_key_size = LOG_BODY.length(); this->fos_unknown_key_size = 0; - for (auto& ldh_line_value : this->fos_log_helper.ldh_line_values) { + for (auto& ldh_line_value : this->fos_log_helper.ldh_line_values.lvv_values) + { auto& meta = ldh_line_value.lv_meta; int this_key_size = meta.lvm_name.size(); @@ -260,13 +262,13 @@ field_overlay_source::build_field_lines(const listview_curses& lv) this->fos_lines.emplace_back(pattern_al); } - if (this->fos_log_helper.ldh_line_values.empty()) { + if (this->fos_log_helper.ldh_line_values.lvv_values.empty()) { this->fos_lines.emplace_back(" No known message fields"); } const log_format* last_format = nullptr; - for (auto& lv : this->fos_log_helper.ldh_line_values) { + for (auto& lv : this->fos_log_helper.ldh_line_values.lvv_values) { if (!lv.lv_meta.lvm_format) { continue; } diff --git a/src/hotkeys.cc b/src/hotkeys.cc index e43c7327..98776e53 100644 --- a/src/hotkeys.cc +++ b/src/hotkeys.cc @@ -87,26 +87,22 @@ public: std::shared_ptr lf = this->lh_sub_source.find(cl); auto ll = lf->begin() + cl; auto format = lf->get_format(); - lf->read_full_message(ll, this->lh_msg_buffer); - format->annotate(cl, - this->lh_msg_buffer, - this->lh_string_attrs, - this->lh_line_values, - false); + lf->read_full_message(ll, this->lh_line_values.lvv_sbr); + format->annotate( + cl, this->lh_string_attrs, this->lh_line_values, false); } std::string to_string(const struct line_range& lr) const { - const char* start = this->lh_msg_buffer.get_data(); + const char* start = this->lh_line_values.lvv_sbr.get_data(); - return std::string(&start[lr.lr_start], lr.length()); + return {&start[lr.lr_start], (size_t) lr.length()}; } logfile_sub_source& lh_sub_source; vis_line_t lh_current_line; - shared_buffer_ref lh_msg_buffer; string_attrs_t lh_string_attrs; - std::vector lh_line_values; + logline_value_vector lh_line_values; }; static int @@ -147,7 +143,7 @@ handle_keyseq(const char* keyseq) return false; } - std::vector values; + logline_value_vector values; exec_context ec(&values, key_sql_callback, pipe_callback); auto& var_stack = ec.ec_local_vars; @@ -629,10 +625,10 @@ handle_paging_key(int ch) = find_string_attr_range( next_helper.lh_string_attrs, &logline::L_OPID); const char* start_opid - = start_helper.lh_msg_buffer.get_data_at( + = start_helper.lh_line_values.lvv_sbr.get_data_at( opid_range.lr_start); const char* next_opid - = next_helper.lh_msg_buffer.get_data_at( + = next_helper.lh_line_values.lvv_sbr.get_data_at( opid_next_range.lr_start); if (opid_range.length() != opid_next_range.length() || memcmp( diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index c07fb4d6..b0d58031 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -1413,7 +1413,7 @@ com_pipe_to(exec_context& ec, tmp_str, sizeof(tmp_str), ldh.ldh_line->get_timeval()); setenv("log_time", tmp_str, 1); setenv("log_path", ldh.ldh_file->get_filename().c_str(), 1); - for (auto& ldh_line_value : ldh.ldh_line_values) { + for (auto& ldh_line_value : ldh.ldh_line_values.lvv_values) { setenv(ldh_line_value.lv_meta.lvm_name.get(), ldh_line_value.to_string().c_str(), 1); @@ -4460,7 +4460,7 @@ command_prompt(std::vector& args) ln_mode_t::COMMAND, "numeric-colname", dls_header.hm_name); } } else { - for (auto& ldh_line_value : ldh.ldh_line_values) { + for (auto& ldh_line_value : ldh.ldh_line_values.lvv_values) { auto& meta = ldh_line_value.lv_meta; if (!meta.lvm_format) { diff --git a/src/log.watch.cc b/src/log.watch.cc index eccd46ff..f7a314af 100644 --- a/src/log.watch.cc +++ b/src/log.watch.cc @@ -119,13 +119,13 @@ eval_with(logfile& lf, logfile::iterator ll) sqlite_db_tag>(); char timestamp_buffer[64] = ""; - shared_buffer_ref sbr, raw_sbr; - lf.read_full_message(ll, sbr); + shared_buffer_ref raw_sbr; + logline_value_vector values; + lf.read_full_message(ll, values.lvv_sbr); auto format = lf.get_format(); string_attrs_t sa; - std::vector values; auto line_number = std::distance(lf.begin(), ll); - format->annotate(line_number, sbr, sa, values); + format->annotate(line_number, sa, values); for (auto& watch_pair : exprs.e_watch_exprs) { if (!watch_pair.second.cwe_enabled) { @@ -204,8 +204,11 @@ eval_with(logfile& lf, logfile::iterator ll) continue; } if (strcmp(name, ":log_text") == 0) { - sqlite3_bind_text( - stmt, lpc + 1, sbr.get_data(), sbr.length(), SQLITE_STATIC); + sqlite3_bind_text(stmt, + lpc + 1, + values.lvv_sbr.get_data(), + values.lvv_sbr.length(), + SQLITE_STATIC); continue; } if (strcmp(name, ":log_body") == 0) { @@ -216,7 +219,7 @@ eval_with(logfile& lf, logfile::iterator ll) sqlite3_bind_text(stmt, lpc + 1, - sbr.get_data_at(sar.lr_start), + values.lvv_sbr.get_data_at(sar.lr_start), sar.length(), SQLITE_STATIC); } else { @@ -232,7 +235,7 @@ eval_with(logfile& lf, logfile::iterator ll) sqlite3_bind_text(stmt, lpc + 1, - sbr.get_data_at(sar.lr_start), + values.lvv_sbr.get_data_at(sar.lr_start), sar.length(), SQLITE_STATIC); } else { @@ -254,7 +257,7 @@ eval_with(logfile& lf, logfile::iterator ll) continue; } auto found = false; - for (const auto& lv : values) { + for (const auto& lv : values.lvv_values) { if (lv.lv_meta.lvm_name != &name[1]) { continue; } @@ -322,7 +325,7 @@ eval_with(logfile& lf, logfile::iterator ll) lf.get_format_name().to_string(), timestamp_buffer, }; - for (const auto& lv : values) { + for (const auto& lv : values.lvv_values) { switch (lv.lv_meta.lvm_kind) { case value_kind_t::VALUE_NULL: lmd.md_values[lv.lv_meta.lvm_name.to_string()] diff --git a/src/log_actions.cc b/src/log_actions.cc index 4e909c74..6e474408 100644 --- a/src/log_actions.cc +++ b/src/log_actions.cc @@ -46,7 +46,7 @@ action_delegate::execute_action(const std::string& action_name) auto& ldh = this->ad_log_helper; auto value_index = this->ad_press_value; - logline_value& lv = ldh.ldh_line_values[value_index]; + logline_value& lv = ldh.ldh_line_values.lvv_values[value_index]; auto lf = ldh.ldh_file; const auto format = lf->get_format(); pid_t child_pid; @@ -187,10 +187,12 @@ action_delegate::text_handle_mouse(textview_curses& tc, mouse_event& me) x_offset = this->ad_line_index + mouse_left; if (lr.contains(x_offset)) { for (size_t lpc = 0; - lpc < this->ad_log_helper.ldh_line_values.size(); - lpc++) { - logline_value& lv - = this->ad_log_helper.ldh_line_values[lpc]; + lpc < this->ad_log_helper.ldh_line_values.lvv_values + .size(); + lpc++) + { + auto& lv = this->ad_log_helper.ldh_line_values + .lvv_values[lpc]; if (lv.lv_origin.contains(x_offset)) { this->ad_press_value = lpc; @@ -211,8 +213,8 @@ action_delegate::text_handle_mouse(textview_curses& tc, mouse_event& me) case mouse_button_state_t::BUTTON_STATE_RELEASED: if (this->ad_press_value != -1 && this->ad_press_line == mouse_line) { - logline_value& lv - = this->ad_log_helper.ldh_line_values[this->ad_press_value]; + auto& lv = this->ad_log_helper.ldh_line_values + .lvv_values[this->ad_press_value]; int x_offset = this->ad_line_index + mouse_left; if (lv.lv_origin.contains(x_offset)) { diff --git a/src/log_data_helper.cc b/src/log_data_helper.cc index 96f53ebc..8d5af7dd 100644 --- a/src/log_data_helper.cc +++ b/src/log_data_helper.cc @@ -37,7 +37,7 @@ void log_data_helper::clear() { this->ldh_file = nullptr; - this->ldh_msg.disown(); + this->ldh_line_values.lvv_sbr.disown(); this->ldh_parser.reset(); this->ldh_scanner.reset(); this->ldh_namer.reset(); @@ -76,17 +76,16 @@ log_data_helper::parse_line(content_line_t line, bool allow_middle) this->ldh_line_attrs.clear(); this->ldh_line_values.clear(); - this->ldh_file->read_full_message(ll, this->ldh_msg); - format->annotate( - this->ldh_line_index, this->ldh_msg, sa, this->ldh_line_values); + this->ldh_file->read_full_message(ll, this->ldh_line_values.lvv_sbr); + format->annotate(this->ldh_line_index, sa, this->ldh_line_values); body = find_string_attr_range(sa, &SA_BODY); if (body.lr_start == -1) { - body.lr_start = this->ldh_msg.length(); - body.lr_end = this->ldh_msg.length(); + body.lr_start = this->ldh_line_values.lvv_sbr.length(); + body.lr_end = this->ldh_line_values.lvv_sbr.length(); } this->ldh_scanner = std::make_unique( - this->ldh_msg, body.lr_start, body.lr_end); + this->ldh_line_values.lvv_sbr, body.lr_start, body.lr_end); this->ldh_parser = std::make_unique(this->ldh_scanner.get()); this->ldh_msg_format.clear(); @@ -97,18 +96,18 @@ log_data_helper::parse_line(content_line_t line, bool allow_middle) this->ldh_json_pairs.clear(); this->ldh_xml_pairs.clear(); - for (const auto& lv : this->ldh_line_values) { + for (const auto& lv : this->ldh_line_values.lvv_values) { this->ldh_namer->cn_builtin_names.emplace_back( lv.lv_meta.lvm_name.get()); } - for (auto& ldh_line_value : this->ldh_line_values) { + for (auto& ldh_line_value : this->ldh_line_values.lvv_values) { switch (ldh_line_value.lv_meta.lvm_kind) { case value_kind_t::VALUE_JSON: { json_ptr_walk jpw; - if (jpw.parse(ldh_line_value.lv_sbr.get_data(), - ldh_line_value.lv_sbr.length()) + if (jpw.parse(ldh_line_value.text_value(), + ldh_line_value.text_length()) == yajl_status_ok && jpw.complete_parse() == yajl_status_ok) { @@ -122,8 +121,8 @@ log_data_helper::parse_line(content_line_t line, bool allow_middle) pugi::xml_document doc; auto parse_res - = doc.load_buffer(ldh_line_value.lv_sbr.get_data(), - ldh_line_value.lv_sbr.length()); + = doc.load_buffer(ldh_line_value.text_value(), + ldh_line_value.text_length()); if (parse_res) { pugi::xpath_query query("//*"); @@ -179,11 +178,12 @@ log_data_helper::get_line_bounds(size_t& line_index_out, line_index_out = line_end_index_out; line_end = (const char*) memchr( - this->ldh_msg.get_data() + line_index_out + 1, + this->ldh_line_values.lvv_sbr.get_data() + line_index_out + 1, '\n', - this->ldh_msg.length() - line_index_out - 1); + this->ldh_line_values.lvv_sbr.length() - line_index_out - 1); if (line_end != nullptr) { - line_end_index_out = line_end - this->ldh_msg.get_data(); + line_end_index_out + = line_end - this->ldh_line_values.lvv_sbr.get_data(); } else { line_end_index_out = std::string::npos; } @@ -191,7 +191,7 @@ log_data_helper::get_line_bounds(size_t& line_index_out, } while (retval <= this->ldh_y_offset); if (line_end_index_out == std::string::npos) { - line_end_index_out = this->ldh_msg.length(); + line_end_index_out = this->ldh_line_values.lvv_sbr.length(); } return retval; diff --git a/src/log_data_helper.hh b/src/log_data_helper.hh index d40fa900..235a4d8e 100644 --- a/src/log_data_helper.hh +++ b/src/log_data_helper.hh @@ -64,9 +64,10 @@ public: int get_value_line(const logline_value& lv) const { - return std::count(this->ldh_msg.get_data(), - this->ldh_msg.get_data() + lv.lv_origin.lr_start, - '\n'); + return std::count( + this->ldh_line_values.lvv_sbr.get_data(), + this->ldh_line_values.lvv_sbr.get_data() + lv.lv_origin.lr_start, + '\n'); } std::string format_json_getter(const intern_string_t field, int index); @@ -76,13 +77,12 @@ public: std::shared_ptr ldh_file; int ldh_y_offset{0}; logfile::iterator ldh_line; - shared_buffer_ref ldh_msg; content_line_t ldh_line_index; std::unique_ptr ldh_scanner; std::unique_ptr ldh_parser; std::unique_ptr ldh_namer; string_attrs_t ldh_line_attrs; - std::vector ldh_line_values; + logline_value_vector ldh_line_values; std::map ldh_json_pairs; std::map, std::string> ldh_xml_pairs; diff --git a/src/log_data_table.cc b/src/log_data_table.cc index 2e598cc5..08b3a22d 100644 --- a/src/log_data_table.cc +++ b/src/log_data_table.cc @@ -55,22 +55,21 @@ log_data_table::get_columns_int() std::shared_ptr lf = this->ldt_log_source.find(cl_copy); struct line_range body; string_attrs_t sa; - std::vector line_values; + logline_value_vector line_values; auto format = lf->get_format(); - shared_buffer_ref line; if (this->ldt_format_impl != nullptr) { this->ldt_format_impl->get_columns(cols); } - lf->read_full_message(lf->begin() + cl_copy, line); - format->annotate(cl_copy, line, sa, line_values, false); + lf->read_full_message(lf->begin() + cl_copy, line_values.lvv_sbr); + format->annotate(cl_copy, sa, line_values, false); body = find_string_attr_range(sa, &SA_BODY); if (body.lr_end == -1) { this->ldt_schema_id.clear(); return; } - data_scanner ds(line, body.lr_start, body.lr_end); + data_scanner ds(line_values.lvv_sbr, body.lr_start, body.lr_end); data_parser dp(&ds); column_namer cn{column_namer::language::SQL}; @@ -131,17 +130,16 @@ log_data_table::next(log_cursor& lc, logfile_sub_source& lss) string_attrs_t sa; struct line_range body; - std::vector line_values; + logline_value_vector line_values; - lf->read_full_message(lf_iter, this->ldt_current_line); - lf->get_format()->annotate( - cl, this->ldt_current_line, sa, line_values, false); + lf->read_full_message(lf_iter, line_values.lvv_sbr); + lf->get_format()->annotate(cl, sa, line_values, false); body = find_string_attr_range(sa, &SA_BODY); if (body.lr_end == -1) { return false; } - data_scanner ds(this->ldt_current_line, body.lr_start, body.lr_end); + data_scanner ds(line_values.lvv_sbr, body.lr_start, body.lr_end); data_parser dp(&ds); dp.parse(); @@ -162,12 +160,12 @@ log_data_table::next(log_cursor& lc, logfile_sub_source& lss) void log_data_table::extract(logfile* lf, uint64_t line_number, - shared_buffer_ref& line, - std::vector& values) + logline_value_vector& values) { + auto& line = values.lvv_sbr; auto meta_iter = this->ldt_value_metas.begin(); - this->ldt_format_impl->extract(lf, line_number, line, values); + this->ldt_format_impl->extract(lf, line_number, values); for (const auto& ldt_pair : this->ldt_pairs) { const auto& pvalue = ldt_pair.get_pair_value(); @@ -183,15 +181,16 @@ log_data_table::extract(logfile* lf, if (sscanf(scan_value, "%lf", &d) != 1) { d = 0.0; } - values.emplace_back(*meta_iter, d); + values.lvv_values.emplace_back(*meta_iter, d); break; } default: { - values.emplace_back(*meta_iter, - line, - line_range{pvalue.e_capture.c_begin, - pvalue.e_capture.c_end}); + values.lvv_values.emplace_back( + *meta_iter, + line, + line_range{pvalue.e_capture.c_begin, + pvalue.e_capture.c_end}); break; } } diff --git a/src/log_data_table.hh b/src/log_data_table.hh index 7a6c5ea3..1ca39427 100644 --- a/src/log_data_table.hh +++ b/src/log_data_table.hh @@ -64,14 +64,12 @@ public: void extract(logfile* lf, uint64_t line_number, - shared_buffer_ref& line, - std::vector& values) override; + logline_value_vector& values) override; private: logfile_sub_source& ldt_log_source; const content_line_t ldt_template_line; data_parser::schema_id_t ldt_schema_id; - shared_buffer_ref ldt_current_line; data_parser::element_list_t ldt_pairs; std::shared_ptr ldt_format_impl; std::vector ldt_cols; diff --git a/src/log_format.cc b/src/log_format.cc index 649b3b1d..b5e3fddf 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -122,7 +122,9 @@ logline_value::logline_value(logline_value_meta lvm, case value_kind_t::VALUE_QUOTED: case value_kind_t::VALUE_W3C_QUOTED: case value_kind_t::VALUE_TIMESTAMP: - this->lv_sbr.subset(sbr, origin.lr_start, origin.length()); + require(origin.lr_end != -1); + this->lv_frag = string_fragment{ + sbr.get_data(), origin.lr_start, origin.lr_end}; break; case value_kind_t::VALUE_NULL: @@ -179,33 +181,36 @@ logline_value::to_string() const case value_kind_t::VALUE_STRUCT: case value_kind_t::VALUE_TEXT: case value_kind_t::VALUE_TIMESTAMP: - if (this->lv_sbr.empty()) { + if (this->lv_str) { + return this->lv_str.value(); + } + if (this->lv_frag.empty()) { return this->lv_intern_string.to_string(); } - return {this->lv_sbr.get_data(), this->lv_sbr.length()}; + return this->lv_frag.to_string(); case value_kind_t::VALUE_QUOTED: case value_kind_t::VALUE_W3C_QUOTED: - if (this->lv_sbr.length() == 0) { + if (this->lv_frag.empty()) { return ""; } else { - switch (this->lv_sbr.get_data()[0]) { + switch (this->lv_frag.data()[0]) { case '\'': case '"': { auto unquote_func = this->lv_meta.lvm_kind == value_kind_t::VALUE_W3C_QUOTED ? unquote_w3c : unquote; - char unquoted_str[this->lv_sbr.length()]; + char unquoted_str[this->lv_frag.length()]; size_t unquoted_len; unquoted_len = unquote_func(unquoted_str, - this->lv_sbr.get_data(), - this->lv_sbr.length()); + this->lv_frag.data(), + this->lv_frag.length()); return {unquoted_str, unquoted_len}; } default: - return {this->lv_sbr.get_data(), this->lv_sbr.length()}; + return this->lv_frag.to_string(); } } break; @@ -547,15 +552,14 @@ json_array_end(void* ctx) if (ypc->ypc_path_index_stack.size() == 1) { const intern_string_t field_name = ypc->get_path_fragment_i(0); size_t sub_end = yajl_get_bytes_consumed(jlu->jlu_handle); - shared_buffer_ref sbr; - - sbr.subset(jlu->jlu_shared_buffer, - jlu->jlu_sub_start, - sub_end - jlu->jlu_sub_start); - jlu->jlu_format->jlf_line_values.emplace_back( + jlu->jlu_format->jlf_line_values.lvv_values.emplace_back( jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_JSON), - sbr); + string_fragment{ + jlu->jlu_shared_buffer.get_data(), + (int) jlu->jlu_sub_start, + (int) sub_end, + }); } return 1; @@ -583,7 +587,7 @@ rewrite_json_null(yajlpp_parse_context* ypc) if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) { return 1; } - jlu->jlu_format->jlf_line_values.emplace_back( + jlu->jlu_format->jlf_line_values.lvv_values.emplace_back( jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_NULL)); return 1; @@ -598,7 +602,7 @@ rewrite_json_bool(yajlpp_parse_context* ypc, int val) if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) { return 1; } - jlu->jlu_format->jlf_line_values.emplace_back( + jlu->jlu_format->jlf_line_values.lvv_values.emplace_back( jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_BOOLEAN), (bool) val); @@ -614,7 +618,7 @@ rewrite_json_int(yajlpp_parse_context* ypc, long long val) if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) { return 1; } - jlu->jlu_format->jlf_line_values.emplace_back( + jlu->jlu_format->jlf_line_values.lvv_values.emplace_back( jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_INTEGER), (int64_t) val); @@ -630,7 +634,7 @@ rewrite_json_double(yajlpp_parse_context* ypc, double val) if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) { return 1; } - jlu->jlu_format->jlf_line_values.emplace_back( + jlu->jlu_format->jlf_line_values.lvv_values.emplace_back( jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_FLOAT), val); @@ -899,8 +903,7 @@ external_log_format::scan(logfile& lf, pcre_input mod_pi( pi.get_substr_start(body_cap), 0, body_cap->length()); int mod_pat_index = mod_elf->last_pattern_index(); - pattern& mod_pat - = *mod_elf->elf_pattern_order[mod_pat_index]; + auto& mod_pat = *mod_elf->elf_pattern_order[mod_pat_index]; if (mod_pat.p_pcre->match( mod_pc, mod_pi, PCRE_NO_UTF8_CHECK)) { @@ -923,18 +926,15 @@ external_log_format::scan(logfile& lf, const struct scaling_factor* scaling = nullptr; if (ivd.ivd_unit_field_index >= 0) { - pcre_context::iterator unit_cap - = pc[ivd.ivd_unit_field_index]; + auto unit_cap = pc[ivd.ivd_unit_field_index]; if (unit_cap != nullptr && unit_cap->is_valid()) { intern_string_t unit_val = intern_string::lookup( pi.get_substr_start(unit_cap), unit_cap->length()); - std::map::const_iterator unit_iter; - unit_iter = vd.vd_unit_scaling.find(unit_val); + auto unit_iter = vd.vd_unit_scaling.find(unit_val); if (unit_iter != vd.vd_unit_scaling.end()) { - const struct scaling_factor& sf = unit_iter->second; + const auto& sf = unit_iter->second; scaling = &sf; } @@ -1035,11 +1035,11 @@ external_log_format::module_scan(const pcre_input& pi, void external_log_format::annotate(uint64_t line_number, - shared_buffer_ref& line, string_attrs_t& sa, - std::vector& values, + logline_value_vector& values, bool annotate_module) const { + auto& line = values.lvv_sbr; pcre_context_static<128> pc; pcre_input pi(line.get_data(), 0, line.length()); struct line_range lr; @@ -1055,10 +1055,10 @@ external_log_format::annotate(uint64_t line_number, return; } - values.reserve(this->elf_value_defs.size()); + values.lvv_values.reserve(this->elf_value_defs.size()); int pat_index = this->pattern_index_for_line(line_number); - pattern& pat = *this->elf_pattern_order[pat_index]; + auto& pat = *this->elf_pattern_order[pat_index]; sa.reserve(pat.p_pcre->get_capture_count()); if (!pat.p_pcre->match(pc, pi, PCRE_NO_UTF8_CHECK)) { @@ -1105,11 +1105,11 @@ external_log_format::annotate(uint64_t line_number, for (size_t lpc = 0; lpc < pat.p_value_by_index.size(); lpc++) { const indexed_value_def& ivd = pat.p_value_by_index[lpc]; const struct scaling_factor* scaling = nullptr; - pcre_context::capture_t* cap = pc[ivd.ivd_index]; - const value_def& vd = *ivd.ivd_value_def; + auto* cap = pc[ivd.ivd_index]; + const auto& vd = *ivd.ivd_value_def; if (ivd.ivd_unit_field_index >= 0) { - pcre_context::iterator unit_cap = pc[ivd.ivd_unit_field_index]; + auto* unit_cap = pc[ivd.ivd_unit_field_index]; if (unit_cap != nullptr && unit_cap->c_begin != -1) { intern_string_t unit_val = intern_string::lookup( @@ -1124,14 +1124,14 @@ external_log_format::annotate(uint64_t line_number, } if (cap->is_valid()) { - values.emplace_back( + values.lvv_values.emplace_back( vd.vd_meta, line, line_range{cap->c_begin, cap->c_end}); - values.back().apply_scaling(scaling); + values.lvv_values.back().apply_scaling(scaling); } else { - values.emplace_back(vd.vd_meta); + values.lvv_values.emplace_back(vd.vd_meta); } if (pat.p_module_format) { - values.back().lv_meta.lvm_from_module = true; + values.lvv_values.back().lv_meta.lvm_from_module = true; } } @@ -1146,22 +1146,23 @@ external_log_format::annotate(uint64_t line_number, if (mod_iter != MODULE_FORMATS.end() && mod_iter->second.mf_mod_format != nullptr) { - module_format& mf = mod_iter->second; - shared_buffer_ref body_ref; + auto& mf = mod_iter->second; body_cap->ltrim(line.get_data()); - body_ref.subset(line, body_cap->c_begin, body_cap->length()); - - auto pre_mod_values_size = values.size(); + auto narrow_res + = line.narrow(body_cap->c_begin, body_cap->length()); + auto pre_mod_values_size = values.lvv_values.size(); auto pre_mod_sa_size = sa.size(); - mf.mf_mod_format->annotate( - line_number, body_ref, sa, values, false); - for (size_t lpc = pre_mod_values_size; lpc < values.size(); lpc++) { - values[lpc].lv_origin.shift(0, body_cap->c_begin); + mf.mf_mod_format->annotate(line_number, sa, values, false); + for (size_t lpc = pre_mod_values_size; + lpc < values.lvv_values.size(); + lpc++) { + values.lvv_values[lpc].lv_origin.shift(0, body_cap->c_begin); } for (size_t lpc = pre_mod_sa_size; lpc < sa.size(); lpc++) { sa[lpc].sa_range.shift(0, body_cap->c_begin); } + line.widen(narrow_res); did_mod_annotate_body = true; } } @@ -1188,7 +1189,9 @@ external_log_format::rewrite(exec_context& ec, value_out.assign(line.get_data(), line.length()); - for (auto iter = values.begin(); iter != values.end(); ++iter) { + for (auto iter = values.lvv_values.begin(); iter != values.lvv_values.end(); + ++iter) + { if (!iter->lv_origin.is_valid()) { log_debug("not rewriting value with invalid origin -- %s", iter->lv_meta.lvm_name.get()); @@ -1225,8 +1228,10 @@ external_log_format::rewrite(exec_context& ec, int32_t shift_amount = field_value.length() - adj_origin.length(); value_out.insert(adj_origin.lr_start, field_value); - for (shift_iter = values.begin(); shift_iter != values.end(); - ++shift_iter) { + for (shift_iter = values.lvv_values.begin(); + shift_iter != values.lvv_values.end(); + ++shift_iter) + { shift_iter->lv_origin.shift(adj_origin.lr_start, shift_amount); } shift_string_attrs(sa, adj_origin.lr_start, shift_amount); @@ -1309,48 +1314,49 @@ rewrite_json_field(yajlpp_parse_context* ypc, sql_strftime( time_buf, sizeof(time_buf), jlu->jlu_line->get_timeval(), 'T'); } - tmp_shared_buffer tsb(time_buf); - jlu->jlu_format->jlf_line_values.emplace_back( + jlu->jlu_format->jlf_line_values.lvv_values.emplace_back( jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_TEXT), - tsb.tsb_ref); + std::string{time_buf}); } else if (jlu->jlu_shared_buffer.contains((const char*) str)) { - shared_buffer_ref sbr; - - sbr.subset(jlu->jlu_shared_buffer, - (off_t) ((const char*) str - jlu->jlu_line_value), - len); + auto str_offset = (int) ((const char*) str - jlu->jlu_line_value); if (field_name == jlu->jlu_format->elf_body_field) { - jlu->jlu_format->jlf_line_values.emplace_back( + jlu->jlu_format->jlf_line_values.lvv_values.emplace_back( jlu->jlu_format->get_value_meta(body_name, value_kind_t::VALUE_TEXT), - sbr); + string_fragment{ + jlu->jlu_shared_buffer.get_data(), + str_offset, + str_offset + (int) len, + }); } if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) { return 1; } - jlu->jlu_format->jlf_line_values.emplace_back( + jlu->jlu_format->jlf_line_values.lvv_values.emplace_back( jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_TEXT), - sbr); + string_fragment{ + jlu->jlu_shared_buffer.get_data(), + str_offset, + str_offset + (int) len, + }); } else { - tmp_shared_buffer tsb((const char*) str, len); - if (field_name == jlu->jlu_format->elf_body_field) { - jlu->jlu_format->jlf_line_values.emplace_back( + jlu->jlu_format->jlf_line_values.lvv_values.emplace_back( jlu->jlu_format->get_value_meta(body_name, value_kind_t::VALUE_TEXT), - tsb.tsb_ref); + std::string{(const char*) str, len}); } if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) { return 1; } - jlu->jlu_format->jlf_line_values.emplace_back( + jlu->jlu_format->jlf_line_values.lvv_values.emplace_back( jlu->jlu_format->get_value_meta(field_name, value_kind_t::VALUE_TEXT), - tsb.tsb_ref); + std::string{(const char*) str, len}); } return 1; @@ -1419,13 +1425,13 @@ external_log_format::get_subline(const logline& ll, SA_INVALID.value("JSON line failed to parse")); } else { std::vector::iterator lv_iter; - bool used_values[this->jlf_line_values.size()]; + bool used_values[this->jlf_line_values.lvv_values.size()]; struct line_range lr; memset(used_values, 0, sizeof(used_values)); - for (lv_iter = this->jlf_line_values.begin(); - lv_iter != this->jlf_line_values.end(); + for (lv_iter = this->jlf_line_values.lvv_values.begin(); + lv_iter != this->jlf_line_values.lvv_values.end(); ++lv_iter) { lv_iter->lv_meta.lvm_format = this; @@ -1447,10 +1453,10 @@ external_log_format::get_subline(const logline& ll, break; case json_log_field::VARIABLE: lv_iter = find_if( - this->jlf_line_values.begin(), - this->jlf_line_values.end(), + this->jlf_line_values.lvv_values.begin(), + this->jlf_line_values.lvv_values.end(), logline_value_cmp(&jfe.jfe_value.pp_value)); - if (lv_iter != this->jlf_line_values.end()) { + if (lv_iter != this->jlf_line_values.lvv_values.end()) { auto str = lv_iter->to_string(); size_t nl_pos = str.find('\n'); @@ -1520,8 +1526,9 @@ external_log_format::get_subline(const logline& ll, lr, logline::L_OPID.value()); } lv_iter->lv_origin = lr; - used_values[distance(this->jlf_line_values.begin(), - lv_iter)] + used_values[distance( + this->jlf_line_values.lvv_values.begin(), + lv_iter)] = true; } else if (jfe.jfe_value.pp_value == ts_field) { struct line_range lr; @@ -1547,12 +1554,14 @@ external_log_format::get_subline(const logline& ll, lr, logline::L_TIMESTAMP.value()); lv_iter = find_if( - this->jlf_line_values.begin(), - this->jlf_line_values.end(), + this->jlf_line_values.lvv_values.begin(), + this->jlf_line_values.lvv_values.end(), logline_value_cmp(&this->lf_timestamp_field)); - if (lv_iter != this->jlf_line_values.end()) { + if (lv_iter + != this->jlf_line_values.lvv_values.end()) { used_values[distance( - this->jlf_line_values.begin(), lv_iter)] + this->jlf_line_values.lvv_values.begin(), + lv_iter)] = true; } } else if (jfe.jfe_value.pp_value == level_field @@ -1612,10 +1621,11 @@ external_log_format::get_subline(const logline& ll, } this->json_append_to_cache("\n", 1); - for (size_t lpc = 0; lpc < this->jlf_line_values.size(); lpc++) { + for (size_t lpc = 0; lpc < this->jlf_line_values.lvv_values.size(); + lpc++) { static const intern_string_t body_name = intern_string::lookup("body", -1); - logline_value& lv = this->jlf_line_values[lpc]; + auto& lv = this->jlf_line_values.lvv_values[lpc]; if (lv.lv_meta.lvm_hidden || used_values[lpc] || body_name == lv.lv_meta.lvm_name) @@ -1688,6 +1698,7 @@ external_log_format::get_subline(const logline& ll, this->jlf_cached_line.data() + this_off, next_off - this_off); } + this->jlf_line_values.lvv_sbr = sbr; } void @@ -2437,22 +2448,19 @@ public: return lf->read_line(lf_iter) .map([this, format, cl](auto line) { - std::vector values; - shared_buffer_ref body_ref; + logline_value_vector values; struct line_range mod_name_range; intern_string_t mod_name; this->vi_attrs.clear(); - format->annotate(cl, line, this->vi_attrs, values, false); + values.lvv_sbr = line; + format->annotate(cl, this->vi_attrs, values, false); this->elt_container_body = find_string_attr_range(this->vi_attrs, &SA_BODY); if (!this->elt_container_body.is_valid()) { return false; } this->elt_container_body.ltrim(line.get_data()); - body_ref.subset(line, - this->elt_container_body.lr_start, - this->elt_container_body.length()); mod_name_range = find_string_attr_range(this->vi_attrs, &logline::L_MODULE); if (!mod_name_range.is_valid()) { @@ -2478,9 +2486,9 @@ public: void extract(logfile* lf, uint64_t line_number, - shared_buffer_ref& line, - std::vector& values) override + logline_value_vector& values) override { + auto& line = values.lvv_sbr; auto format = lf->get_format(); if (this->elt_module_format.mf_mod_format != nullptr) { @@ -2490,12 +2498,16 @@ public: this->elt_container_body.lr_start, this->elt_container_body.length()); this->vi_attrs.clear(); - values.clear(); + auto narrow_res + = values.lvv_sbr.narrow(this->elt_container_body.lr_start, + this->elt_container_body.length()); + values.lvv_values.clear(); this->elt_module_format.mf_mod_format->annotate( - line_number, body_ref, this->vi_attrs, values, false); + line_number, this->vi_attrs, values, false); + values.lvv_sbr.widen(narrow_res); } else { this->vi_attrs.clear(); - format->annotate(line_number, line, this->vi_attrs, values, false); + format->annotate(line_number, this->vi_attrs, values, false); } } diff --git a/src/log_format.hh b/src/log_format.hh index 9201913c..f2bca86c 100644 --- a/src/log_format.hh +++ b/src/log_format.hh @@ -146,29 +146,40 @@ public: { this->lv_meta.lvm_kind = value_kind_t::VALUE_NULL; } + logline_value(logline_value_meta lvm, bool b) : lv_meta(std::move(lvm)), lv_value((int64_t) (b ? 1 : 0)) { this->lv_meta.lvm_kind = value_kind_t::VALUE_BOOLEAN; } + logline_value(logline_value_meta lvm, int64_t i) : lv_meta(std::move(lvm)), lv_value(i) { this->lv_meta.lvm_kind = value_kind_t::VALUE_INTEGER; } + logline_value(logline_value_meta lvm, double i) : lv_meta(std::move(lvm)), lv_value(i) { this->lv_meta.lvm_kind = value_kind_t::VALUE_FLOAT; } - logline_value(logline_value_meta lvm, shared_buffer_ref& sbr) - : lv_meta(std::move(lvm)), lv_sbr(sbr) + + logline_value(logline_value_meta lvm, string_fragment frag) + : lv_meta(std::move(lvm)), lv_frag(frag) { } + logline_value(logline_value_meta lvm, const intern_string_t val) : lv_meta(std::move(lvm)), lv_intern_string(val) { } + + logline_value(logline_value_meta lvm, std::string val) + : lv_meta(std::move(lvm)), lv_str(std::move(val)) + { + } + logline_value(logline_value_meta lvm, shared_buffer_ref& sbr, struct line_range origin); @@ -193,21 +204,27 @@ public: const char* text_value() const { - if (this->lv_sbr.empty()) { + if (this->lv_str) { + return this->lv_str->c_str(); + } + if (this->lv_frag.empty()) { if (this->lv_intern_string.empty()) { return ""; } return this->lv_intern_string.get(); } - return this->lv_sbr.get_data(); + return this->lv_frag.data(); } size_t text_length() const { - if (this->lv_sbr.empty()) { + if (this->lv_str) { + return this->lv_str->size(); + } + if (this->lv_frag.empty()) { return this->lv_intern_string.size(); } - return this->lv_sbr.length(); + return this->lv_frag.length(); } struct line_range origin_in_full_msg(const char* msg, ssize_t len) const; @@ -217,16 +234,28 @@ public: int64_t i; double d; - value_u() : i(0){}; - value_u(int64_t i) : i(i){}; - value_u(double d) : d(d){}; + value_u() : i(0) {} + value_u(int64_t i) : i(i) {} + value_u(double d) : d(d) {} } lv_value; - shared_buffer_ref lv_sbr; + nonstd::optional lv_str; + string_fragment lv_frag; int lv_sub_offset{0}; intern_string_t lv_intern_string; struct line_range lv_origin; }; +struct logline_value_vector { + void clear() + { + this->lvv_values.clear(); + this->lvv_sbr.disown(); + } + + shared_buffer_ref lvv_sbr; + std::vector lvv_values; +}; + struct logline_value_stats { logline_value_stats() { this->clear(); } @@ -360,12 +389,11 @@ public: * * @param line The log line to edit. */ - virtual void scrub(std::string& line){}; + virtual void scrub(std::string& line) {} virtual void annotate(uint64_t line_number, - shared_buffer_ref& sbr, string_attrs_t& sa, - std::vector& values, + logline_value_vector& values, bool annotate_module = true) const { } diff --git a/src/log_format_ext.hh b/src/log_format_ext.hh index e8d38089..18f5a716 100644 --- a/src/log_format_ext.hh +++ b/src/log_format_ext.hh @@ -149,9 +149,8 @@ public: bool scan_for_partial(shared_buffer_ref& sbr, size_t& len_out) const; void annotate(uint64_t line_number, - shared_buffer_ref& line, string_attrs_t& sa, - std::vector& values, + logline_value_vector& values, bool annotate_module = true) const; void rewrite(exec_context& ec, @@ -397,12 +396,12 @@ public: bool jlf_hide_extra{false}; std::vector jlf_line_format; int jlf_line_format_init_count{0}; - std::vector jlf_line_values; + shared_buffer jlf_share_manager; + logline_value_vector jlf_line_values; off_t jlf_cached_offset{-1}; bool jlf_cached_full{false}; std::vector jlf_line_offsets; - shared_buffer jlf_share_manager; std::vector jlf_cached_line; string_attrs_t jlf_line_attrs; std::shared_ptr jlf_parse_context; diff --git a/src/log_format_impls.cc b/src/log_format_impls.cc index ad9529b3..47c1a792 100644 --- a/src/log_format_impls.cc +++ b/src/log_format_impls.cc @@ -198,11 +198,11 @@ class generic_log_format : public log_format { } void annotate(uint64_t line_number, - shared_buffer_ref& line, string_attrs_t& sa, - std::vector& values, + logline_value_vector& values, bool annotate_module) const override { + auto& line = values.lvv_sbr; int pat_index = this->pattern_index_for_line(line_number); pcre_format& fmt = get_pcre_log_formats()[pat_index]; struct line_range lr; @@ -674,14 +674,14 @@ public: } void annotate(uint64_t line_number, - shared_buffer_ref& sbr, string_attrs_t& sa, - std::vector& values, + logline_value_vector& values, bool annotate_module) const override { static const intern_string_t TS = intern_string::lookup("bro_ts"); static const intern_string_t UID = intern_string::lookup("bro_uid"); + auto& sbr = values.lvv_sbr; separated_string ss(sbr.get_data(), sbr.length()); ss.with_separator(this->blf_separator.get()); @@ -709,9 +709,9 @@ public: } if (lr.is_valid()) { - values.emplace_back(fd.fd_meta, sbr, lr); + values.lvv_values.emplace_back(fd.fd_meta, sbr, lr); } else { - values.emplace_back(fd.fd_meta); + values.lvv_values.emplace_back(fd.fd_meta); } } } @@ -1295,11 +1295,11 @@ public: } void annotate(uint64_t line_number, - shared_buffer_ref& sbr, string_attrs_t& sa, - std::vector& values, + logline_value_vector& values, bool annotate_module) const override { + auto& sbr = values.lvv_sbr; ws_separated_string ss(sbr.get_data(), sbr.length()); for (auto iter = ss.begin(); iter != ss.end(); ++iter) { @@ -1320,9 +1320,9 @@ public: auto lr = line_range(sf.sf_begin, sf.sf_end); if (lr.is_valid()) { - values.emplace_back(fd.fd_meta, sbr, lr); + values.lvv_values.emplace_back(fd.fd_meta, sbr, lr); if (sf.startswith("\"")) { - auto& meta = values.back().lv_meta; + auto& meta = values.lvv_values.back().lv_meta; if (meta.lvm_kind == value_kind_t::VALUE_TEXT) { meta.lvm_kind = value_kind_t::VALUE_W3C_QUOTED; @@ -1331,7 +1331,7 @@ public: } } } else { - values.emplace_back(fd.fd_meta); + values.lvv_values.emplace_back(fd.fd_meta); } } } @@ -1722,13 +1722,13 @@ public: } void annotate(uint64_t line_number, - shared_buffer_ref& sbr, string_attrs_t& sa, - std::vector& values, + logline_value_vector& values, bool annotate_module) const override { static const auto FIELDS_NAME = intern_string::lookup("fields"); + auto& sbr = values.lvv_sbr; auto p = logfmt::parser( string_fragment{sbr.get_data(), 0, (int) sbr.length()}); bool done = false; @@ -1738,7 +1738,7 @@ public: done = parse_result.match( [](const logfmt::parser::end_of_input&) { return true; }, - [this, &sa, &values, &sbr](const logfmt::parser::kvpair& kvp) { + [this, &sa, &values](const logfmt::parser::kvpair& kvp) { auto value_frag = kvp.second.match( [this, &kvp, &values]( const logfmt::parser::bool_value& bv) { @@ -1749,7 +1749,7 @@ public: 0, (log_format*) this} .with_struct_name(FIELDS_NAME); - values.emplace_back(lvm, bv.bv_value); + values.lvv_values.emplace_back(lvm, bv.bv_value); return bv.bv_str_value; }, @@ -1762,7 +1762,7 @@ public: 0, (log_format*) this} .with_struct_name(FIELDS_NAME); - values.emplace_back(lvm, iv.iv_value); + values.lvv_values.emplace_back(lvm, iv.iv_value); return iv.iv_str_value; }, @@ -1775,7 +1775,7 @@ public: 0, (log_format*) this} .with_struct_name(FIELDS_NAME); - values.emplace_back(lvm, fv.fv_value); + values.lvv_values.emplace_back(lvm, fv.fv_value); return fv.fv_str_value; }, @@ -1805,11 +1805,7 @@ public: 0, (log_format*) this} .with_struct_name(FIELDS_NAME); - shared_buffer_ref value_sbr; - - value_sbr.subset( - sbr, value_frag.sf_begin, value_frag.length()); - values.emplace_back(lvm, value_sbr); + values.lvv_values.emplace_back(lvm, value_frag); } return false; diff --git a/src/log_search_table.cc b/src/log_search_table.cc index 1e002a9d..32d1a1da 100644 --- a/src/log_search_table.cc +++ b/src/log_search_table.cc @@ -116,7 +116,7 @@ bool log_search_table::next(log_cursor& lc, logfile_sub_source& lss) { this->vi_attrs.clear(); - this->lst_line_values_cache.clear(); + this->lst_line_values_cache.lvv_values.clear(); if (this->lst_match_index >= 0) { this->lst_input.pi_offset = this->lst_input.pi_next_offset; @@ -158,14 +158,12 @@ log_search_table::next(log_cursor& lc, logfile_sub_source& lss) } // log_debug("%d: doing message", (int) lc.lc_curr_line); - lf->read_full_message(lf_iter, this->lst_current_line); - lf->get_format()->annotate(cl, - this->lst_current_line, - this->vi_attrs, - this->lst_line_values_cache, - false); - this->lst_input.reset( - this->lst_current_line.get_data(), 0, this->lst_current_line.length()); + lf->read_full_message(lf_iter, this->lst_line_values_cache.lvv_sbr); + lf->get_format()->annotate( + cl, this->vi_attrs, this->lst_line_values_cache, false); + this->lst_input.reset(this->lst_line_values_cache.lvv_sbr.get_data(), + 0, + this->lst_line_values_cache.lvv_sbr.length()); if (!this->lst_regex.match( this->lst_match_context, this->lst_input, PCRE_NO_UTF8_CHECK)) @@ -182,20 +180,27 @@ log_search_table::next(log_cursor& lc, logfile_sub_source& lss) void log_search_table::extract(logfile* lf, uint64_t line_number, - shared_buffer_ref& line, - std::vector& values) + logline_value_vector& values) { + auto& line = values.lvv_sbr; if (this->lst_format != nullptr) { values = this->lst_line_values_cache; } - values.emplace_back(this->lst_column_metas[this->lst_format_column_count], - this->lst_match_index); + values.lvv_values.emplace_back( + this->lst_column_metas[this->lst_format_column_count], + this->lst_match_index); for (int lpc = 0; lpc < this->lst_regex.get_capture_count(); lpc++) { const auto* cap = this->lst_match_context[lpc]; - values.emplace_back( - this->lst_column_metas[this->lst_format_column_count + 1 + lpc], - line, - line_range{cap->c_begin, cap->c_end}); + if (cap->is_valid()) { + values.lvv_values.emplace_back( + this->lst_column_metas[this->lst_format_column_count + 1 + lpc], + line, + line_range{cap->c_begin, cap->c_end}); + } else { + values.lvv_values.emplace_back( + this->lst_column_metas[this->lst_format_column_count + 1 + + lpc]); + } } } diff --git a/src/log_search_table.hh b/src/log_search_table.hh index 7b8e628c..b1db2a65 100644 --- a/src/log_search_table.hh +++ b/src/log_search_table.hh @@ -63,21 +63,19 @@ public: void extract(logfile* lf, uint64_t line_number, - shared_buffer_ref& line, - std::vector& values) override; + logline_value_vector& values) override; pcrepp lst_regex; log_format* lst_format{nullptr}; mutable size_t lst_format_column_count{0}; std::string lst_log_path_glob; nonstd::optional lst_log_level; - shared_buffer_ref lst_current_line; pcre_input lst_input{""}; pcre_context_static<128> lst_match_context; mutable std::vector lst_column_metas; int64_t lst_match_index{-1}; mutable std::vector lst_cols; - std::vector lst_line_values_cache; + logline_value_vector lst_line_values_cache; auto_buffer lst_mismatch_bitmap{auto_buffer::alloc_bitmap(0)}; int32_t lst_index_generation{0}; }; diff --git a/src/log_vtab_impl.cc b/src/log_vtab_impl.cc index 2fe53fe3..ad8c73a8 100644 --- a/src/log_vtab_impl.cc +++ b/src/log_vtab_impl.cc @@ -209,13 +209,12 @@ log_vtab_impl::get_foreign_keys(std::vector& keys_inout) const void log_vtab_impl::extract(logfile* lf, uint64_t line_number, - shared_buffer_ref& line, - std::vector& values) + logline_value_vector& values) { auto format = lf->get_format(); this->vi_attrs.clear(); - format->annotate(line_number, line, this->vi_attrs, values, false); + format->annotate(line_number, this->vi_attrs, values, false); } bool @@ -297,15 +296,14 @@ struct vtab_cursor { if (this->log_msg_line == this->log_cursor.lc_curr_line) { return; } - lf->read_full_message(ll, this->log_msg); + lf->read_full_message(ll, this->line_values.lvv_sbr); this->log_msg_line = this->log_cursor.lc_curr_line; } sqlite3_vtab_cursor base; struct log_cursor log_cursor; vis_line_t log_msg_line{-1_vl}; - shared_buffer_ref log_msg; - std::vector line_values; + logline_value_vector line_values; }; static int vt_destructor(sqlite3_vtab* p_svt); @@ -441,14 +439,14 @@ populate_indexed_columns(vtab_cursor* vc, vtab* vt) auto ll = lf->begin() + line_number; vc->cache_msg(lf, ll); - vt->vi->extract(lf, line_number, vc->log_msg, vc->line_values); + vt->vi->extract(lf, line_number, vc->line_values); } int sub_col = ic.cc_column - VT_COL_MAX; - auto lv_iter = find_if(vc->line_values.begin(), - vc->line_values.end(), + auto lv_iter = find_if(vc->line_values.lvv_values.begin(), + vc->line_values.lvv_values.end(), logline_value_cmp(nullptr, sub_col)); - if (lv_iter == vc->line_values.end() + if (lv_iter == vc->line_values.lvv_values.end() || lv_iter->lv_meta.lvm_kind == value_kind_t::VALUE_NULL) { continue; @@ -652,10 +650,9 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) char buffer[64]; if (ll->is_time_skewed()) { - if (vc->line_values.empty()) { + if (vc->line_values.lvv_values.empty()) { vc->cache_msg(lf, ll); - vt->vi->extract( - lf, line_number, vc->log_msg, vc->line_values); + vt->vi->extract(lf, line_number, vc->line_values); } struct line_range time_range; @@ -664,7 +661,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) &logline::L_TIMESTAMP); const auto* time_src - = vc->log_msg.get_data() + time_range.lr_start; + = vc->line_values.lvv_sbr.get_data() + time_range.lr_start; struct timeval actual_tv; struct exttm tm; @@ -805,10 +802,9 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) switch (footer_column) { case log_footer_columns::opid: { - if (vc->line_values.empty()) { + if (vc->line_values.lvv_values.empty()) { vc->cache_msg(lf, ll); - vt->vi->extract( - lf, line_number, vc->log_msg, vc->line_values); + vt->vi->extract(lf, line_number, vc->line_values); } auto opid_opt = get_string_attr(vt->vi->vi_attrs, @@ -819,7 +815,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) to_sqlite( ctx, - vc->log_msg.to_string_fragment( + vc->line_values.lvv_sbr.to_string_fragment( opid_range.lr_start, opid_range.length())); } else { sqlite3_result_null(ctx); @@ -872,10 +868,9 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) break; } case log_footer_columns::body: { - if (vc->line_values.empty()) { + if (vc->line_values.lvv_values.empty()) { vc->cache_msg(lf, ll); - vt->vi->extract( - lf, line_number, vc->log_msg, vc->line_values); + vt->vi->extract(lf, line_number, vc->line_values); } struct line_range body_range; @@ -885,7 +880,8 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) if (!body_range.is_valid()) { sqlite3_result_null(ctx); } else { - const char* msg_start = vc->log_msg.get_data(); + const char* msg_start + = vc->line_values.lvv_sbr.get_data(); sqlite3_result_text(ctx, &msg_start[body_range.lr_start], @@ -941,18 +937,17 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) } } } else { - if (vc->line_values.empty()) { + if (vc->line_values.lvv_values.empty()) { vc->cache_msg(lf, ll); - vt->vi->extract( - lf, line_number, vc->log_msg, vc->line_values); + vt->vi->extract(lf, line_number, vc->line_values); } int sub_col = col - VT_COL_MAX; - auto lv_iter = find_if(vc->line_values.begin(), - vc->line_values.end(), + auto lv_iter = find_if(vc->line_values.lvv_values.begin(), + vc->line_values.lvv_values.end(), logline_value_cmp(nullptr, sub_col)); - if (lv_iter != vc->line_values.end()) { + if (lv_iter != vc->line_values.lvv_values.end()) { if (!lv_iter->lv_meta.lvm_struct_name.empty()) { yajlpp_gen gen; yajl_gen_config(gen, yajl_gen_beautify, false); @@ -960,7 +955,8 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) { yajlpp_map root(gen); - for (const auto& lv_struct : vc->line_values) { + for (const auto& lv_struct : + vc->line_values.lvv_values) { if (lv_struct.lv_meta.lvm_column != sub_col) { continue; } @@ -1051,13 +1047,13 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) } case value_kind_t::VALUE_W3C_QUOTED: case value_kind_t::VALUE_QUOTED: - if (lv_iter->lv_sbr.empty()) { + if (lv_iter->text_length() == 0) { sqlite3_result_text( ctx, "", 0, SQLITE_STATIC); } else { const char* text_value - = lv_iter->lv_sbr.get_data(); - size_t text_len = lv_iter->lv_sbr.length(); + = lv_iter->text_value(); + size_t text_len = lv_iter->text_length(); switch (text_value[0]) { case '\'': @@ -1091,7 +1087,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col) sqlite3_result_text( ctx, text_value, - lv_iter->lv_sbr.length(), + lv_iter->text_length(), SQLITE_TRANSIENT); break; } diff --git a/src/log_vtab_impl.hh b/src/log_vtab_impl.hh index 5ed9a43a..64ad3ee3 100644 --- a/src/log_vtab_impl.hh +++ b/src/log_vtab_impl.hh @@ -231,8 +231,7 @@ public: virtual void extract(logfile* lf, uint64_t line_number, - shared_buffer_ref& line, - std::vector& values); + logline_value_vector& values); struct column_index { robin_hood::unordered_map> diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc index 768060a7..0b4dfbc5 100644 --- a/src/logfile_sub_source.cc +++ b/src/logfile_sub_source.cc @@ -207,7 +207,7 @@ logfile_sub_source::text_value_for_line(textview_curses& tc, format->scrub(value_out); } - shared_buffer_ref sbr; + auto& sbr = this->lss_token_values.lvv_sbr; sbr.share(this->lss_share_manager, (char*) this->lss_token_value.c_str(), @@ -217,8 +217,7 @@ logfile_sub_source::text_value_for_line(textview_curses& tc, line_range{0, (int) this->lss_token_value.length()}, SA_BODY.value()); } else { - format->annotate( - line, sbr, this->lss_token_attrs, this->lss_token_values); + format->annotate(line, this->lss_token_attrs, this->lss_token_values); } if (this->lss_token_line->get_sub_offset() != 0) { this->lss_token_attrs.clear(); @@ -369,7 +368,7 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv, attrs |= A_UNDERLINE; } - const std::vector& line_values = this->lss_token_values; + const auto& line_values = this->lss_token_values; lr.lr_start = 0; lr.lr_end = this->lss_token_value.length(); @@ -393,7 +392,7 @@ logfile_sub_source::text_attrs_for_line(textview_curses& lv, } } - for (const auto& line_value : line_values) { + for (const auto& line_value : line_values.lvv_values) { if ((!(this->lss_token_flags & RF_FULL) && line_value.lv_sub_offset != this->lss_token_line->get_sub_offset()) @@ -1385,13 +1384,14 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt, auto* lf = (*ld)->get_file_ptr(); char timestamp_buffer[64]; - shared_buffer_ref sbr, raw_sbr; + shared_buffer_ref raw_sbr; + logline_value_vector values; + auto& sbr = values.lvv_sbr; lf->read_full_message(ll, sbr); auto format = lf->get_format(); string_attrs_t sa; - std::vector values; auto line_number = std::distance(lf->cbegin(), ll); - format->annotate(line_number, sbr, sa, values); + format->annotate(line_number, sa, values); sqlite3_reset(stmt); sqlite3_clear_bindings(stmt); @@ -1554,7 +1554,7 @@ logfile_sub_source::eval_sql_filter(sqlite3_stmt* stmt, } continue; } - for (const auto& lv : values) { + for (const auto& lv : values.lvv_values) { if (lv.lv_meta.lvm_name != &name[1]) { continue; } @@ -2003,7 +2003,6 @@ logline_window::logmsg_info::next_msg() { this->li_file = nullptr; this->li_logline = logfile::iterator{}; - this->li_msg_buffer.disown(); this->li_string_attrs.clear(); this->li_line_values.clear(); ++this->li_line; @@ -2033,9 +2032,9 @@ logline_window::logmsg_info::load_msg() const } auto format = this->li_file->get_format(); - this->li_file->read_full_message(this->li_logline, this->li_msg_buffer); + this->li_file->read_full_message(this->li_logline, + this->li_line_values.lvv_sbr); format->annotate(std::distance(this->li_file->cbegin(), this->li_logline), - this->li_msg_buffer, this->li_string_attrs, this->li_line_values, false); @@ -2046,7 +2045,8 @@ logline_window::logmsg_info::to_string(const struct line_range& lr) const { this->load_msg(); - return this->li_msg_buffer.to_string_fragment(lr.lr_start, lr.length()) + return this->li_line_values.lvv_sbr + .to_string_fragment(lr.lr_start, lr.length()) .to_string(); } @@ -2173,12 +2173,12 @@ logfile_sub_source::text_crumbs_for_line(int line, uniq_path.template get())); }); - shared_buffer_ref sbr; string_attrs_t sa; - std::vector values; + logline_value_vector values; + auto& sbr = values.lvv_sbr; lf->read_full_message(msg_start_iter, sbr); - format->annotate(file_line_number, sbr, sa, values); + format->annotate(file_line_number, sa, values); auto opid_opt = get_string_attr(sa, logline::L_OPID); if (opid_opt && !opid_opt.value().saw_string_attr->sa_range.empty()) { diff --git a/src/logfile_sub_source.hh b/src/logfile_sub_source.hh index 6c35ccbf..ff8e37f5 100644 --- a/src/logfile_sub_source.hh +++ b/src/logfile_sub_source.hh @@ -180,7 +180,7 @@ public: return this->li_string_attrs; } - const std::vector& get_values() const + const logline_value_vector& get_values() const { this->load_msg(); return this->li_line_values; @@ -198,9 +198,8 @@ public: vis_line_t li_line; logfile* li_file{nullptr}; logfile::const_iterator li_logline; - mutable shared_buffer_ref li_msg_buffer; mutable string_attrs_t li_string_attrs; - mutable std::vector li_line_values; + mutable logline_value_vector li_line_values; }; class iterator { @@ -978,7 +977,7 @@ private: lnav::document::metadata lss_token_meta; int lss_token_meta_line{-1}; int lss_token_meta_size{0}; - std::vector lss_token_values; + logline_value_vector lss_token_values; int lss_token_shift_start{0}; int lss_token_shift_size{0}; shared_buffer lss_share_manager; diff --git a/src/readline_possibilities.cc b/src/readline_possibilities.cc index 08f16afd..35d17b7e 100644 --- a/src/readline_possibilities.cc +++ b/src/readline_possibilities.cc @@ -249,13 +249,12 @@ add_filter_expr_possibilities(readline_curses* rlc, } auto format = lf->get_format(); - shared_buffer_ref sbr; string_attrs_t sa; - std::vector values; + logline_value_vector values; - lf->read_full_message(ll, sbr); - format->annotate(cl, sbr, sa, values); - for (auto& lv : values) { + lf->read_full_message(ll, values.lvv_sbr); + format->annotate(cl, sa, values); + for (auto& lv : values.lvv_values) { if (!lv.lv_meta.lvm_struct_name.empty()) { continue; } diff --git a/src/shared_buffer.cc b/src/shared_buffer.cc index 357c91b7..fcdbc1ae 100644 --- a/src/shared_buffer.cc +++ b/src/shared_buffer.cc @@ -162,3 +162,18 @@ shared_buffer_ref::copy_ref(const shared_buffer_ref& other) this->sb_length = other.sb_length; } } + +shared_buffer_ref::narrow_result +shared_buffer_ref::narrow(size_t new_data, size_t new_length) +{ + return std::make_pair( + std::exchange(this->sb_data, this->sb_data + new_data), + std::exchange(this->sb_length, new_length)); +} + +void +shared_buffer_ref::widen(narrow_result old_data_length) +{ + this->sb_data = old_data_length.first; + this->sb_length = old_data_length.second; +} diff --git a/src/shared_buffer.hh b/src/shared_buffer.hh index 8e372133..a21db244 100644 --- a/src/shared_buffer.hh +++ b/src/shared_buffer.hh @@ -126,6 +126,11 @@ public: return string_fragment{this->sb_data, 0, (int) this->length()}; } + using narrow_result = std::pair; + narrow_result narrow(size_t new_data, size_t new_length); + + void widen(narrow_result old_data_length); + void share(shared_buffer& sb, char* data, size_t len); bool subset(shared_buffer_ref& other, off_t offset, size_t len); diff --git a/src/spectro_impls.cc b/src/spectro_impls.cc index f26fdfaf..e5f496d7 100644 --- a/src/spectro_impls.cc +++ b/src/spectro_impls.cc @@ -186,11 +186,11 @@ log_spectro_value_source::spectro_row(spectrogram_request& sr, } const auto& values = msg_info.get_values(); - auto lv_iter = find_if(values.begin(), - values.end(), + auto lv_iter = find_if(values.lvv_values.begin(), + values.lvv_values.end(), logline_value_cmp(&this->lsvs_colname)); - if (lv_iter != values.end()) { + if (lv_iter != values.lvv_values.end()) { switch (lv_iter->lv_meta.lvm_kind) { case value_kind_t::VALUE_FLOAT: row_out.add_value(sr, lv_iter->lv_value.d, ll.is_marked()); @@ -223,11 +223,11 @@ log_spectro_value_source::spectro_row(spectrogram_request& sr, } const auto& values = msg_info.get_values(); - auto lv_iter = find_if(values.begin(), - values.end(), + auto lv_iter = find_if(values.lvv_values.begin(), + values.lvv_values.end(), logline_value_cmp(&this->lsvs_colname)); - if (lv_iter != values.end()) { + if (lv_iter != values.lvv_values.end()) { switch (lv_iter->lv_meta.lvm_kind) { case value_kind_t::VALUE_FLOAT: if (range_min <= lv_iter->lv_value.d @@ -266,7 +266,7 @@ log_spectro_value_source::spectro_mark(textview_curses& tc, vis_line_t begin_line = lss.find_from_time(begin_time).value_or(0_vl); vis_line_t end_line = lss.find_from_time(end_time).value_or(lss.text_line_count()); - std::vector values; + logline_value_vector values; string_attrs_t sa; for (vis_line_t curr_line = begin_line; curr_line < end_line; ++curr_line) { @@ -274,22 +274,21 @@ log_spectro_value_source::spectro_mark(textview_curses& tc, std::shared_ptr lf = lss.find(cl); auto ll = lf->begin() + cl; auto format = lf->get_format(); - shared_buffer_ref sbr; if (!ll->is_message()) { continue; } - lf->read_full_message(ll, sbr); - sa.clear(); values.clear(); - format->annotate(cl, sbr, sa, values, false); + lf->read_full_message(ll, values.lvv_sbr); + sa.clear(); + format->annotate(cl, sa, values, false); - auto lv_iter = find_if(values.begin(), - values.end(), + auto lv_iter = find_if(values.lvv_values.begin(), + values.lvv_values.end(), logline_value_cmp(&this->lsvs_colname)); - if (lv_iter != values.end()) { + if (lv_iter != values.lvv_values.end()) { switch (lv_iter->lv_meta.lvm_kind) { case value_kind_t::VALUE_FLOAT: if (range_min <= lv_iter->lv_value.d diff --git a/test/drive_data_scanner.cc b/test/drive_data_scanner.cc index 1b466cc1..63f40404 100644 --- a/test/drive_data_scanner.cc +++ b/test/drive_data_scanner.cc @@ -144,7 +144,8 @@ main(int argc, char* argv[]) auto sub_line = line.substr(13); struct line_range body(0, sub_line.length()); shared_buffer share_manager; - shared_buffer_ref sbr; + logline_value_vector ll_values; + auto& sbr = ll_values.lvv_sbr; sbr.share( share_manager, (char*) sub_line.c_str(), sub_line.size()); @@ -178,11 +179,10 @@ main(int argc, char* argv[]) } } - std::vector ll_values; string_attrs_t sa; if (format.get() != nullptr) { - format->annotate(0, sbr, sa, ll_values); + format->annotate(0, sa, ll_values); body = find_string_attr_range(sa, &SA_BODY); } diff --git a/test/expected/test_sql_search_table.sh_5aaae556ecb1661602f176215e28f661d3404032.out b/test/expected/test_sql_search_table.sh_5aaae556ecb1661602f176215e28f661d3404032.out index db7b9054..f6f9606d 100644 --- a/test/expected/test_sql_search_table.sh_5aaae556ecb1661602f176215e28f661d3404032.out +++ b/test/expected/test_sql_search_table.sh_5aaae556ecb1661602f176215e28f661d3404032.out @@ -1,4 +1,4 @@ log_line log_part  log_time log_idle_msecs log_level log_mark log_comment log_tags log_filters match_index user pid cpu_pct mem_pct vsz rss tty stat start_time cpu_time  cmd  cmd_name cmd_args  - 0  <NULL> 2022-06-02 00:01:01.000  0 info   0  <NULL>  <NULL>  <NULL>  1 root  2  0  0  0  0 ?  S  Jun01  0:00  [kthreadd] [kthreadd]   - 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  1 root  2  0  0  0  0 ?  S  Jun01  0:00  [kthreadd] [kthreadd]   - 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  1 root  2  0  0  0  0 ?  S  Jun01  0:00  [kthreadd] [kthreadd]    + 0  <NULL> 2022-06-02 00:01:01.000  0 info   0  <NULL>  <NULL>  <NULL>  1 root  2  0  0  0  0 ?  S  Jun01  0:00  [kthreadd] [kthreadd]  <NULL>  + 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  1 root  2  0  0  0  0 ?  S  Jun01  0:00  [kthreadd] [kthreadd]  <NULL> + 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  1 root  2  0  0  0  0 ?  S  Jun01  0:00  [kthreadd] [kthreadd]  <NULL>  diff --git a/test/expected/test_sql_search_table.sh_df0fd242f57a96d40f466493938cda0789a094fa.out b/test/expected/test_sql_search_table.sh_df0fd242f57a96d40f466493938cda0789a094fa.out index b141c1f3..08e4905a 100644 --- a/test/expected/test_sql_search_table.sh_df0fd242f57a96d40f466493938cda0789a094fa.out +++ b/test/expected/test_sql_search_table.sh_df0fd242f57a96d40f466493938cda0789a094fa.out @@ -1,24 +1,24 @@ log_line log_part  log_time log_idle_msecs log_level log_mark log_comment log_tags log_filters match_index user pid cpu_pct mem_pct  vsz rss tty stat start_time cpu_time  cmd  cmd_name  cmd_args   0  <NULL> 2022-06-02 00:01:01.000  0 info   0  <NULL>  <NULL>  <NULL>  0 root  1  0  0 158392 7792 ?  Ss  Jun01  0:14  /lib/systemd/systemd --switched-root --system --deserialize 16 /lib/systemd/systemd  --switched-root --system --deserialize 16  - 0 2022-06-02 00:01:01.000 0 info 0 1 root 2 0 0 0 0 ? S Jun01 0:00 [kthreadd] [kthreadd] - 0  <NULL> 2022-06-02 00:01:01.000  0 info   0  <NULL>  <NULL>  <NULL>  2 root  3  0  0  0  0 ?  I<  Jun01  0:00  [rcu_gp]  [rcu_gp]    - 0 2022-06-02 00:01:01.000 0 info 0 3 root 4 0 0 0 0 ? I< Jun01 0:00 [rcu_par_gp] [rcu_par_gp] - 0  <NULL> 2022-06-02 00:01:01.000  0 info   0  <NULL>  <NULL>  <NULL>  4 root  6  0  0  0  0 ?  I<  Jun01  0:00  [kworker/0:0H-kblockd]  [kworker/0:0H-kblockd]   + 0 2022-06-02 00:01:01.000 0 info 0 1 root 2 0 0 0 0 ? S Jun01 0:00 [kthreadd] [kthreadd] + 0  <NULL> 2022-06-02 00:01:01.000  0 info   0  <NULL>  <NULL>  <NULL>  2 root  3  0  0  0  0 ?  I<  Jun01  0:00  [rcu_gp]  [rcu_gp]  <NULL>  + 0 2022-06-02 00:01:01.000 0 info 0 3 root 4 0 0 0 0 ? I< Jun01 0:00 [rcu_par_gp] [rcu_par_gp] + 0  <NULL> 2022-06-02 00:01:01.000  0 info   0  <NULL>  <NULL>  <NULL>  4 root  6  0  0  0  0 ?  I<  Jun01  0:00  [kworker/0:0H-kblockd]  [kworker/0:0H-kblockd] <NULL>   12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  0 root  1  0  0 158392 7792 ?  Ss  Jun01  0:14  /lib/systemd/systemd --switched-root --system --deserialize 16 /lib/systemd/systemd  --switched-root --system --deserialize 16 - 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  1 root  2  0  0  0  0 ?  S  Jun01  0:00  [kthreadd]  [kthreadd]    - 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL> 2 root 3 0 0 0 0 ? I< Jun01 0:00 [rcu_gp] [rcu_gp] - 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  3 root  4  0  0  0  0 ?  I<  Jun01  0:00  [rcu_par_gp]  [rcu_par_gp]    - 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL> 4 root 6 0 0 0 0 ? I< Jun01 0:00 [kworker/0:0H-kblockd] [kworker/0:0H-kblockd] - 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  5 root  8  0  0  0  0 ?  I<  Jun01  0:00  [mm_percpu_wq]  [mm_percpu_wq]    - 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL> 6 root 9 0 0 0 0 ? S Jun01 0:00 [ksoftirqd/0] [ksoftirqd/0] - 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  7 root  10  0  0  0  0 ?  I  Jun01  0:23  [rcu_sched]  [rcu_sched]    - 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL> 8 root 11 0 0 0 0 ? I Jun01 0:00 [rcu_bh] [rcu_bh] - 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  9 root  12  0  0  0  0 ?  S  Jun01  0:00  [migration/0]  [migration/0]    - 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL> 10 root 14 0 0 0 0 ? S Jun01 0:00 [cpuhp/0] [cpuhp/0] + 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  1 root  2  0  0  0  0 ?  S  Jun01  0:00  [kthreadd]  [kthreadd]  <NULL>  + 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL> 2 root 3 0 0 0 0 ? I< Jun01 0:00 [rcu_gp] [rcu_gp] + 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  3 root  4  0  0  0  0 ?  I<  Jun01  0:00  [rcu_par_gp]  [rcu_par_gp]  <NULL>  + 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL> 4 root 6 0 0 0 0 ? I< Jun01 0:00 [kworker/0:0H-kblockd] [kworker/0:0H-kblockd] + 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  5 root  8  0  0  0  0 ?  I<  Jun01  0:00  [mm_percpu_wq]  [mm_percpu_wq]  <NULL>  + 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL> 6 root 9 0 0 0 0 ? S Jun01 0:00 [ksoftirqd/0] [ksoftirqd/0] + 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  7 root  10  0  0  0  0 ?  I  Jun01  0:23  [rcu_sched]  [rcu_sched]  <NULL>  + 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL> 8 root 11 0 0 0 0 ? I Jun01 0:00 [rcu_bh] [rcu_bh] + 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  9 root  12  0  0  0  0 ?  S  Jun01  0:00  [migration/0]  [migration/0]  <NULL>  + 12  <NULL> 2022-06-02 00:02:01.000  60000 info   0  <NULL> 10 root 14 0 0 0 0 ? S Jun01 0:00 [cpuhp/0] [cpuhp/0]  30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  0 root  1  0  0 158392 7792 ?  Ss  Jun01  0:14  /lib/systemd/systemd --switched-root --system --deserialize 16 /lib/systemd/systemd  --switched-root --system --deserialize 16  - 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL> 1 root 2 0 0 0 0 ? S Jun01 0:00 [kthreadd] [kthreadd] - 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  2 root  3  0  0  0  0 ?  I<  Jun01  0:00  [rcu_gp]  [rcu_gp]    - 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL> 3 root 4 0 0 0 0 ? I< Jun01 0:00 [rcu_par_gp] [rcu_par_gp] - 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  4 root  6  0  0  0  0 ?  I<  Jun01  0:00  [kworker/0:0H-kblockd]  [kworker/0:0H-kblockd]   - 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL> 5 root 8 0 0 0 0 ? I< Jun01 0:00 [mm_percpu_wq] [mm_percpu_wq] - 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  6 root  9  0  0  0  0 ?  S  Jun01  0:00  [ksoftirqd/0]  [ksoftirqd/0]    + 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL> 1 root 2 0 0 0 0 ? S Jun01 0:00 [kthreadd] [kthreadd] + 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  2 root  3  0  0  0  0 ?  I<  Jun01  0:00  [rcu_gp]  [rcu_gp]  <NULL>  + 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL> 3 root 4 0 0 0 0 ? I< Jun01 0:00 [rcu_par_gp] [rcu_par_gp] + 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  4 root  6  0  0  0  0 ?  I<  Jun01  0:00  [kworker/0:0H-kblockd]  [kworker/0:0H-kblockd] <NULL>  + 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL> 5 root 8 0 0 0 0 ? I< Jun01 0:00 [mm_percpu_wq] [mm_percpu_wq] + 30  <NULL> 2022-06-02 00:03:01.000  60000 info   0  <NULL>  <NULL>  <NULL>  6 root  9  0  0  0  0 ?  S  Jun01  0:00  [ksoftirqd/0]  [ksoftirqd/0]  <NULL>