diff --git a/src/lnav.indexing.cc b/src/lnav.indexing.cc index 9006c26a..fe709de6 100644 --- a/src/lnav.indexing.cc +++ b/src/lnav.indexing.cc @@ -290,8 +290,6 @@ rebuild_indexes(nonstd::optional deadline) scroll_downs[LNV_LOG] = false; } - log_view.reload_data(); - { std::unordered_map>> id_to_files; diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index 142e0a91..18c80ab8 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -3847,7 +3847,7 @@ com_toggle_field(exec_context& ec, // TODO: highlight the fields to be hidden. retval = ""; } else { - logfile_sub_source& lss = lnav_data.ld_log_source; + auto& lss = lnav_data.ld_log_source; bool hide = args[0] == "hide-fields"; std::vector found_fields, missing_fields; @@ -3870,8 +3870,8 @@ com_toggle_field(exec_context& ec, } else if (tc->get_inner_height() == 0) { return ec.make_error("no log messages to hide"); } else { - content_line_t cl = lss.at(tc->get_selection()); - std::shared_ptr lf = lss.find(cl); + auto cl = lss.at(tc->get_selection()); + auto lf = lss.find(cl); format = lf->get_format(); name = intern_string::lookup(args[lpc]); } diff --git a/src/log_format.cc b/src/log_format.cc index 4f7598aa..87f4541f 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -2926,6 +2926,7 @@ external_log_format::specialized(int fmt_lock) this->lf_value_stats.clear(); this->lf_value_stats.resize(this->elf_value_defs.size()); + this->elf_specialized_value_defs_state = *this->elf_value_defs_state; return retval; } @@ -3250,6 +3251,72 @@ external_log_format::get_value_metadata() const return retval; } +const logline_value_stats* +external_log_format::stats_for_value(const intern_string_t& name) const +{ + auto iter = this->elf_value_defs.find(name); + if (iter != this->elf_value_defs.end() + && iter->second->vd_meta.lvm_values_index) + { + return &this->lf_value_stats[iter->second->vd_meta.lvm_values_index + .value()]; + } + + return nullptr; +} + +std::string +external_log_format::get_pattern_regex(uint64_t line_number) const +{ + if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) { + return ""; + } + int pat_index = this->pattern_index_for_line(line_number); + return this->elf_pattern_order[pat_index]->p_pcre.pp_value->get_pattern(); +} + +bool +external_log_format::hide_field(const intern_string_t field_name, bool val) +{ + auto vd_iter = this->elf_value_defs.find(field_name); + + if (vd_iter == this->elf_value_defs.end()) { + return false; + } + + vd_iter->second->vd_meta.lvm_user_hidden = val; + if (this->elf_type == elf_type_t::ELF_TYPE_JSON) { + bool found = false; + + for (const auto& jfe : this->jlf_line_format) { + if (jfe.jfe_value.pp_value == field_name) { + found = true; + } + } + if (!found) { + log_info("format field %s.%s changed, rebuilding", + this->elf_name.get(), + field_name.get()); + this->elf_value_defs_state->vds_generation += 1; + } + } + return true; +} + +bool +external_log_format::format_changed() +{ + if (this->elf_specialized_value_defs_state.vds_generation + != this->elf_value_defs_state->vds_generation) + { + this->elf_specialized_value_defs_state = *this->elf_value_defs_state; + this->jlf_cached_offset = -1; + return true; + } + + return false; +} + bool format_tag_def::path_restriction::matches(const char* fn) const { diff --git a/src/log_format.hh b/src/log_format.hh index 319592f0..fdca056c 100644 --- a/src/log_format.hh +++ b/src/log_format.hh @@ -118,7 +118,13 @@ struct logline_value_meta { { } - bool is_hidden() const { return this->lvm_hidden || this->lvm_user_hidden; } + bool is_hidden() const + { + if (this->lvm_user_hidden) { + return this->lvm_user_hidden.value(); + } + return this->lvm_hidden; + } logline_value_meta& with_struct_name(intern_string_t name) { @@ -132,7 +138,7 @@ struct logline_value_meta { nonstd::optional lvm_values_index; bool lvm_identifier{false}; bool lvm_hidden{false}; - bool lvm_user_hidden{false}; + nonstd::optional lvm_user_hidden; bool lvm_from_module{false}; intern_string_t lvm_struct_name; nonstd::optional lvm_format; @@ -479,6 +485,8 @@ public: return {}; } + virtual bool format_changed() { return false; } + struct pattern_for_lines { pattern_for_lines(uint32_t pfl_line, uint32_t pfl_pat_index); diff --git a/src/log_format_ext.hh b/src/log_format_ext.hh index c475d967..f1022b75 100644 --- a/src/log_format_ext.hh +++ b/src/log_format_ext.hh @@ -137,29 +137,30 @@ public: this->jlf_line_offsets.reserve(128); } - const intern_string_t get_name() const { return this->elf_name; } + const intern_string_t get_name() const override { return this->elf_name; } - bool match_name(const std::string& filename); + bool match_name(const std::string& filename) override; - bool match_mime_type(const file_format_t ff) const; + bool match_mime_type(const file_format_t ff) const override; scan_result_t scan(logfile& lf, std::vector& dst, const line_info& offset, shared_buffer_ref& sbr, - scan_batch_context& sbc); + scan_batch_context& sbc) override; - bool scan_for_partial(shared_buffer_ref& sbr, size_t& len_out) const; + bool scan_for_partial(shared_buffer_ref& sbr, + size_t& len_out) const override; void annotate(uint64_t line_number, string_attrs_t& sa, logline_value_vector& values, - bool annotate_module = true) const; + bool annotate_module = true) const override; void rewrite(exec_context& ec, shared_buffer_ref& line, string_attrs_t& sa, - std::string& value_out); + std::string& value_out) override; void build(std::vector& errors); @@ -168,41 +169,21 @@ public: bool match_samples(const std::vector& samples) const; - bool hide_field(const intern_string_t field_name, bool val) - { - auto vd_iter = this->elf_value_defs.find(field_name); - - if (vd_iter == this->elf_value_defs.end()) { - return false; - } - - vd_iter->second->vd_meta.lvm_user_hidden = val; - return true; - } + bool hide_field(const intern_string_t field_name, bool val) override; - std::shared_ptr specialized(int fmt_lock); + std::shared_ptr specialized(int fmt_lock) override; const logline_value_stats* stats_for_value( - const intern_string_t& name) const - { - auto iter = this->elf_value_defs.find(name); - if (iter != this->elf_value_defs.end() - && iter->second->vd_meta.lvm_values_index) - { - return &this->lf_value_stats[iter->second->vd_meta.lvm_values_index - .value()]; - } - - return nullptr; - } + const intern_string_t& name) const override; void get_subline(const logline& ll, shared_buffer_ref& sbr, - bool full_message); + bool full_message) override; - std::shared_ptr get_vtab_impl() const; + std::shared_ptr get_vtab_impl() const override; - const std::vector* get_actions(const logline_value& lv) const + const std::vector* get_actions( + const logline_value& lv) const override { const std::vector* retval = nullptr; @@ -214,12 +195,14 @@ public: return retval; } - std::set get_source_path() const + bool format_changed() override; + + std::set get_source_path() const override { return this->elf_source_path; } - std::vector get_value_metadata() const; + std::vector get_value_metadata() const override; enum class json_log_field { CONSTANT, @@ -305,7 +288,7 @@ public: return iter != this->elf_value_defs.end(); } - std::string get_pattern_path(uint64_t line_number) const + std::string get_pattern_path(uint64_t line_number) const override { if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) { return "structured"; @@ -314,17 +297,9 @@ public: return this->elf_pattern_order[pat_index]->p_config_path; } - intern_string_t get_pattern_name(uint64_t line_number) const; + intern_string_t get_pattern_name(uint64_t line_number) const override; - std::string get_pattern_regex(uint64_t line_number) const - { - if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) { - return ""; - } - int pat_index = this->pattern_index_for_line(line_number); - return this->elf_pattern_order[pat_index] - ->p_pcre.pp_value->get_pattern(); - } + std::string get_pattern_regex(uint64_t line_number) const override; log_level_t convert_level(string_fragment str, scan_batch_context* sbc) const; @@ -345,6 +320,15 @@ public: std::vector elf_samples; std::unordered_map> elf_value_defs; + + struct value_defs_state { + size_t vds_generation{0}; + }; + + std::shared_ptr elf_value_defs_state{ + std::make_shared()}; + value_defs_state elf_specialized_value_defs_state; + std::vector> elf_value_def_order; std::vector> elf_numeric_value_defs; int elf_column_count{0}; diff --git a/src/logfile.cc b/src/logfile.cc index eaa9e313..d057cf2a 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -455,6 +455,16 @@ logfile::rebuild_index(nonstd::optional deadline) return rebuild_result_t::NO_NEW_LINES; } + if (this->lf_format != nullptr && this->lf_format->format_changed()) { + log_info("%s: format has changed, rebuilding", + this->lf_filename.c_str()); + this->lf_index.clear(); + this->lf_index_size = 0; + this->lf_partial_line = false; + this->lf_longest_line = 0; + this->lf_sort_needed = true; + } + auto retval = rebuild_result_t::NO_NEW_LINES; struct stat st; @@ -555,8 +565,7 @@ logfile::rebuild_index(nonstd::optional deadline) this->lf_logline_observer->logline_restart(*this, rollback_size); } - bool sort_needed = this->lf_sort_needed; - this->lf_sort_needed = false; + bool sort_needed = std::exchange(this->lf_sort_needed, false); size_t limit = SIZE_MAX; if (deadline) { diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc index 62ac21a2..ab2429d9 100644 --- a/src/logfile_sub_source.cc +++ b/src/logfile_sub_source.cc @@ -1039,14 +1039,17 @@ logfile_sub_source::rebuild_index( case rebuild_result::rr_full_rebuild: log_debug("redoing search"); this->lss_index_generation += 1; + this->tss_view->reload_data(); this->tss_view->redo_search(); break; case rebuild_result::rr_partial_rebuild: log_debug("redoing search from: %d", (int) search_start); this->lss_index_generation += 1; + this->tss_view->reload_data(); this->tss_view->search_new_data(search_start); break; case rebuild_result::rr_appended_lines: + this->tss_view->reload_data(); this->tss_view->search_new_data(); break; } diff --git a/src/session_data.cc b/src/session_data.cc index acc0f7c7..d9b28810 100644 --- a/src/session_data.cc +++ b/src/session_data.cc @@ -1526,9 +1526,16 @@ save_session_with_id(const std::string& session_id) continue; } - cmd_array.gen("hide-fields " - + elf->get_name().to_string() - + "." + vd.first.to_string()); + if (vd.second->vd_meta.lvm_user_hidden.value()) + { + cmd_array.gen("hide-fields " + + elf->get_name().to_string() + + "." + vd.first.to_string()); + } else if (vd.second->vd_meta.lvm_hidden) { + cmd_array.gen("show-fields " + + elf->get_name().to_string() + + "." + vd.first.to_string()); + } } } @@ -1654,8 +1661,15 @@ reset_session() continue; } + bool changed = false; for (const auto& vd : elf->elf_value_defs) { - vd.second->vd_meta.lvm_user_hidden = false; + if (vd.second->vd_meta.lvm_user_hidden) { + vd.second->vd_meta.lvm_user_hidden = nonstd::nullopt; + changed = true; + } + } + if (changed) { + elf->elf_value_defs_state->vds_generation += 1; } } } diff --git a/src/textview_curses.cc b/src/textview_curses.cc index 15acedce..0d89a1df 100644 --- a/src/textview_curses.cc +++ b/src/textview_curses.cc @@ -825,7 +825,7 @@ void text_time_translator::scroll_invoked(textview_curses* tc) { if (tc->get_inner_height() > 0) { - this->time_for_row(tc->get_top()) | + this->time_for_row(tc->get_selection()) | [this](auto new_top_time) { this->ttt_top_time = new_top_time; }; } } @@ -836,22 +836,23 @@ text_time_translator::data_reloaded(textview_curses* tc) if (tc->get_inner_height() == 0) { return; } - if (tc->get_top() > tc->get_inner_height()) { + if (tc->get_selection() > tc->get_inner_height()) { if (this->ttt_top_time.tv_sec != 0) { this->row_for_time(this->ttt_top_time) | - [tc](auto new_top) { tc->set_top(new_top); }; + [tc](auto new_top) { tc->set_selection(new_top); }; } return; } - this->time_for_row(tc->get_top()) | [this, tc](auto top_time) { + this->time_for_row(tc->get_selection()) | [this, tc](auto top_time) { if (top_time != this->ttt_top_time) { if (this->ttt_top_time.tv_sec != 0) { this->row_for_time(this->ttt_top_time) | - [tc](auto new_top) { tc->set_top(new_top); }; + [tc](auto new_top) { tc->set_selection(new_top); }; } - this->time_for_row(tc->get_top()) | [this](auto new_top_time) { - this->ttt_top_time = new_top_time; - }; + this->time_for_row(tc->get_selection()) | + [this](auto new_top_time) { + this->ttt_top_time = new_top_time; + }; } }; } diff --git a/test/expected/expected.am b/test/expected/expected.am index e359d3f2..6a88ee82 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -296,6 +296,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_json_format.sh_d7362cffc8335c2fe6b6527315de59bd6f5dcc7f.out \ $(srcdir)/%reldir%/test_json_format.sh_dfff27a651650a04d93de9a06ab5480e94ce3a79.err \ $(srcdir)/%reldir%/test_json_format.sh_dfff27a651650a04d93de9a06ab5480e94ce3a79.out \ + $(srcdir)/%reldir%/test_json_format.sh_e36401aa54bc61de71f8dcbe66ea16effa59ea52.err \ + $(srcdir)/%reldir%/test_json_format.sh_e36401aa54bc61de71f8dcbe66ea16effa59ea52.out \ $(srcdir)/%reldir%/test_json_format.sh_f740026626ab554dacb249762d8be7d6539b8c6e.err \ $(srcdir)/%reldir%/test_json_format.sh_f740026626ab554dacb249762d8be7d6539b8c6e.out \ $(srcdir)/%reldir%/test_json_format.sh_fe19b7ebd349cd689b3f5c22618eab5ce995e68e.err \ diff --git a/test/test_json_format.sh b/test/test_json_format.sh index 8a50ad29..435e8e32 100644 --- a/test/test_json_format.sh +++ b/test/test_json_format.sh @@ -150,3 +150,7 @@ run_cap_test ${lnav_test} -n \ run_cap_test ${lnav_test} -n \ ${test_dir}/logfile_cloudflare.json + +run_cap_test ${lnav_test} -n \ + -c ':show-fields RayID' \ + ${test_dir}/logfile_cloudflare.json