[log_format] fix hiding/showing JSON fields not in the line-format

Related to #1081

src/lnav.indexing.cc

@@ -290,8 +290,6 @@ rebuild_indexes(nonstd::optional<ui_clock::time_point> deadline)
             scroll_downs[LNV_LOG] = false;
         }
 
-        log_view.reload_data();
-
         {
             std::unordered_map<std::string, std::list<std::shared_ptr<logfile>>>
                 id_to_files;

src/lnav_commands.cc

@@ -3847,7 +3847,7 @@ com_toggle_field(exec_context& ec,
             // TODO: highlight the fields to be hidden.
             retval = "";
         } else {
-            logfile_sub_source& lss = lnav_data.ld_log_source;
+            auto& lss = lnav_data.ld_log_source;
             bool hide = args[0] == "hide-fields";
             std::vector<std::string> found_fields, missing_fields;
 
@@ -3870,8 +3870,8 @@ com_toggle_field(exec_context& ec,
                 } else if (tc->get_inner_height() == 0) {
                     return ec.make_error("no log messages to hide");
                 } else {
-                    content_line_t cl = lss.at(tc->get_selection());
-                    std::shared_ptr<logfile> lf = lss.find(cl);
+                    auto cl = lss.at(tc->get_selection());
+                    auto lf = lss.find(cl);
                     format = lf->get_format();
                     name = intern_string::lookup(args[lpc]);
                 }

src/log_format.cc

@@ -2926,6 +2926,7 @@ external_log_format::specialized(int fmt_lock)
 
     this->lf_value_stats.clear();
     this->lf_value_stats.resize(this->elf_value_defs.size());
+    this->elf_specialized_value_defs_state = *this->elf_value_defs_state;
 
     return retval;
 }
@@ -3250,6 +3251,72 @@ external_log_format::get_value_metadata() const
     return retval;
 }
 
+const logline_value_stats*
+external_log_format::stats_for_value(const intern_string_t& name) const
+{
+    auto iter = this->elf_value_defs.find(name);
+    if (iter != this->elf_value_defs.end()
+        && iter->second->vd_meta.lvm_values_index)
+    {
+        return &this->lf_value_stats[iter->second->vd_meta.lvm_values_index
+                                         .value()];
+    }
+
+    return nullptr;
+}
+
+std::string
+external_log_format::get_pattern_regex(uint64_t line_number) const
+{
+    if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
+        return "";
+    }
+    int pat_index = this->pattern_index_for_line(line_number);
+    return this->elf_pattern_order[pat_index]->p_pcre.pp_value->get_pattern();
+}
+
+bool
+external_log_format::hide_field(const intern_string_t field_name, bool val)
+{
+    auto vd_iter = this->elf_value_defs.find(field_name);
+
+    if (vd_iter == this->elf_value_defs.end()) {
+        return false;
+    }
+
+    vd_iter->second->vd_meta.lvm_user_hidden = val;
+    if (this->elf_type == elf_type_t::ELF_TYPE_JSON) {
+        bool found = false;
+
+        for (const auto& jfe : this->jlf_line_format) {
+            if (jfe.jfe_value.pp_value == field_name) {
+                found = true;
+            }
+        }
+        if (!found) {
+            log_info("format field %s.%s changed, rebuilding",
+                     this->elf_name.get(),
+                     field_name.get());
+            this->elf_value_defs_state->vds_generation += 1;
+        }
+    }
+    return true;
+}
+
+bool
+external_log_format::format_changed()
+{
+    if (this->elf_specialized_value_defs_state.vds_generation
+        != this->elf_value_defs_state->vds_generation)
+    {
+        this->elf_specialized_value_defs_state = *this->elf_value_defs_state;
+        this->jlf_cached_offset = -1;
+        return true;
+    }
+
+    return false;
+}
+
 bool
 format_tag_def::path_restriction::matches(const char* fn) const
 {

src/log_format.hh

@@ -118,7 +118,13 @@ struct logline_value_meta {
     {
     }
 
-    bool is_hidden() const { return this->lvm_hidden || this->lvm_user_hidden; }
+    bool is_hidden() const
+    {
+        if (this->lvm_user_hidden) {
+            return this->lvm_user_hidden.value();
+        }
+        return this->lvm_hidden;
+    }
 
     logline_value_meta& with_struct_name(intern_string_t name)
     {
@@ -132,7 +138,7 @@ struct logline_value_meta {
     nonstd::optional<size_t> lvm_values_index;
     bool lvm_identifier{false};
     bool lvm_hidden{false};
-    bool lvm_user_hidden{false};
+    nonstd::optional<bool> lvm_user_hidden;
     bool lvm_from_module{false};
     intern_string_t lvm_struct_name;
     nonstd::optional<log_format*> lvm_format;
@@ -479,6 +485,8 @@ public:
         return {};
     }
 
+    virtual bool format_changed() { return false; }
+
     struct pattern_for_lines {
         pattern_for_lines(uint32_t pfl_line, uint32_t pfl_pat_index);
 

src/log_format_ext.hh

@@ -137,29 +137,30 @@ public:
         this->jlf_line_offsets.reserve(128);
     }
 
-    const intern_string_t get_name() const { return this->elf_name; }
+    const intern_string_t get_name() const override { return this->elf_name; }
 
-    bool match_name(const std::string& filename);
+    bool match_name(const std::string& filename) override;
 
-    bool match_mime_type(const file_format_t ff) const;
+    bool match_mime_type(const file_format_t ff) const override;
 
     scan_result_t scan(logfile& lf,
                        std::vector<logline>& dst,
                        const line_info& offset,
                        shared_buffer_ref& sbr,
-                       scan_batch_context& sbc);
+                       scan_batch_context& sbc) override;
 
-    bool scan_for_partial(shared_buffer_ref& sbr, size_t& len_out) const;
+    bool scan_for_partial(shared_buffer_ref& sbr,
+                          size_t& len_out) const override;
 
     void annotate(uint64_t line_number,
                   string_attrs_t& sa,
                   logline_value_vector& values,
-                  bool annotate_module = true) const;
+                  bool annotate_module = true) const override;
 
     void rewrite(exec_context& ec,
                  shared_buffer_ref& line,
                  string_attrs_t& sa,
-                 std::string& value_out);
+                 std::string& value_out) override;
 
     void build(std::vector<lnav::console::user_message>& errors);
 
@@ -168,41 +169,21 @@ public:
 
     bool match_samples(const std::vector<sample>& samples) const;
 
-    bool hide_field(const intern_string_t field_name, bool val)
-    {
-        auto vd_iter = this->elf_value_defs.find(field_name);
-
-        if (vd_iter == this->elf_value_defs.end()) {
-            return false;
-        }
-
-        vd_iter->second->vd_meta.lvm_user_hidden = val;
-        return true;
-    }
+    bool hide_field(const intern_string_t field_name, bool val) override;
 
-    std::shared_ptr<log_format> specialized(int fmt_lock);
+    std::shared_ptr<log_format> specialized(int fmt_lock) override;
 
     const logline_value_stats* stats_for_value(
-        const intern_string_t& name) const
-    {
-        auto iter = this->elf_value_defs.find(name);
-        if (iter != this->elf_value_defs.end()
-            && iter->second->vd_meta.lvm_values_index)
-        {
-            return &this->lf_value_stats[iter->second->vd_meta.lvm_values_index
-                                             .value()];
-        }
-
-        return nullptr;
-    }
+        const intern_string_t& name) const override;
 
     void get_subline(const logline& ll,
                      shared_buffer_ref& sbr,
-                     bool full_message);
+                     bool full_message) override;
 
-    std::shared_ptr<log_vtab_impl> get_vtab_impl() const;
+    std::shared_ptr<log_vtab_impl> get_vtab_impl() const override;
 
-    const std::vector<std::string>* get_actions(const logline_value& lv) const
+    const std::vector<std::string>* get_actions(
+        const logline_value& lv) const override
     {
         const std::vector<std::string>* retval = nullptr;
 
@@ -214,12 +195,14 @@ public:
         return retval;
     }
 
-    std::set<std::string> get_source_path() const
+    bool format_changed() override;
+
+    std::set<std::string> get_source_path() const override
     {
         return this->elf_source_path;
     }
 
-    std::vector<logline_value_meta> get_value_metadata() const;
+    std::vector<logline_value_meta> get_value_metadata() const override;
 
     enum class json_log_field {
         CONSTANT,
@@ -305,7 +288,7 @@ public:
         return iter != this->elf_value_defs.end();
     }
 
-    std::string get_pattern_path(uint64_t line_number) const
+    std::string get_pattern_path(uint64_t line_number) const override
     {
         if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
             return "structured";
@@ -314,17 +297,9 @@ public:
         return this->elf_pattern_order[pat_index]->p_config_path;
     }
 
-    intern_string_t get_pattern_name(uint64_t line_number) const;
+    intern_string_t get_pattern_name(uint64_t line_number) const override;
 
-    std::string get_pattern_regex(uint64_t line_number) const
-    {
-        if (this->elf_type != elf_type_t::ELF_TYPE_TEXT) {
-            return "";
-        }
-        int pat_index = this->pattern_index_for_line(line_number);
-        return this->elf_pattern_order[pat_index]
-            ->p_pcre.pp_value->get_pattern();
-    }
+    std::string get_pattern_regex(uint64_t line_number) const override;
 
     log_level_t convert_level(string_fragment str,
                               scan_batch_context* sbc) const;
@@ -345,6 +320,15 @@ public:
     std::vector<sample> elf_samples;
     std::unordered_map<const intern_string_t, std::shared_ptr<value_def>>
         elf_value_defs;
+
+    struct value_defs_state {
+        size_t vds_generation{0};
+    };
+
+    std::shared_ptr<value_defs_state> elf_value_defs_state{
+        std::make_shared<value_defs_state>()};
+    value_defs_state elf_specialized_value_defs_state;
+
     std::vector<std::shared_ptr<value_def>> elf_value_def_order;
     std::vector<std::shared_ptr<value_def>> elf_numeric_value_defs;
     int elf_column_count{0};

src/logfile.cc

@@ -455,6 +455,16 @@ logfile::rebuild_index(nonstd::optional<ui_clock::time_point> deadline)
         return rebuild_result_t::NO_NEW_LINES;
     }
 
+    if (this->lf_format != nullptr && this->lf_format->format_changed()) {
+        log_info("%s: format has changed, rebuilding",
+                 this->lf_filename.c_str());
+        this->lf_index.clear();
+        this->lf_index_size = 0;
+        this->lf_partial_line = false;
+        this->lf_longest_line = 0;
+        this->lf_sort_needed = true;
+    }
+
     auto retval = rebuild_result_t::NO_NEW_LINES;
     struct stat st;
 
@@ -555,8 +565,7 @@ logfile::rebuild_index(nonstd::optional<ui_clock::time_point> deadline)
             this->lf_logline_observer->logline_restart(*this, rollback_size);
         }
 
-        bool sort_needed = this->lf_sort_needed;
-        this->lf_sort_needed = false;
+        bool sort_needed = std::exchange(this->lf_sort_needed, false);
         size_t limit = SIZE_MAX;
 
         if (deadline) {

src/logfile_sub_source.cc

@@ -1039,14 +1039,17 @@ logfile_sub_source::rebuild_index(
         case rebuild_result::rr_full_rebuild:
             log_debug("redoing search");
             this->lss_index_generation += 1;
+            this->tss_view->reload_data();
             this->tss_view->redo_search();
             break;
         case rebuild_result::rr_partial_rebuild:
             log_debug("redoing search from: %d", (int) search_start);
             this->lss_index_generation += 1;
+            this->tss_view->reload_data();
             this->tss_view->search_new_data(search_start);
             break;
         case rebuild_result::rr_appended_lines:
+            this->tss_view->reload_data();
             this->tss_view->search_new_data();
             break;
     }

src/session_data.cc

@@ -1526,9 +1526,16 @@ save_session_with_id(const std::string& session_id)
                                     continue;
                                 }
 
-                                cmd_array.gen("hide-fields "
-                                              + elf->get_name().to_string()
-                                              + "." + vd.first.to_string());
+                                if (vd.second->vd_meta.lvm_user_hidden.value())
+                                {
+                                    cmd_array.gen("hide-fields "
+                                                  + elf->get_name().to_string()
+                                                  + "." + vd.first.to_string());
+                                } else if (vd.second->vd_meta.lvm_hidden) {
+                                    cmd_array.gen("show-fields "
+                                                  + elf->get_name().to_string()
+                                                  + "." + vd.first.to_string());
+                                }
                             }
                         }
 
@@ -1654,8 +1661,15 @@ reset_session()
             continue;
         }
 
+        bool changed = false;
         for (const auto& vd : elf->elf_value_defs) {
-            vd.second->vd_meta.lvm_user_hidden = false;
+            if (vd.second->vd_meta.lvm_user_hidden) {
+                vd.second->vd_meta.lvm_user_hidden = nonstd::nullopt;
+                changed = true;
+            }
+        }
+        if (changed) {
+            elf->elf_value_defs_state->vds_generation += 1;
         }
     }
 }

src/textview_curses.cc

@@ -825,7 +825,7 @@ void
 text_time_translator::scroll_invoked(textview_curses* tc)
 {
     if (tc->get_inner_height() > 0) {
-        this->time_for_row(tc->get_top()) |
+        this->time_for_row(tc->get_selection()) |
             [this](auto new_top_time) { this->ttt_top_time = new_top_time; };
     }
 }
@@ -836,22 +836,23 @@ text_time_translator::data_reloaded(textview_curses* tc)
     if (tc->get_inner_height() == 0) {
         return;
     }
-    if (tc->get_top() > tc->get_inner_height()) {
+    if (tc->get_selection() > tc->get_inner_height()) {
         if (this->ttt_top_time.tv_sec != 0) {
             this->row_for_time(this->ttt_top_time) |
-                [tc](auto new_top) { tc->set_top(new_top); };
+                [tc](auto new_top) { tc->set_selection(new_top); };
         }
         return;
     }
-    this->time_for_row(tc->get_top()) | [this, tc](auto top_time) {
+    this->time_for_row(tc->get_selection()) | [this, tc](auto top_time) {
         if (top_time != this->ttt_top_time) {
             if (this->ttt_top_time.tv_sec != 0) {
                 this->row_for_time(this->ttt_top_time) |
-                    [tc](auto new_top) { tc->set_top(new_top); };
+                    [tc](auto new_top) { tc->set_selection(new_top); };
             }
-            this->time_for_row(tc->get_top()) | [this](auto new_top_time) {
-                this->ttt_top_time = new_top_time;
-            };
+            this->time_for_row(tc->get_selection()) |
+                [this](auto new_top_time) {
+                    this->ttt_top_time = new_top_time;
+                };
         }
     };
 }

test/expected/expected.am

@@ -296,6 +296,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_json_format.sh_d7362cffc8335c2fe6b6527315de59bd6f5dcc7f.out \
     $(srcdir)/%reldir%/test_json_format.sh_dfff27a651650a04d93de9a06ab5480e94ce3a79.err \
     $(srcdir)/%reldir%/test_json_format.sh_dfff27a651650a04d93de9a06ab5480e94ce3a79.out \
+    $(srcdir)/%reldir%/test_json_format.sh_e36401aa54bc61de71f8dcbe66ea16effa59ea52.err \
+    $(srcdir)/%reldir%/test_json_format.sh_e36401aa54bc61de71f8dcbe66ea16effa59ea52.out \
     $(srcdir)/%reldir%/test_json_format.sh_f740026626ab554dacb249762d8be7d6539b8c6e.err \
     $(srcdir)/%reldir%/test_json_format.sh_f740026626ab554dacb249762d8be7d6539b8c6e.out \
     $(srcdir)/%reldir%/test_json_format.sh_fe19b7ebd349cd689b3f5c22618eab5ce995e68e.err \

test/test_json_format.sh

@@ -150,3 +150,7 @@ run_cap_test ${lnav_test} -n \
 
 run_cap_test ${lnav_test} -n \
     ${test_dir}/logfile_cloudflare.json
+
+run_cap_test ${lnav_test} -n \
+    -c ':show-fields RayID' \
+    ${test_dir}/logfile_cloudflare.json