[gantt] reduce memory usage

src/base/map_util.hh

@@ -68,6 +68,58 @@ from_vec(const std::vector<std::pair<K, V>>& container)
     return retval;
 }
 
+template<typename K, typename V>
+class small : public std::vector<std::pair<K, V>> {
+public:
+    auto insert(const K& key, const V& value)
+    {
+        auto pos = this->begin();
+
+        while (pos != this->end() && pos->first < key) {
+            ++pos;
+        }
+        return this->emplace(pos, std::make_pair(key, value));
+    }
+
+    auto find(const K& key)
+    {
+        auto retval = this->begin();
+
+        while (retval != this->end()
+               && (retval->first < key || key < retval->first))
+        {
+            ++retval;
+        }
+
+        return retval;
+    }
+
+    auto find(const K& key) const
+    {
+        auto retval = this->begin();
+
+        while (retval != this->end()
+               && (retval->first < key || key < retval->first))
+        {
+            ++retval;
+        }
+
+        return retval;
+    }
+
+    V& operator[](const K& key)
+    {
+        auto iter = this->find(key);
+        if (iter != this->end()) {
+            return iter->second;
+        }
+
+        this->emplace_back(key, V{});
+
+        return this->back().second;
+    }
+};
+
 }  // namespace map
 }  // namespace lnav
 

src/file_collection.cc

@@ -197,6 +197,9 @@ file_collection::regenerate_unique_file_names()
 void
 file_collection::merge(file_collection& other)
 {
+    bool do_regen = !other.fc_files.empty() || !other.fc_other_files.empty()
+        || !other.fc_name_to_errors->readAccess()->empty();
+
     this->fc_recursive = this->fc_recursive || other.fc_recursive;
     this->fc_rotated = this->fc_rotated || other.fc_rotated;
 
@@ -205,9 +208,9 @@ file_collection::merge(file_collection& other)
 
     std::map<std::string, file_error_info> new_errors;
     {
-        safe::WriteAccess<safe_name_to_errors> errs(*other.fc_name_to_errors);
+        safe::ReadAccess<safe_name_to_errors> errs(*other.fc_name_to_errors);
 
-        new_errors = std::move(*errs);
+        new_errors.insert(errs->cbegin(), errs->cend());
     }
     {
         safe::WriteAccess<safe_name_to_errors> errs(*this->fc_name_to_errors);
@@ -238,6 +241,10 @@ file_collection::merge(file_collection& other)
             std::make_move_iterator(other.fc_child_pollers.end()));
         other.fc_child_pollers.clear();
     }
+
+    if (do_regen) {
+        this->regenerate_unique_file_names();
+    }
 }
 
 /**
@@ -663,7 +670,7 @@ file_collection::expand_filename(
 
                         if (gl->gl_pathc == 1) {
                             log_error("failed to find path: %s -- %s",
-                                      path.c_str(),
+                                      filename_key.c_str(),
                                       errmsg);
                             retval.fc_name_to_errors->writeAccess()->emplace(
                                 filename_key,

src/files_sub_source.cc

@@ -222,9 +222,10 @@ size_t
 files_sub_source::text_line_count()
 {
     const auto& fc = lnav_data.ld_active_files;
+    auto retval = fc.fc_name_to_errors->readAccess()->size()
+        + fc.fc_other_files.size() + fc.fc_files.size();
 
-    return fc.fc_name_to_errors->readAccess()->size() + fc.fc_other_files.size()
-        + fc.fc_files.size();
+    return retval;
 }
 
 size_t

src/gantt_source.cc

@@ -35,9 +35,11 @@
 
 #include "base/humanize.hh"
 #include "base/humanize.time.hh"
+#include "base/itertools.hh"
 #include "base/math_util.hh"
 #include "command_executor.hh"
 #include "intervaltree/IntervalTree.h"
+#include "lnav_util.hh"
 #include "md4cpp.hh"
 #include "sql_util.hh"
 
@@ -516,7 +518,7 @@ gantt_source::rebuild_indexes()
     this->gs_total_width = 0;
     this->gs_filtered_count = 0;
     this->gs_active_opids.clear();
-    this->gs_opid_map.clear();
+    this->gs_descriptions.clear();
     this->gs_subid_map.clear();
     this->gs_allocator.reset();
     this->gs_preview_source.clear();
@@ -524,7 +526,6 @@ gantt_source::rebuild_indexes()
 
     auto min_log_time_opt = this->gs_lss.get_min_log_time();
     auto max_log_time_opt = this->gs_lss.get_max_log_time();
-
     auto max_desc_width = size_t{0};
 
     for (const auto& ld : this->gs_lss) {
@@ -538,26 +539,24 @@ gantt_source::rebuild_indexes()
         auto format = ld->get_file_ptr()->get_format();
         safe::ReadAccess<logfile::safe_opid_state> r_opid_map(
             ld->get_file_ptr()->get_opids());
-
         for (const auto& pair : r_opid_map->los_opid_ranges) {
             auto& otr = pair.second;
-            auto iter = this->gs_opid_map.find(pair.first);
-            if (iter == this->gs_opid_map.end()) {
-                auto opid = pair.first.to_owned(this->gs_allocator);
-                auto emp_res
-                    = this->gs_opid_map.emplace(opid, opid_description_defs{});
-                iter = emp_res.first;
-            }
-
             auto active_iter = this->gs_active_opids.find(pair.first);
             if (active_iter == this->gs_active_opids.end()) {
+                auto opid = pair.first.to_owned(this->gs_allocator);
                 auto active_emp_res = this->gs_active_opids.emplace(
-                    iter->first, opid_row{iter->first, otr});
+                    opid,
+                    opid_row{
+                        opid,
+                        otr,
+                        string_fragment::invalid(),
+                    });
                 active_iter = active_emp_res.first;
             } else {
                 active_iter->second.or_value |= otr;
             }
 
+            auto& row = active_iter->second;
             for (auto& sub : active_iter->second.or_value.otr_sub_ops) {
                 auto subid_iter = this->gs_subid_map.find(sub.ostr_subid);
 
@@ -584,14 +583,21 @@ gantt_source::rebuild_indexes()
 
                 if (desc_def_iter == format->lf_opid_description_def->end()) {
                     log_error("cannot find description: %s",
-                              iter->first.data());
+                              active_iter->first.data());
                 } else {
-                    auto& format_descs
-                        = iter->second.odd_format_to_desc[format->get_name()];
-                    format_descs[desc_id] = desc_def_iter->second;
+                    auto desc_key
+                        = opid_description_def_key{format->get_name(), desc_id};
+                    auto desc_defs_iter
+                        = row.or_description_defs.odd_defs.find(desc_key);
+                    if (desc_defs_iter
+                        == row.or_description_defs.odd_defs.end())
+                    {
+                        row.or_description_defs.odd_defs.insert(
+                            desc_key, desc_def_iter->second);
+                    }
 
                     auto& all_descs = active_iter->second.or_descriptions;
-                    auto& curr_desc_m = all_descs[format->get_name()][desc_id];
+                    auto& curr_desc_m = all_descs[desc_key];
                     const auto& new_desc_v = otr.otr_description.lod_elements;
 
                     for (const auto& desc_pair : new_desc_v) {
@@ -601,6 +607,7 @@ gantt_source::rebuild_indexes()
             } else {
                 ensure(otr.otr_description.lod_elements.empty());
             }
+            active_iter->second.or_value.otr_description.lod_elements.clear();
         }
     }
 
@@ -618,19 +625,24 @@ gantt_source::rebuild_indexes()
     this->gs_time_order.reserve(this->gs_active_opids.size());
     for (auto& pair : this->gs_active_opids) {
         auto& otr = pair.second.or_value;
-        auto& full_desc = pair.second.or_description;
+        std::string full_desc;
+        const auto& desc_defs = pair.second.or_description_defs.odd_defs;
         for (auto& desc : pair.second.or_descriptions) {
-            const auto& format_desc_defs
-                = this->gs_opid_map[pair.second.or_name]
-                      .odd_format_to_desc[desc.first];
-
-            require(!format_desc_defs.empty());
-            for (auto& desc_format_pairs : desc.second) {
-                const auto& desc_def
-                    = format_desc_defs.find(desc_format_pairs.first)->second;
-                full_desc = desc_def.to_string(desc_format_pairs.second);
+            auto desc_def_iter = desc_defs.find(desc.first);
+            if (desc_def_iter == desc_defs.end()) {
+                continue;
             }
+            const auto& desc_def = desc_def_iter->second;
+            full_desc = desc_def.to_string(desc.second);
+        }
+        pair.second.or_descriptions.clear();
+        auto full_desc_sf = string_fragment::from_str(full_desc);
+        auto desc_sf_iter = this->gs_descriptions.find(full_desc_sf);
+        if (desc_sf_iter == this->gs_descriptions.end()) {
+            full_desc_sf = string_fragment::from_str(full_desc).to_owned(
+                this->gs_allocator);
         }
+        pair.second.or_description = full_desc_sf;
 
         shared_buffer sb_opid;
         shared_buffer_ref sbr_opid;

src/gantt_source.hh

@@ -30,6 +30,7 @@
 #ifndef lnav_gantt_source_hh
 #define lnav_gantt_source_hh
 
+#include "base/map_util.hh"
 #include "gantt_status_source.hh"
 #include "logfile_sub_source.hh"
 #include "plain_text_source.hh"
@@ -84,17 +85,27 @@ public:
     gantt_status_source& gs_preview_status_source;
     ArenaAlloc::Alloc<char> gs_allocator{64 * 1024};
 
-    struct opid_description_defs {
-        std::map<intern_string_t,
-                 std::map<intern_string_t, log_format::opid_descriptors>>
-            odd_format_to_desc;
+    struct opid_description_def_key {
+        intern_string_t oddk_format_name;
+        intern_string_t oddk_desc_name;
+
+        bool operator<(const opid_description_def_key& rhs) const
+        {
+            if (this->oddk_format_name < rhs.oddk_format_name) {
+                return true;
+            }
+            if (this->oddk_format_name == rhs.oddk_format_name) {
+                return this->oddk_desc_name < rhs.oddk_desc_name;
+            }
+
+            return false;
+        }
     };
 
-    using gantt_opid_map
-        = robin_hood::unordered_map<string_fragment,
-                                    opid_description_defs,
-                                    frag_hasher,
-                                    std::equal_to<string_fragment>>;
+    struct opid_description_defs {
+        lnav::map::small<opid_description_def_key, log_format::opid_descriptors>
+            odd_defs;
+    };
 
     using gantt_subid_map
         = robin_hood::unordered_map<string_fragment,
@@ -102,17 +113,17 @@ public:
                                     frag_hasher,
                                     std::equal_to<string_fragment>>;
 
-    gantt_opid_map gs_opid_map;
     gantt_subid_map gs_subid_map;
 
     struct opid_row {
         string_fragment or_name;
         opid_time_range or_value;
-        std::map<intern_string_t,
-                 std::map<intern_string_t, std::map<size_t, std::string>>>
+        string_fragment or_description;
+        opid_description_defs or_description_defs;
+        lnav::map::small<opid_description_def_key,
+                         lnav::map::small<size_t, std::string>>
             or_descriptions;
         size_t or_max_subid_width{0};
-        std::string or_description;
 
         bool operator<(const opid_row& rhs) const
         {
@@ -135,11 +146,16 @@ public:
                                     opid_row,
                                     frag_hasher,
                                     std::equal_to<string_fragment>>;
+    using gantt_desc_map
+        = robin_hood::unordered_set<string_fragment,
+                                    frag_hasher,
+                                    std::equal_to<string_fragment>>;
 
     attr_line_t gs_rendered_line;
     size_t gs_opid_width{0};
     size_t gs_total_width{0};
     gantt_opid_row_map gs_active_opids;
+    gantt_desc_map gs_descriptions;
     std::vector<std::reference_wrapper<opid_row>> gs_time_order;
     struct timeval gs_lower_bound {};
     struct timeval gs_upper_bound {};

src/listview_curses.cc

@@ -97,6 +97,7 @@ listview_curses::reload_data()
 {
     if (this->lv_source == nullptr) {
         this->lv_top = 0_vl;
+        this->lv_selection = -1_vl;
         this->lv_focused_overlay_top = 0_vl;
         this->lv_focused_overlay_selection = 0_vl;
         this->lv_left = 0;
@@ -109,7 +110,7 @@ listview_curses::reload_data()
         }
         if (this->lv_selectable) {
             if (this->get_inner_height() == 0) {
-                this->set_selection(0_vl);
+                this->set_selection(-1_vl);
             } else if (this->lv_selection >= this->get_inner_height()) {
                 this->set_selection(this->get_inner_height() - 1_vl);
             } else {

src/listview_curses.hh

@@ -255,9 +255,9 @@ public:
 
     void set_sync_selection_and_top(bool value)
     {
-        this->lv_sync_selection_and_top = value;
-        if (value) {
-            this->set_top(this->get_selection());
+        if (this->lv_sync_selection_and_top != value) {
+            this->lv_sync_selection_and_top = value;
+            this->set_needs_update();
         }
     }
 

src/lnav.indexing.cc

@@ -408,11 +408,6 @@ update_active_files(file_collection& new_files)
         }
     }
     lnav_data.ld_active_files.merge(new_files);
-    if (!new_files.fc_files.empty() || !new_files.fc_other_files.empty()
-        || !new_files.fc_name_to_errors->readAccess()->empty())
-    {
-        lnav_data.ld_active_files.regenerate_unique_file_names();
-    }
     lnav_data.ld_child_pollers.insert(
         lnav_data.ld_child_pollers.begin(),
         std::make_move_iterator(

src/lnav_commands.cc

@@ -4870,6 +4870,7 @@ com_spectrogram(exec_context& ec,
         }
 
         if (found) {
+            lnav_data.ld_views[LNV_SPECTRO].reload_data();
             ss.text_selection_changed(lnav_data.ld_views[LNV_SPECTRO]);
             ensure_view(&lnav_data.ld_views[LNV_SPECTRO]);
 

src/log_format.cc

@@ -220,14 +220,16 @@ log_format::opid_descriptor::matches(const string_fragment& sf) const
 
 std::string
 log_format::opid_descriptors::to_string(
-    const std::map<size_t, std::string>& lod) const
+    const lnav::map::small<size_t, std::string>& lod) const
 {
     std::string retval;
 
     for (size_t lpc = 0; lpc < this->od_descriptors->size(); lpc++) {
         retval.append(this->od_descriptors->at(lpc).od_prefix);
-        lnav::map::find(lod, lpc) |
-            [&retval](const auto str) { retval.append(str); };
+        auto iter = lod.find(lpc);
+        if (iter != lod.end()) {
+            retval.append(iter->second);
+        }
         retval.append(this->od_descriptors->at(lpc).od_suffix);
     }
 
@@ -1159,8 +1161,8 @@ external_log_format::scan(logfile& lf,
                             auto& sub_desc_def
                                 = this->lf_subid_description_def->at(
                                     sub_desc.lod_id.value());
-                            ostr->ostr_description = sub_desc_def.to_string(
-                                lnav::map::from_vec(sub_desc.lod_elements));
+                            ostr->ostr_description
+                                = sub_desc_def.to_string(sub_desc.lod_elements);
                         }
                     }
                 }
@@ -1360,8 +1362,8 @@ external_log_format::scan(logfile& lf,
                     if (!sub_desc.lod_elements.empty()) {
                         auto& sub_desc_def = this->lf_subid_description_def->at(
                             sub_desc.lod_id.value());
-                        ostr->ostr_description = sub_desc_def.to_string(
-                            lnav::map::from_vec(sub_desc.lod_elements));
+                        ostr->ostr_description
+                            = sub_desc_def.to_string(sub_desc.lod_elements);
                     }
                 }
             }

src/log_format.hh

@@ -574,7 +574,8 @@ public:
     struct opid_descriptors {
         std::shared_ptr<std::vector<opid_descriptor>> od_descriptors;
 
-        std::string to_string(const std::map<size_t, std::string>& lod) const;
+        std::string to_string(
+            const lnav::map::small<size_t, std::string>& lod) const;
     };
 
     std::shared_ptr<std::map<intern_string_t, opid_descriptors>>

src/log_format_fwd.hh

@@ -38,6 +38,7 @@
 
 #include "ArenaAlloc/arenaalloc.h"
 #include "base/file_range.hh"
+#include "base/map_util.hh"
 #include "base/string_attr_type.hh"
 #include "byte_array.hh"
 #include "log_level.hh"
@@ -58,7 +59,7 @@ struct log_level_stats {
 
 struct log_op_description {
     nonstd::optional<intern_string_t> lod_id;
-    std::vector<std::pair<size_t, std::string>> lod_elements;
+    lnav::map::small<size_t, std::string> lod_elements;
 
     log_op_description& operator|=(const log_op_description& rhs);
 };

src/spectro_source.cc

@@ -532,6 +532,9 @@ spectrogram_source::text_selection_changed(textview_curses& tc)
         return;
     }
 
+    if (tc.get_selection() == -1_vl) {
+        tc.set_selection(0_vl);
+    }
     const auto& s_row = this->load_row(tc, tc.get_selection());
     this->ss_cursor_column
         = s_row.nearest_column(this->ss_cursor_column.value_or(0));