[format] move foreign_key flag to value meta

Fix some chart issues as well.

NEWS.md

@@ -101,6 +101,7 @@ Bug Fixes:
 * A crash during initialization on Apple Silicon and MacOS 12
   has been fixed.
 * A crash when previewing non-text files.
+* Optimized ANSI-escape processing.
 * Various fixes to make lnav usable as a `PAGER`.
 
 ## lnav v0.12.1

src/db_sub_source.cc

@@ -286,6 +286,7 @@ db_label_source::push_column(const scoped_value_t& sv)
             }
         }
     }
+    hm.hm_chart.next_row();
 }
 
 void

src/files_sub_source.cc

@@ -359,6 +359,7 @@ files_sub_source::text_value_for_line(textview_curses& tc,
         .append(start_time)
         .append(" \u2014 ")
         .append(end_time)
+        .append(" ")
         .appendf(FMT_STRING("{}"), fmt::join(file_notes, "; "));
     if (selected) {
         al.with_attr_for_all(VC_ROLE.value(role_t::VCR_FOCUSED));

src/hist_source.cc

@@ -67,9 +67,13 @@ hist_source2::text_value_for_line(textview_curses& tc,
                                   std::string& value_out,
                                   text_sub_source::line_flags_t flags)
 {
-    bucket_t& bucket = this->find_bucket(row);
+    auto& bucket = this->find_bucket(row);
     struct tm bucket_tm;
 
+    if (this->hs_needs_flush) {
+        this->end_of_row();
+    }
+
     value_out.clear();
     if (gmtime_r(&bucket.b_time, &bucket_tm) != nullptr) {
         fmt::format_to(std::back_inserter(value_out),
@@ -130,12 +134,14 @@ hist_source2::add_value(time_t row,
     auto& bucket = this->find_bucket(this->hs_last_bucket);
     bucket.b_time = row;
     bucket.b_values[htype].hv_value += value;
+
+    this->hs_needs_flush = true;
 }
 
 void
 hist_source2::init()
 {
-    view_colors& vc = view_colors::singleton();
+    auto& vc = view_colors::singleton();
 
     this->hs_chart.with_show_state(stacked_bar_chart_base::show_all{})
         .with_attrs_for_ident(HT_NORMAL, vc.attrs_for_role(role_t::VCR_TEXT))
@@ -160,12 +166,13 @@ void
 hist_source2::end_of_row()
 {
     if (this->hs_last_bucket >= 0) {
-        bucket_t& last_bucket = this->find_bucket(this->hs_last_bucket);
+        auto& last_bucket = this->find_bucket(this->hs_last_bucket);
 
         for (int lpc = 0; lpc < HT__MAX; lpc++) {
             this->hs_chart.add_value((const hist_type_t) lpc,
                                      last_bucket.b_values[lpc].hv_value);
         }
+        this->hs_chart.next_row();
     }
 }
 

src/hist_source.hh

@@ -204,18 +204,23 @@ public:
             if (this->sbc_show_state.template is<show_all>()
                 || lpc == (size_t) ident_to_show)
             {
-                overall_stats.merge(this->sbc_idents[lpc].ci_stats,
+                overall_stats.merge(this->sbc_idents[lpc].ci_stats);
-                                    this->sbc_do_stacking);
             }
         }
+        if (this->sbc_max_row_value > overall_stats.bs_max_value) {
+            overall_stats.bs_max_value = this->sbc_max_row_value;
+        }
+        if (this->sbc_row_sum > overall_stats.bs_max_value) {
+            overall_stats.bs_max_value = this->sbc_row_sum;
+        }
 
         if (this->sbc_show_state.template is<show_all>()) {
             if (this->sbc_idents.size() == 1) {
                 avail_width = width;
-            } else if (width < this->sbc_idents.size()) {
+            } else if (width < this->sbc_max_row_items) {
                 avail_width = 0;
             } else {
-                avail_width = width - this->sbc_idents.size();
+                avail_width = width;
             }
         } else {
             avail_width = width - 1;
@@ -256,12 +261,34 @@ public:
         this->sbc_idents.clear();
         this->sbc_ident_lookup.clear();
         this->sbc_show_state = show_none();
+        this->sbc_row_sum = 0;
+        this->sbc_row_items = 0;
+        this->sbc_max_row_value = 0;
+        this->sbc_max_row_items = 0;
     }
 
     void add_value(const T& ident, double amount = 1.0)
     {
         struct chart_ident& ci = this->find_ident(ident);
         ci.ci_stats.update(amount);
+        this->sbc_row_sum += amount;
+        if (ci.ci_last_seen_row != this->sbc_row_counter) {
+            ci.ci_last_seen_row = this->sbc_row_counter;
+            this->sbc_row_items += 1;
+        }
+    }
+
+    void next_row()
+    {
+        if (this->sbc_row_sum > this->sbc_max_row_value) {
+            this->sbc_max_row_value = this->sbc_row_sum;
+        }
+        if (this->sbc_row_items > this->sbc_max_row_items) {
+            this->sbc_max_row_items = this->sbc_row_items;
+        }
+        this->sbc_row_sum = 0;
+        this->sbc_row_items = 0;
+        this->sbc_row_counter += 1;
     }
 
     struct bucket_stats_t {
@@ -270,15 +297,10 @@ public:
         {
         }
 
-        void merge(const bucket_stats_t& rhs, bool do_stacking)
+        void merge(const bucket_stats_t& rhs)
         {
             this->bs_min_value = std::min(this->bs_min_value, rhs.bs_min_value);
-            if (do_stacking) {
+            this->bs_max_value = std::max(this->bs_max_value, rhs.bs_max_value);
-                this->bs_max_value += rhs.bs_max_value;
-            } else {
-                this->bs_max_value
-                    = std::max(this->bs_max_value, rhs.bs_max_value);
-            }
         }
 
         double width() const
@@ -310,6 +332,7 @@ protected:
         T ci_ident;
         text_attrs ci_attrs;
         bucket_stats_t ci_stats;
+        ssize_t ci_last_seen_row{-1};
     };
 
     struct chart_ident& find_ident(const T& ident)
@@ -328,6 +351,12 @@ protected:
     std::vector<struct chart_ident> sbc_idents;
     std::unordered_map<T, unsigned int> sbc_ident_lookup;
     show_state sbc_show_state{show_none()};
+
+    ssize_t sbc_row_counter{0};
+    double sbc_row_sum{0};
+    size_t sbc_row_items{0};
+    double sbc_max_row_value{0};
+    size_t sbc_max_row_items{0};
 };
 
 class hist_source2
@@ -417,6 +446,7 @@ private:
     time_t hs_last_row;
     std::map<int64_t, struct bucket_block> hs_blocks;
     stacked_bar_chart<hist_type_t> hs_chart;
+    bool hs_needs_flush{false};
 };
 
 #endif

src/listview_curses.cc

@@ -267,6 +267,8 @@ listview_curses::handle_key(int ch)
                 if (this->lv_selection <= top_for_last) {
                     this->set_selection(top_for_last + 1_vl);
                 }
+            } else if (this->lv_top > top_for_last) {
+                this->set_top(top_for_last);
             } else {
                 this->shift_top(rows_avail);
 

src/log_format.cc

@@ -245,7 +245,7 @@ logline_value_meta::to_chart_type() const
             retval = chart_type_t::none;
             break;
         case value_kind_t::VALUE_INTEGER:
-            if (!this->lvm_identifier) {
+            if (!this->lvm_identifier && !this->lvm_foreign_key) {
                 retval = chart_type_t::spectro;
             }
             break;
@@ -2843,7 +2843,7 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
             auto& ivd = pat.p_value_by_index[lpc];
             auto vd = ivd.ivd_value_def;
 
-            if (!vd->vd_foreign_key && !vd->vd_meta.lvm_identifier) {
+            if (!vd->vd_meta.lvm_foreign_key && !vd->vd_meta.lvm_identifier) {
                 switch (vd->vd_meta.lvm_kind) {
                     case value_kind_t::VALUE_INTEGER:
                     case value_kind_t::VALUE_FLOAT:
@@ -3606,7 +3606,7 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
         elf_value_def->vd_meta.lvm_values_index
             = nonstd::make_optional(value_def_index++);
 
-        if (elf_value_def->vd_foreign_key
+        if (elf_value_def->vd_meta.lvm_foreign_key
             || elf_value_def->vd_meta.lvm_identifier)
         {
             continue;
@@ -3856,7 +3856,7 @@ public:
         log_vtab_impl::get_foreign_keys(keys_inout);
 
         for (const auto& elf_value_def : this->elt_format.elf_value_defs) {
-            if (elf_value_def.second->vd_foreign_key) {
+            if (elf_value_def.second->vd_meta.lvm_foreign_key) {
                 keys_inout.emplace_back(elf_value_def.first.to_string());
             }
         }

src/log_format.hh

@@ -163,6 +163,7 @@ struct logline_value_meta {
     column_t lvm_column{external_column{}};
     nonstd::optional<size_t> lvm_values_index;
     bool lvm_identifier{false};
+    bool lvm_foreign_key{false};
     bool lvm_hidden{false};
     nonstd::optional<bool> lvm_user_hidden;
     bool lvm_from_module{false};

src/log_format_ext.hh

@@ -68,7 +68,6 @@ public:
 
         logline_value_meta vd_meta;
         std::string vd_collate;
-        bool vd_foreign_key{false};
         intern_string_t vd_unit_field;
         std::map<const intern_string_t, scaling_factor> vd_unit_scaling;
         bool vd_internal{false};

src/log_format_impls.cc

@@ -489,10 +489,12 @@ public:
 
         field_def& with_kind(value_kind_t kind,
                              bool identifier = false,
+                             bool foreign_key = false,
                              const std::string& collator = "")
         {
             this->fd_meta.lvm_kind = kind;
             this->fd_meta.lvm_identifier = identifier;
+            this->fd_meta.lvm_foreign_key = foreign_key;
             this->fd_collator = collator;
             return *this;
         }
@@ -768,12 +770,14 @@ public:
                     "bro_referrer",
                     "bro_resp_fuids",
                     "bro_service",
-                    "bro_status_code",
                     "bro_uid",
                     "bro_uri",
                     "bro_user_agent",
                     "bro_username",
                 };
+                static const char* KNOWN_FOREIGN[] = {
+                    "bro_status_code",
+                };
 
                 int numeric_count = 0;
 
@@ -792,7 +796,12 @@ public:
                         bool ident = std::binary_search(std::begin(KNOWN_IDS),
                                                         std::end(KNOWN_IDS),
                                                         fd.fd_meta.lvm_name);
-                        fd.with_kind(value_kind_t::VALUE_INTEGER, ident)
+                        bool foreign
+                            = std::binary_search(std::begin(KNOWN_FOREIGN),
+                                                 std::end(KNOWN_FOREIGN),
+                                                 fd.fd_meta.lvm_name);
+                        fd.with_kind(
+                              value_kind_t::VALUE_INTEGER, ident, foreign)
                             .with_numeric_index(numeric_count);
                         numeric_count += 1;
                     } else if (field_type == "bool") {
@@ -950,7 +959,7 @@ public:
             this->log_vtab_impl::get_foreign_keys(keys_inout);
 
             for (const auto& fd : this->blt_format.blf_field_defs) {
-                if (fd.fd_meta.lvm_identifier) {
+                if (fd.fd_meta.lvm_identifier || fd.fd_meta.lvm_foreign_key) {
                     keys_inout.push_back(fd.fd_meta.lvm_name.to_string());
                 }
             }
@@ -1124,6 +1133,7 @@ public:
                   const char* name,
                   value_kind_t kind,
                   bool ident = false,
+                  bool foreign_key = false,
                   std::string coll = "")
             : fd_name(intern_string::lookup(name)),
               fd_meta(
@@ -1133,6 +1143,7 @@ public:
               fd_collator(std::move(coll))
         {
             this->fd_meta.lvm_identifier = ident;
+            this->fd_meta.lvm_foreign_key = foreign_key;
         }
 
         field_def& with_kind(value_kind_t kind,
@@ -1625,7 +1636,7 @@ public:
             this->log_vtab_impl::get_foreign_keys(keys_inout);
 
             for (const auto& fd : KNOWN_FIELDS) {
-                if (fd.fd_meta.lvm_identifier) {
+                if (fd.fd_meta.lvm_identifier || fd.fd_meta.lvm_foreign_key) {
                     keys_inout.push_back(fd.fd_meta.lvm_name.to_string());
                 }
             }
@@ -1687,6 +1698,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
         "c-ip",
         value_kind_t::VALUE_TEXT,
         true,
+        false,
         "ipaddress",
     },
     {
@@ -1706,6 +1718,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
         "cs-uri-stem",
         value_kind_t::VALUE_TEXT,
         true,
+        false,
         "naturalnocase",
     },
     {
@@ -1731,6 +1744,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
         "s-ip",
         value_kind_t::VALUE_TEXT,
         true,
+        false,
         "ipaddress",
     },
     {
@@ -1762,6 +1776,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
         "sc-status",
         value_kind_t::VALUE_INTEGER,
         false,
+        true,
     },
     {
         KNOWN_FIELD_INDEX++,

src/log_format_loader.cc

@@ -456,7 +456,8 @@ static const json_path_handler_base::enum_value_t OVERFLOW_ENUM[] = {
     {"truncate",
      external_log_format::json_format_element::overflow_t::TRUNCATE},
     {"dot-dot", external_log_format::json_format_element::overflow_t::DOTDOT},
-    {"last-word", external_log_format::json_format_element::overflow_t::LASTWORD},
+    {"last-word",
+     external_log_format::json_format_element::overflow_t::LASTWORD},
 
     json_path_handler_base::ENUM_TERMINATOR,
 };
@@ -616,7 +617,8 @@ static const struct json_path_container value_def_handlers = {
         .with_synopsis("<bool>")
         .with_description("Indicates whether or not this field should be "
                           "treated as a foreign key for row in another table")
-        .for_field(&external_log_format::value_def::vd_foreign_key),
+        .for_field(&external_log_format::value_def::vd_meta,
+                   &logline_value_meta::lvm_foreign_key),
 
     yajlpp::property_handler("hidden")
         .with_synopsis("<bool>")

src/prql/stats.prql

@@ -25,9 +25,24 @@ let hist = func column slice:'1h' top:10 rel <relation> -> (
   )
   group { tslice } (
     window (
-        sort {-total}
+      sort {-total}
-        take top
+      derive {rn = row_number column}
-        aggregate { v = json.group_object column total }
+    )
+  )
+  derive top_value = case [
+    rn < top => column,
+    rn >= top => 'Other',
+  ]
+  group { tslice, top_value } (
+    aggregate { total2 = sum total }
+  )
+  group { tslice } (
+    window (
+      sort {-total2}
+      # XXX The `take` here is necessary to workaround a
+      # PRQL issue where the above sort is dropped entirely
+      take top
+      aggregate { v = json.group_object top_value total2 }
     )
   )
 )

src/textview_curses.cc

@@ -595,7 +595,9 @@ textview_curses::handle_mouse(mouse_event& me)
                                            (int) this->lv_left + me.me_x);
 
                     this->set_selection_without_context(mc.mc_line);
-                    if (me.me_button == mouse_button_t::BUTTON_LEFT) {
+                    if (this->tc_supports_marks
+                        && me.me_button == mouse_button_t::BUTTON_LEFT)
+                    {
                         this->textview_value_for_row(mc.mc_line, al);
                         auto line_sf
                             = string_fragment::from_str(al.get_string());

test/expected/expected.am

@@ -474,10 +474,8 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_prql.sh_06900fac5c2e854b1208320b753fcd43d4ba63a3.out \
     $(srcdir)/%reldir%/test_prql.sh_15ebcd913f56cde35af26c6300533d6e76bf1b55.err \
     $(srcdir)/%reldir%/test_prql.sh_15ebcd913f56cde35af26c6300533d6e76bf1b55.out \
-    $(srcdir)/%reldir%/test_prql.sh_451e242cdfa2db9005d4fe752a7b05d1ab5cba29.err \
+    $(srcdir)/%reldir%/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.err \
-    $(srcdir)/%reldir%/test_prql.sh_451e242cdfa2db9005d4fe752a7b05d1ab5cba29.out \
+    $(srcdir)/%reldir%/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.out \
-    $(srcdir)/%reldir%/test_prql.sh_45d57a042092ffdcd28ea35a892f02859e78f33d.err \
-    $(srcdir)/%reldir%/test_prql.sh_45d57a042092ffdcd28ea35a892f02859e78f33d.out \
     $(srcdir)/%reldir%/test_regex101.sh_0fa3663a45aca6a328cb728872af7ed7ee896f1c.err \
     $(srcdir)/%reldir%/test_regex101.sh_0fa3663a45aca6a328cb728872af7ed7ee896f1c.out \
     $(srcdir)/%reldir%/test_regex101.sh_182ae9244db314a953af2bee969726e381bc5a32.err \

test/expected/test_cmds.sh_a00943ef715598c7554b85de8502454e41bb9e28.out

@@ -1,4 +1,4 @@
- Thu Nov 03 09:20:00          1 normal         2 errors         0 warnings         0 marks
+ Thu Nov 03 09:20:00          1 normal         2 errors         0 warnings         0 marks
- Thu Nov 03 09:45:00          1 normal         0 errors         0 warnings         0 marks
+ Thu Nov 03 09:45:00          1 normal         0 errors         0 warnings         0 marks
- Fri Feb 03 09:20:00          0 normal         1 errors         0 warnings         0 marks
+ Fri Feb 03 09:20:00          0 normal         1 errors         0 warnings         0 marks
- Wed Jan 03 09:20:00          1 normal         0 errors         0 warnings         0 marks
+ Wed Jan 03 09:20:00          1 normal         0 errors         0 warnings         0 marks

test/expected/test_cmds.sh_af0fcbd30b3fd0d13477aa3325ef0302052a4d9f.out

@@ -1 +1 @@
- Sat Nov 03 08:00:00          1 normal         0 errors         0 warnings         0 marks
+ Sat Nov 03 08:00:00          1 normal         0 errors         0 warnings         0 marks

test/expected/test_cmds.sh_c7fabc25374ff47c47931f63b1d697061b816a28.out

@@ -1,2 +1,2 @@
- Sat Nov 03 09:20:00          1 normal         2 errors         0 warnings         1 marks
+ Sat Nov 03 09:20:00          1 normal         2 errors         0 warnings         1 marks
- Sat Nov 03 09:45:00          1 normal         0 errors         0 warnings         0 marks
+ Sat Nov 03 09:45:00          1 normal         0 errors         0 warnings         0 marks

test/expected/test_cmds.sh_ccd326da92d1cacda63501cd1a3077381a18e8f2.out

@@ -1 +1 @@
- Sat Nov 03 00:00:00          2 normal         2 errors         0 warnings         0 marks
+ Sat Nov 03 00:00:00          2 normal         2 errors         0 warnings         0 marks

test/expected/test_cmds.sh_dbdd62995fdefc8318053af05a32416eccfa79fc.out

@@ -1 +1 @@
- Sat Nov 03 08:00:00          2 normal         2 errors         0 warnings         0 marks
+ Sat Nov 03 08:00:00          2 normal         2 errors         0 warnings         0 marks

test/expected/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.out

@@ -0,0 +1,4 @@
+        tslice                                                                     v                                                             
+2011-11-03 00:19:00.000 {"a.fsdn.com":33,"Other":31,"ad.doubleclick.net":10,"www.goo⋯"\xaf"dia.com":6,"s0.2mdn.net":5,"googleads.g.doubleclick.net":3} 
+2011-11-03 00:20:00.000 {"www.bro-ids.org":24,"www.google.com":15,"search.twitter.c⋯"\x8b""\xaf"ds.org":2,"ad.doubleclick.net":1,"cont-sjl-1.pandora.com":1} 
+2011-11-03 00:21:00.000 {"www.google.com":2,"ajax.googleapis.com":1,"api.twitter.com":1,"search.twitter.com":1,"www.bro-ids.org":1}              

test/expected/test_text_file.sh_7b00f32a3fff7fc2d78a87045ae842e58be88480.out

@@ -5,6 +5,6 @@
  * FEATURES:   network-sandbox preserve-libs sandbox test userpriv usersandbox
  * Checking whether python3_11 is suitable ...
  *   >=dev-lang/python-3.11.1-r1:3.11 ...
-                                                                                                                                                                                                                                       [ ok ]
+ [ ok ]
  * Using python3.11 to build (via PYTHON_COMPAT iteration)
 >>> Unpacking source...

test/test_prql.sh

@@ -11,3 +11,7 @@ run_cap_test ${lnav_test} -n \
 run_cap_test ${lnav_test} -n \
     -c ";from access_log | take abc" \
     ${test_dir}/logfile_access_log.0
+
+run_cap_test ${lnav_test} -n \
+    -c ";from bro_http_log | stats.hist bro_host slice:'1m'" \
+    ${test_dir}/logfile_bro_http.log.0