diff --git a/NEWS.md b/NEWS.md index 911f2aa6..4660ea54 100644 --- a/NEWS.md +++ b/NEWS.md @@ -101,6 +101,7 @@ Bug Fixes: * A crash during initialization on Apple Silicon and MacOS 12 has been fixed. * A crash when previewing non-text files. +* Optimized ANSI-escape processing. * Various fixes to make lnav usable as a `PAGER`. ## lnav v0.12.1 diff --git a/src/db_sub_source.cc b/src/db_sub_source.cc index d7220ad9..94538766 100644 --- a/src/db_sub_source.cc +++ b/src/db_sub_source.cc @@ -286,6 +286,7 @@ db_label_source::push_column(const scoped_value_t& sv) } } } + hm.hm_chart.next_row(); } void diff --git a/src/files_sub_source.cc b/src/files_sub_source.cc index 9815cd26..4f2fff24 100644 --- a/src/files_sub_source.cc +++ b/src/files_sub_source.cc @@ -359,6 +359,7 @@ files_sub_source::text_value_for_line(textview_curses& tc, .append(start_time) .append(" \u2014 ") .append(end_time) + .append(" ") .appendf(FMT_STRING("{}"), fmt::join(file_notes, "; ")); if (selected) { al.with_attr_for_all(VC_ROLE.value(role_t::VCR_FOCUSED)); diff --git a/src/hist_source.cc b/src/hist_source.cc index 422c1d2d..98819162 100644 --- a/src/hist_source.cc +++ b/src/hist_source.cc @@ -67,9 +67,13 @@ hist_source2::text_value_for_line(textview_curses& tc, std::string& value_out, text_sub_source::line_flags_t flags) { - bucket_t& bucket = this->find_bucket(row); + auto& bucket = this->find_bucket(row); struct tm bucket_tm; + if (this->hs_needs_flush) { + this->end_of_row(); + } + value_out.clear(); if (gmtime_r(&bucket.b_time, &bucket_tm) != nullptr) { fmt::format_to(std::back_inserter(value_out), @@ -130,12 +134,14 @@ hist_source2::add_value(time_t row, auto& bucket = this->find_bucket(this->hs_last_bucket); bucket.b_time = row; bucket.b_values[htype].hv_value += value; + + this->hs_needs_flush = true; } void hist_source2::init() { - view_colors& vc = view_colors::singleton(); + auto& vc = view_colors::singleton(); this->hs_chart.with_show_state(stacked_bar_chart_base::show_all{}) .with_attrs_for_ident(HT_NORMAL, vc.attrs_for_role(role_t::VCR_TEXT)) @@ -160,12 +166,13 @@ void hist_source2::end_of_row() { if (this->hs_last_bucket >= 0) { - bucket_t& last_bucket = this->find_bucket(this->hs_last_bucket); + auto& last_bucket = this->find_bucket(this->hs_last_bucket); for (int lpc = 0; lpc < HT__MAX; lpc++) { this->hs_chart.add_value((const hist_type_t) lpc, last_bucket.b_values[lpc].hv_value); } + this->hs_chart.next_row(); } } diff --git a/src/hist_source.hh b/src/hist_source.hh index f3704ab3..81245af2 100644 --- a/src/hist_source.hh +++ b/src/hist_source.hh @@ -204,18 +204,23 @@ public: if (this->sbc_show_state.template is() || lpc == (size_t) ident_to_show) { - overall_stats.merge(this->sbc_idents[lpc].ci_stats, - this->sbc_do_stacking); + overall_stats.merge(this->sbc_idents[lpc].ci_stats); } } + if (this->sbc_max_row_value > overall_stats.bs_max_value) { + overall_stats.bs_max_value = this->sbc_max_row_value; + } + if (this->sbc_row_sum > overall_stats.bs_max_value) { + overall_stats.bs_max_value = this->sbc_row_sum; + } if (this->sbc_show_state.template is()) { if (this->sbc_idents.size() == 1) { avail_width = width; - } else if (width < this->sbc_idents.size()) { + } else if (width < this->sbc_max_row_items) { avail_width = 0; } else { - avail_width = width - this->sbc_idents.size(); + avail_width = width; } } else { avail_width = width - 1; @@ -256,12 +261,34 @@ public: this->sbc_idents.clear(); this->sbc_ident_lookup.clear(); this->sbc_show_state = show_none(); + this->sbc_row_sum = 0; + this->sbc_row_items = 0; + this->sbc_max_row_value = 0; + this->sbc_max_row_items = 0; } void add_value(const T& ident, double amount = 1.0) { struct chart_ident& ci = this->find_ident(ident); ci.ci_stats.update(amount); + this->sbc_row_sum += amount; + if (ci.ci_last_seen_row != this->sbc_row_counter) { + ci.ci_last_seen_row = this->sbc_row_counter; + this->sbc_row_items += 1; + } + } + + void next_row() + { + if (this->sbc_row_sum > this->sbc_max_row_value) { + this->sbc_max_row_value = this->sbc_row_sum; + } + if (this->sbc_row_items > this->sbc_max_row_items) { + this->sbc_max_row_items = this->sbc_row_items; + } + this->sbc_row_sum = 0; + this->sbc_row_items = 0; + this->sbc_row_counter += 1; } struct bucket_stats_t { @@ -270,15 +297,10 @@ public: { } - void merge(const bucket_stats_t& rhs, bool do_stacking) + void merge(const bucket_stats_t& rhs) { this->bs_min_value = std::min(this->bs_min_value, rhs.bs_min_value); - if (do_stacking) { - this->bs_max_value += rhs.bs_max_value; - } else { - this->bs_max_value - = std::max(this->bs_max_value, rhs.bs_max_value); - } + this->bs_max_value = std::max(this->bs_max_value, rhs.bs_max_value); } double width() const @@ -310,6 +332,7 @@ protected: T ci_ident; text_attrs ci_attrs; bucket_stats_t ci_stats; + ssize_t ci_last_seen_row{-1}; }; struct chart_ident& find_ident(const T& ident) @@ -328,6 +351,12 @@ protected: std::vector sbc_idents; std::unordered_map sbc_ident_lookup; show_state sbc_show_state{show_none()}; + + ssize_t sbc_row_counter{0}; + double sbc_row_sum{0}; + size_t sbc_row_items{0}; + double sbc_max_row_value{0}; + size_t sbc_max_row_items{0}; }; class hist_source2 @@ -417,6 +446,7 @@ private: time_t hs_last_row; std::map hs_blocks; stacked_bar_chart hs_chart; + bool hs_needs_flush{false}; }; #endif diff --git a/src/listview_curses.cc b/src/listview_curses.cc index 7274bdcd..5e5579f1 100644 --- a/src/listview_curses.cc +++ b/src/listview_curses.cc @@ -267,6 +267,8 @@ listview_curses::handle_key(int ch) if (this->lv_selection <= top_for_last) { this->set_selection(top_for_last + 1_vl); } + } else if (this->lv_top > top_for_last) { + this->set_top(top_for_last); } else { this->shift_top(rows_avail); diff --git a/src/log_format.cc b/src/log_format.cc index d3ce2712..0569d395 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -245,7 +245,7 @@ logline_value_meta::to_chart_type() const retval = chart_type_t::none; break; case value_kind_t::VALUE_INTEGER: - if (!this->lvm_identifier) { + if (!this->lvm_identifier && !this->lvm_foreign_key) { retval = chart_type_t::spectro; } break; @@ -2843,7 +2843,7 @@ external_log_format::build(std::vector& errors) auto& ivd = pat.p_value_by_index[lpc]; auto vd = ivd.ivd_value_def; - if (!vd->vd_foreign_key && !vd->vd_meta.lvm_identifier) { + if (!vd->vd_meta.lvm_foreign_key && !vd->vd_meta.lvm_identifier) { switch (vd->vd_meta.lvm_kind) { case value_kind_t::VALUE_INTEGER: case value_kind_t::VALUE_FLOAT: @@ -3606,7 +3606,7 @@ external_log_format::build(std::vector& errors) elf_value_def->vd_meta.lvm_values_index = nonstd::make_optional(value_def_index++); - if (elf_value_def->vd_foreign_key + if (elf_value_def->vd_meta.lvm_foreign_key || elf_value_def->vd_meta.lvm_identifier) { continue; @@ -3856,7 +3856,7 @@ public: log_vtab_impl::get_foreign_keys(keys_inout); for (const auto& elf_value_def : this->elt_format.elf_value_defs) { - if (elf_value_def.second->vd_foreign_key) { + if (elf_value_def.second->vd_meta.lvm_foreign_key) { keys_inout.emplace_back(elf_value_def.first.to_string()); } } diff --git a/src/log_format.hh b/src/log_format.hh index bef8c4a2..8766558a 100644 --- a/src/log_format.hh +++ b/src/log_format.hh @@ -163,6 +163,7 @@ struct logline_value_meta { column_t lvm_column{external_column{}}; nonstd::optional lvm_values_index; bool lvm_identifier{false}; + bool lvm_foreign_key{false}; bool lvm_hidden{false}; nonstd::optional lvm_user_hidden; bool lvm_from_module{false}; diff --git a/src/log_format_ext.hh b/src/log_format_ext.hh index d59a15a8..35361b3c 100644 --- a/src/log_format_ext.hh +++ b/src/log_format_ext.hh @@ -68,7 +68,6 @@ public: logline_value_meta vd_meta; std::string vd_collate; - bool vd_foreign_key{false}; intern_string_t vd_unit_field; std::map vd_unit_scaling; bool vd_internal{false}; diff --git a/src/log_format_impls.cc b/src/log_format_impls.cc index 474794b9..3bd45ad0 100644 --- a/src/log_format_impls.cc +++ b/src/log_format_impls.cc @@ -489,10 +489,12 @@ public: field_def& with_kind(value_kind_t kind, bool identifier = false, + bool foreign_key = false, const std::string& collator = "") { this->fd_meta.lvm_kind = kind; this->fd_meta.lvm_identifier = identifier; + this->fd_meta.lvm_foreign_key = foreign_key; this->fd_collator = collator; return *this; } @@ -768,12 +770,14 @@ public: "bro_referrer", "bro_resp_fuids", "bro_service", - "bro_status_code", "bro_uid", "bro_uri", "bro_user_agent", "bro_username", }; + static const char* KNOWN_FOREIGN[] = { + "bro_status_code", + }; int numeric_count = 0; @@ -792,7 +796,12 @@ public: bool ident = std::binary_search(std::begin(KNOWN_IDS), std::end(KNOWN_IDS), fd.fd_meta.lvm_name); - fd.with_kind(value_kind_t::VALUE_INTEGER, ident) + bool foreign + = std::binary_search(std::begin(KNOWN_FOREIGN), + std::end(KNOWN_FOREIGN), + fd.fd_meta.lvm_name); + fd.with_kind( + value_kind_t::VALUE_INTEGER, ident, foreign) .with_numeric_index(numeric_count); numeric_count += 1; } else if (field_type == "bool") { @@ -950,7 +959,7 @@ public: this->log_vtab_impl::get_foreign_keys(keys_inout); for (const auto& fd : this->blt_format.blf_field_defs) { - if (fd.fd_meta.lvm_identifier) { + if (fd.fd_meta.lvm_identifier || fd.fd_meta.lvm_foreign_key) { keys_inout.push_back(fd.fd_meta.lvm_name.to_string()); } } @@ -1124,6 +1133,7 @@ public: const char* name, value_kind_t kind, bool ident = false, + bool foreign_key = false, std::string coll = "") : fd_name(intern_string::lookup(name)), fd_meta( @@ -1133,6 +1143,7 @@ public: fd_collator(std::move(coll)) { this->fd_meta.lvm_identifier = ident; + this->fd_meta.lvm_foreign_key = foreign_key; } field_def& with_kind(value_kind_t kind, @@ -1625,7 +1636,7 @@ public: this->log_vtab_impl::get_foreign_keys(keys_inout); for (const auto& fd : KNOWN_FIELDS) { - if (fd.fd_meta.lvm_identifier) { + if (fd.fd_meta.lvm_identifier || fd.fd_meta.lvm_foreign_key) { keys_inout.push_back(fd.fd_meta.lvm_name.to_string()); } } @@ -1687,6 +1698,7 @@ const std::vector w3c_log_format::KNOWN_FIELDS = { "c-ip", value_kind_t::VALUE_TEXT, true, + false, "ipaddress", }, { @@ -1706,6 +1718,7 @@ const std::vector w3c_log_format::KNOWN_FIELDS = { "cs-uri-stem", value_kind_t::VALUE_TEXT, true, + false, "naturalnocase", }, { @@ -1731,6 +1744,7 @@ const std::vector w3c_log_format::KNOWN_FIELDS = { "s-ip", value_kind_t::VALUE_TEXT, true, + false, "ipaddress", }, { @@ -1762,6 +1776,7 @@ const std::vector w3c_log_format::KNOWN_FIELDS = { "sc-status", value_kind_t::VALUE_INTEGER, false, + true, }, { KNOWN_FIELD_INDEX++, diff --git a/src/log_format_loader.cc b/src/log_format_loader.cc index 0fe628b8..c3544793 100644 --- a/src/log_format_loader.cc +++ b/src/log_format_loader.cc @@ -456,7 +456,8 @@ static const json_path_handler_base::enum_value_t OVERFLOW_ENUM[] = { {"truncate", external_log_format::json_format_element::overflow_t::TRUNCATE}, {"dot-dot", external_log_format::json_format_element::overflow_t::DOTDOT}, - {"last-word", external_log_format::json_format_element::overflow_t::LASTWORD}, + {"last-word", + external_log_format::json_format_element::overflow_t::LASTWORD}, json_path_handler_base::ENUM_TERMINATOR, }; @@ -616,7 +617,8 @@ static const struct json_path_container value_def_handlers = { .with_synopsis("") .with_description("Indicates whether or not this field should be " "treated as a foreign key for row in another table") - .for_field(&external_log_format::value_def::vd_foreign_key), + .for_field(&external_log_format::value_def::vd_meta, + &logline_value_meta::lvm_foreign_key), yajlpp::property_handler("hidden") .with_synopsis("") diff --git a/src/prql/stats.prql b/src/prql/stats.prql index 155be6af..bce69f47 100644 --- a/src/prql/stats.prql +++ b/src/prql/stats.prql @@ -25,9 +25,24 @@ let hist = func column slice:'1h' top:10 rel -> ( ) group { tslice } ( window ( - sort {-total} - take top - aggregate { v = json.group_object column total } + sort {-total} + derive {rn = row_number column} + ) + ) + derive top_value = case [ + rn < top => column, + rn >= top => 'Other', + ] + group { tslice, top_value } ( + aggregate { total2 = sum total } + ) + group { tslice } ( + window ( + sort {-total2} + # XXX The `take` here is necessary to workaround a + # PRQL issue where the above sort is dropped entirely + take top + aggregate { v = json.group_object top_value total2 } ) ) ) diff --git a/src/textview_curses.cc b/src/textview_curses.cc index 2020e296..c0783664 100644 --- a/src/textview_curses.cc +++ b/src/textview_curses.cc @@ -595,7 +595,9 @@ textview_curses::handle_mouse(mouse_event& me) (int) this->lv_left + me.me_x); this->set_selection_without_context(mc.mc_line); - if (me.me_button == mouse_button_t::BUTTON_LEFT) { + if (this->tc_supports_marks + && me.me_button == mouse_button_t::BUTTON_LEFT) + { this->textview_value_for_row(mc.mc_line, al); auto line_sf = string_fragment::from_str(al.get_string()); diff --git a/test/expected/expected.am b/test/expected/expected.am index 5c278b0f..bedc4dcf 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -474,10 +474,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_prql.sh_06900fac5c2e854b1208320b753fcd43d4ba63a3.out \ $(srcdir)/%reldir%/test_prql.sh_15ebcd913f56cde35af26c6300533d6e76bf1b55.err \ $(srcdir)/%reldir%/test_prql.sh_15ebcd913f56cde35af26c6300533d6e76bf1b55.out \ - $(srcdir)/%reldir%/test_prql.sh_451e242cdfa2db9005d4fe752a7b05d1ab5cba29.err \ - $(srcdir)/%reldir%/test_prql.sh_451e242cdfa2db9005d4fe752a7b05d1ab5cba29.out \ - $(srcdir)/%reldir%/test_prql.sh_45d57a042092ffdcd28ea35a892f02859e78f33d.err \ - $(srcdir)/%reldir%/test_prql.sh_45d57a042092ffdcd28ea35a892f02859e78f33d.out \ + $(srcdir)/%reldir%/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.err \ + $(srcdir)/%reldir%/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.out \ $(srcdir)/%reldir%/test_regex101.sh_0fa3663a45aca6a328cb728872af7ed7ee896f1c.err \ $(srcdir)/%reldir%/test_regex101.sh_0fa3663a45aca6a328cb728872af7ed7ee896f1c.out \ $(srcdir)/%reldir%/test_regex101.sh_182ae9244db314a953af2bee969726e381bc5a32.err \ diff --git a/test/expected/test_cmds.sh_a00943ef715598c7554b85de8502454e41bb9e28.out b/test/expected/test_cmds.sh_a00943ef715598c7554b85de8502454e41bb9e28.out index 95268f80..bcef8b02 100644 --- a/test/expected/test_cmds.sh_a00943ef715598c7554b85de8502454e41bb9e28.out +++ b/test/expected/test_cmds.sh_a00943ef715598c7554b85de8502454e41bb9e28.out @@ -1,4 +1,4 @@ - Thu Nov 03 09:20:00  1 normal 2 errors 0 warnings  0 marks - Thu Nov 03 09:45:00  1 normal 0 errors 0 warnings 0 marks - Fri Feb 03 09:20:00  0 normal 1 errors 0 warnings 0 marks - Wed Jan 03 09:20:00  1 normal 0 errors 0 warnings 0 marks + Thu Nov 03 09:20:00  1 normal 2 errors 0 warnings  0 marks + Thu Nov 03 09:45:00  1 normal 0 errors 0 warnings 0 marks + Fri Feb 03 09:20:00  0 normal 1 errors 0 warnings 0 marks + Wed Jan 03 09:20:00  1 normal 0 errors 0 warnings 0 marks diff --git a/test/expected/test_cmds.sh_af0fcbd30b3fd0d13477aa3325ef0302052a4d9f.out b/test/expected/test_cmds.sh_af0fcbd30b3fd0d13477aa3325ef0302052a4d9f.out index be068828..f4f4bcda 100644 --- a/test/expected/test_cmds.sh_af0fcbd30b3fd0d13477aa3325ef0302052a4d9f.out +++ b/test/expected/test_cmds.sh_af0fcbd30b3fd0d13477aa3325ef0302052a4d9f.out @@ -1 +1 @@ - Sat Nov 03 08:00:00 1 normal 0 errors 0 warnings 0 marks + Sat Nov 03 08:00:00 1 normal 0 errors 0 warnings  0 marks diff --git a/test/expected/test_cmds.sh_c7fabc25374ff47c47931f63b1d697061b816a28.out b/test/expected/test_cmds.sh_c7fabc25374ff47c47931f63b1d697061b816a28.out index 76c53dd8..fe0221c7 100644 --- a/test/expected/test_cmds.sh_c7fabc25374ff47c47931f63b1d697061b816a28.out +++ b/test/expected/test_cmds.sh_c7fabc25374ff47c47931f63b1d697061b816a28.out @@ -1,2 +1,2 @@ - Sat Nov 03 09:20:00 1 normal 2 errors  0 warnings  1 marks - Sat Nov 03 09:45:00 1 normal 0 errors 0 warnings 0 marks + Sat Nov 03 09:20:00 1 normal 2 errors  0 warnings  1 marks + Sat Nov 03 09:45:00 1 normal 0 errors 0 warnings 0 marks diff --git a/test/expected/test_cmds.sh_ccd326da92d1cacda63501cd1a3077381a18e8f2.out b/test/expected/test_cmds.sh_ccd326da92d1cacda63501cd1a3077381a18e8f2.out index bc678377..0c0137a1 100644 --- a/test/expected/test_cmds.sh_ccd326da92d1cacda63501cd1a3077381a18e8f2.out +++ b/test/expected/test_cmds.sh_ccd326da92d1cacda63501cd1a3077381a18e8f2.out @@ -1 +1 @@ - Sat Nov 03 00:00:00 2 normal 2 errors 0 warnings 0 marks + Sat Nov 03 00:00:00 2 normal  2 errors 0 warnings  0 marks diff --git a/test/expected/test_cmds.sh_dbdd62995fdefc8318053af05a32416eccfa79fc.out b/test/expected/test_cmds.sh_dbdd62995fdefc8318053af05a32416eccfa79fc.out index f3d0606c..24a89e78 100644 --- a/test/expected/test_cmds.sh_dbdd62995fdefc8318053af05a32416eccfa79fc.out +++ b/test/expected/test_cmds.sh_dbdd62995fdefc8318053af05a32416eccfa79fc.out @@ -1 +1 @@ - Sat Nov 03 08:00:00 2 normal 2 errors 0 warnings 0 marks + Sat Nov 03 08:00:00 2 normal  2 errors 0 warnings  0 marks diff --git a/test/expected/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.err b/test/expected/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.out b/test/expected/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.out new file mode 100644 index 00000000..3c626033 --- /dev/null +++ b/test/expected/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.out @@ -0,0 +1,4 @@ + tslice   v   +2011-11-03 00:19:00.000 {"a.fsdn.com":33,"Other":31,"ad.doubleclick.net":10,"www.goo⋯"\xaf"dia.com":6,"s0.2mdn.net":5,"googleads.g.doubleclick.net":3} +2011-11-03 00:20:00.000 {"www.bro-ids.org":24,"www.google.com":15,"search.twitter.c⋯"\x8b""\xaf"ds.org":2,"ad.doubleclick.net":1,"cont-sjl-1.pandora.com":1} +2011-11-03 00:21:00.000 {"www.google.com":2,"ajax.googleapis.com":1,"api.twitter.com":1,"search.twitter.com":1,"www.bro-ids.org":1}  diff --git a/test/expected/test_text_file.sh_7b00f32a3fff7fc2d78a87045ae842e58be88480.out b/test/expected/test_text_file.sh_7b00f32a3fff7fc2d78a87045ae842e58be88480.out index 4855b3de..c9d79429 100644 --- a/test/expected/test_text_file.sh_7b00f32a3fff7fc2d78a87045ae842e58be88480.out +++ b/test/expected/test_text_file.sh_7b00f32a3fff7fc2d78a87045ae842e58be88480.out @@ -5,6 +5,6 @@  * FEATURES: network-sandbox preserve-libs sandbox test userpriv usersandbox * Checking whether python3_11 is suitable ... * >=dev-lang/python-3.11.1-r1:3.11 ... - [ ok ] + [ ok ] * Using python3.11 to build (via PYTHON_COMPAT iteration) >>> Unpacking source... diff --git a/test/test_prql.sh b/test/test_prql.sh index 90e0d737..bc279087 100644 --- a/test/test_prql.sh +++ b/test/test_prql.sh @@ -11,3 +11,7 @@ run_cap_test ${lnav_test} -n \ run_cap_test ${lnav_test} -n \ -c ";from access_log | take abc" \ ${test_dir}/logfile_access_log.0 + +run_cap_test ${lnav_test} -n \ + -c ";from bro_http_log | stats.hist bro_host slice:'1m'" \ + ${test_dir}/logfile_bro_http.log.0