diff --git a/release/loggen.py b/release/loggen.py index 90ecdd9a..955d1b88 100755 --- a/release/loggen.py +++ b/release/loggen.py @@ -159,6 +159,6 @@ while True: fp.write(next(gen)) #if random.uniform(0.0, 1.0) < 0.010: # fp.truncate(0) - time.sleep(random.uniform(0.05, 0.10)) + time.sleep(random.uniform(0.01, 0.02)) #if random.uniform(0.0, 1.0) < 0.001: # os.remove(fname) diff --git a/src/base/attr_line.hh b/src/base/attr_line.hh index dd8bc647..7def6417 100644 --- a/src/base/attr_line.hh +++ b/src/base/attr_line.hh @@ -501,6 +501,15 @@ public: return *this; } + attr_line_t& append_quoted(const attr_line_t& al) + { + this->al_string.append("\u201c"); + this->append(al); + this->al_string.append("\u201d"); + + return *this; + } + template attr_line_t& append_quoted(S s) { diff --git a/src/base/itertools.hh b/src/base/itertools.hh index e57c6b31..070448b5 100644 --- a/src/base/itertools.hh +++ b/src/base/itertools.hh @@ -93,6 +93,16 @@ struct mapper { F m_func; }; +template +struct flat_mapper { + F fm_func; +}; + +template +struct for_eacher { + F fe_func; +}; + template struct folder { R f_func; @@ -243,6 +253,20 @@ map(F func) return details::mapper{func}; } +template +inline details::flat_mapper +flat_map(F func) +{ + return details::flat_mapper{func}; +} + +template +inline details::for_eacher +for_each(F func) +{ + return details::for_eacher{func}; +} + inline auto deref() { @@ -534,6 +558,36 @@ operator|(T in, const lnav::itertools::details::sorted& sorter) return in; } +template::value, int> = 0> +auto +operator|(nonstd::optional in, + const lnav::itertools::details::flat_mapper& mapper) -> + typename std::remove_const_t> +{ + if (!in) { + return nonstd::nullopt; + } + + return lnav::func::invoke(mapper.fm_func, in.value()); +} + +template::value, int> = 0> +void +operator|(nonstd::optional in, + const lnav::itertools::details::for_eacher& eacher) +{ + if (!in) { + return; + } + + lnav::func::invoke(eacher.fe_func, in.value()); +} + template::value, int> = 0> diff --git a/src/db_sub_source.cc b/src/db_sub_source.cc index 5876a2f1..2e6b0bc9 100644 --- a/src/db_sub_source.cc +++ b/src/db_sub_source.cc @@ -205,6 +205,7 @@ db_label_source::push_column(const char* colstr) } if (!this->dls_time_column.empty() && tv < this->dls_time_column.back()) { + this->dls_time_column_invalidated_at = this->dls_time_column.size(); this->dls_time_column_index = -1; this->dls_time_column.clear(); } else { diff --git a/src/db_sub_source.hh b/src/db_sub_source.hh index 6b90cbc1..720d419d 100644 --- a/src/db_sub_source.hh +++ b/src/db_sub_source.hh @@ -43,21 +43,20 @@ class db_label_source : public text_sub_source , public text_time_translator { public: - ~db_label_source() - { - this->clear(); - } + ~db_label_source() override { this->clear(); } bool has_log_time_column() const { return !this->dls_time_column.empty(); } - size_t text_line_count() { return this->dls_rows.size(); } + size_t text_line_count() override { return this->dls_rows.size(); } - size_t text_size_for_line(textview_curses& tc, int line, line_flags_t flags) + size_t text_size_for_line(textview_curses& tc, + int line, + line_flags_t flags) override { return this->text_line_width(tc); } - size_t text_line_width(textview_curses& curses) + size_t text_line_width(textview_curses& curses) override { size_t retval = 0; @@ -70,9 +69,11 @@ public: void text_value_for_line(textview_curses& tc, int row, std::string& label_out, - line_flags_t flags); + line_flags_t flags) override; - void text_attrs_for_line(textview_curses& tc, int row, string_attrs_t& sa); + void text_attrs_for_line(textview_curses& tc, + int row, + string_attrs_t& sa) override; void push_header(const std::string& colstr, int type, bool graphable); @@ -83,9 +84,10 @@ public: nonstd::optional column_name_to_index( const std::string& name) const; - nonstd::optional row_for_time(struct timeval time_bucket); + nonstd::optional row_for_time( + struct timeval time_bucket) override; - nonstd::optional time_for_row(vis_line_t row) + nonstd::optional time_for_row(vis_line_t row) override { if ((row < 0_vl) || (((size_t) row) >= this->dls_time_column.size())) { return nonstd::nullopt; @@ -106,7 +108,6 @@ public: int hm_column_type{SQLITE3_TEXT}; unsigned int hm_sub_type{0}; bool hm_graphable{false}; - bool hm_log_time{false}; size_t hm_column_size{0}; }; @@ -116,6 +117,7 @@ public: std::vector dls_time_column; std::vector dls_cell_width; int dls_time_column_index{-1}; + nonstd::optional dls_time_column_invalidated_at; static const char* NULL_STR; }; diff --git a/src/internals/cmd-ref.rst b/src/internals/cmd-ref.rst index 0fb5eb9c..314d6a79 100644 --- a/src/internals/cmd-ref.rst +++ b/src/internals/cmd-ref.rst @@ -1240,7 +1240,7 @@ :spectrogram *field-name* ^^^^^^^^^^^^^^^^^^^^^^^^^ - Visualize the given message field using a spectrogram + Visualize the given message field or database column using a spectrogram **Parameters** * **field-name\*** --- The name of the numeric field to visualize. diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index ba1db672..d838b2bc 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -4410,7 +4410,7 @@ com_spectrogram(exec_context& ec, } else if (ec.ec_dry_run) { retval = ""; } else if (args.size() == 2) { - std::string colname = remaining_args(cmdline, args); + auto colname = remaining_args(cmdline, args); auto& ss = *lnav_data.ld_spectro_source; bool found = false; @@ -4422,17 +4422,17 @@ com_spectrogram(exec_context& ec, ss.invalidate(); if (*lnav_data.ld_view_stack.top() == &lnav_data.ld_views[LNV_DB]) { - std::unique_ptr dsvs( - new db_spectro_value_source(colname)); + auto dsvs = std::make_unique(colname); - if (!dsvs->dsvs_error_msg.empty()) { - return ec.make_error("{}", dsvs->dsvs_error_msg); + if (dsvs->dsvs_error_msg) { + return Err( + dsvs->dsvs_error_msg.value().with_snippets(ec.ec_source)); } ss.ss_value_source = dsvs.release(); found = true; } else { - std::unique_ptr lsvs( - new log_spectro_value_source(intern_string::lookup(colname))); + auto lsvs = std::make_unique( + intern_string::lookup(colname)); if (!lsvs->lsvs_found) { return ec.make_error("unknown numeric message field -- {}", @@ -5594,7 +5594,8 @@ readline_context::command_t STD_COMMANDS[] = { com_spectrogram, help_text(":spectrogram") - .with_summary("Visualize the given message field using a spectrogram") + .with_summary("Visualize the given message field or database column " + "using a spectrogram") .with_parameter(help_text( "field-name", "The name of the numeric field to visualize.")) .with_example( @@ -5605,12 +5606,10 @@ readline_context::command_t STD_COMMANDS[] = { help_text(":quit").with_summary("Quit lnav")}}; -static std::unordered_map> aliases - = {{"quit", {"q", "q!"}}, - {"write-table-to", - { - "write-cols-to", - }}}; +static std::unordered_map> aliases = { + {"quit", {"q", "q!"}}, + {"write-table-to", {"write-cols-to"}}, +}; void init_lnav_commands(readline_context::command_map_t& cmd_map) diff --git a/src/spectro_impls.cc b/src/spectro_impls.cc index 18903f21..cdf42658 100644 --- a/src/spectro_impls.cc +++ b/src/spectro_impls.cc @@ -29,6 +29,7 @@ #include "spectro_impls.hh" +#include "base/itertools.hh" #include "lnav.hh" #include "logfile_sub_source.hh" @@ -43,40 +44,50 @@ public: std::string& value_out, line_flags_t flags) override { - this->fss_delegate->text_value_for_line( - tc, this->fss_lines[line], value_out, flags); + this->fss_lines | lnav::itertools::nth(line) + | lnav::itertools::for_each([&](const auto row) { + this->fss_delegate->text_value_for_line( + tc, *row, value_out, flags); + }); } size_t text_size_for_line(textview_curses& tc, int line, line_flags_t raw) override { - return this->fss_delegate->text_size_for_line( - tc, this->fss_lines[line], raw); + return this->fss_lines | lnav::itertools::nth(line) + | lnav::itertools::map([&](const auto row) { + return this->fss_delegate->text_size_for_line(tc, *row, raw); + }) + | lnav::itertools::unwrap_or(size_t{0}); } void text_attrs_for_line(textview_curses& tc, int line, string_attrs_t& value_out) override { - this->fss_delegate->text_attrs_for_line( - tc, this->fss_lines[line], value_out); + this->fss_lines | lnav::itertools::nth(line) + | lnav::itertools::for_each([&](const auto row) { + this->fss_delegate->text_attrs_for_line(tc, *row, value_out); + }); } nonstd::optional row_for_time( struct timeval time_bucket) override { - return dynamic_cast(this->fss_delegate) - ->row_for_time(time_bucket); + return this->fss_time_delegate->row_for_time(time_bucket); } nonstd::optional time_for_row(vis_line_t row) override { - return dynamic_cast(this->fss_delegate) - ->time_for_row(this->fss_lines[row]); + return this->fss_lines | lnav::itertools::nth(row) + | lnav::itertools::flat_map([this](const auto row) { + return this->fss_time_delegate->time_for_row(*row); + }); } text_sub_source* fss_delegate; + text_time_translator* fss_time_delegate; std::vector fss_lines; }; @@ -202,6 +213,7 @@ log_spectro_value_source::spectro_row(spectrogram_request& sr, .value_or(lss.text_line_count()); retval->fss_delegate = &lss; + retval->fss_time_delegate = &lss; for (const auto& msg_info : lss.window_at(begin_line, end_line)) { const auto& ll = msg_info.get_logline(); if (ll.get_time() >= sr.sr_end_time) { @@ -311,36 +323,103 @@ db_spectro_value_source::update_stats() this->dsvs_end_time = 0; this->dsvs_stats.clear(); - db_label_source& dls = lnav_data.ld_db_row_source; - stacked_bar_chart& chart = dls.dls_chart; + auto& dls = lnav_data.ld_db_row_source; + auto& chart = dls.dls_chart; date_time_scanner dts; this->dsvs_column_index = dls.column_name_to_index(this->dsvs_colname); if (!dls.has_log_time_column()) { - this->dsvs_error_msg - = "no 'log_time' column found or not in ascending order, " - "unable to create spectrogram"; + if (dls.dls_time_column_invalidated_at) { + static const auto order_by_help + = attr_line_t() + .append(lnav::roles::keyword("ORDER BY")) + .append(" ") + .append(lnav::roles::variable("log_time")) + .append(" ") + .append(lnav::roles::keyword("ASC")); + + this->dsvs_error_msg + = lnav::console::user_message::error( + "Cannot generate spectrogram for database results") + .with_reason( + attr_line_t() + .append("The ") + .append_quoted(lnav::roles::variable("log_time")) + .append( + " column is not in ascending order between " + "rows {} and {}", + dls.dls_time_column_invalidated_at.value() + - 1, + dls.dls_time_column_invalidated_at.value())) + .with_note( + attr_line_t("An ascending ") + .append_quoted(lnav::roles::variable("log_time")) + .append( + " column is needed to render a spectrogram")) + .with_help(attr_line_t("Add an ") + .append_quoted(order_by_help) + .append(" clause to your ") + .append(lnav::roles::keyword("SELECT")) + .append(" statement")); + } else { + this->dsvs_error_msg + = lnav::console::user_message::error( + "Cannot generate spectrogram for database results") + .with_reason( + attr_line_t() + .append("No ") + .append_quoted(lnav::roles::variable("log_time")) + .append(" column found in the result set")) + .with_note( + attr_line_t("An ascending ") + .append_quoted(lnav::roles::variable("log_time")) + .append( + " column is needed to render a spectrogram")) + .with_help( + attr_line_t("Include a ") + .append_quoted(lnav::roles::variable("log_time")) + .append(" column in your ") + .append(" statement. Use an ") + .append(lnav::roles::keyword("AS")) + .append( + " directive to alias a computed timestamp")); + } return; } if (!this->dsvs_column_index) { - this->dsvs_error_msg = "unknown column -- " + this->dsvs_colname; + this->dsvs_error_msg + = lnav::console::user_message::error( + "Cannot generate spectrogram for database results") + .with_reason(attr_line_t("unknown column -- ") + .append_quoted(lnav::roles::variable( + this->dsvs_colname))) + .with_help("Expecting a numeric column to visualize"); return; } if (!dls.dls_headers[this->dsvs_column_index.value()].hm_graphable) { - this->dsvs_error_msg = "column is not numeric -- " + this->dsvs_colname; + this->dsvs_error_msg + = lnav::console::user_message::error( + "Cannot generate spectrogram for database results") + .with_reason(attr_line_t() + .append_quoted(lnav::roles::variable( + this->dsvs_colname)) + .append(" is not a numeric column")) + .with_help("Only numeric columns can be visualized"); return; } if (dls.dls_rows.empty()) { - this->dsvs_error_msg = "empty result set"; + this->dsvs_error_msg + = lnav::console::user_message::error( + "Cannot generate spectrogram for database results") + .with_reason("Result set is empty"); return; } - stacked_bar_chart::bucket_stats_t bs - = chart.get_stats_for(this->dsvs_colname); + auto bs = chart.get_stats_for(this->dsvs_colname); this->dsvs_begin_time = dls.dls_time_column.front().tv_sec; this->dsvs_end_time = dls.dls_time_column.back().tv_sec; @@ -392,6 +471,7 @@ db_spectro_value_source::spectro_row(spectrogram_request& sr, auto retval = std::make_unique(); retval->fss_delegate = &dls; + retval->fss_time_delegate = &dls; auto begin_row = dls.row_for_time({sr.sr_begin_time, 0}).value_or(0_vl); auto end_row = dls.row_for_time({sr.sr_end_time, 0}) .value_or(dls.dls_rows.size()); diff --git a/src/spectro_impls.hh b/src/spectro_impls.hh index 17d0369e..9d4ddeba 100644 --- a/src/spectro_impls.hh +++ b/src/spectro_impls.hh @@ -81,7 +81,7 @@ public: time_t dsvs_begin_time{0}; time_t dsvs_end_time{0}; nonstd::optional dsvs_column_index; - std::string dsvs_error_msg; + nonstd::optional dsvs_error_msg; }; #endif diff --git a/src/spectro_source.cc b/src/spectro_source.cc index 1d331614..0183492c 100644 --- a/src/spectro_source.cc +++ b/src/spectro_source.cc @@ -210,24 +210,30 @@ spectrogram_source::list_value_for_overlay(const listview_curses& lv, + this->ss_cursor_column.value() * sr.sr_column_size; auto range_max = range_min + sr.sr_column_size; - const auto desc = fmt::format( - FMT_STRING("{} value{} in the range {:.6Lg}-{:.6Lg} "), - bucket.rb_counter, - bucket.rb_counter == 1 ? "" : "s", - range_min, - range_max); + auto desc = attr_line_t() + .append(lnav::roles::number( + fmt::to_string(bucket.rb_counter))) + .append(lnav::roles::comment(fmt::format( + FMT_STRING(" value{} in the range "), + bucket.rb_counter == 1 ? "" : "s"))) + .append(lnav::roles::number( + fmt::format(FMT_STRING("{:.2Lf}"), range_min))) + .append(lnav::roles::comment("-")) + .append(lnav::roles::number( + fmt::format(FMT_STRING("{:.2Lf}"), range_max))) + .append(" "); auto mark_offset = this->ss_cursor_column.value(); auto mark_is_before = true; - if (this->ss_cursor_column.value() + desc.size() > width) { - mark_offset -= desc.size(); + if (this->ss_cursor_column.value() + desc.length() + 1 > width) { + mark_offset -= desc.length(); mark_is_before = false; } value_out.append(mark_offset, ' '); if (mark_is_before) { value_out.append("\u25b2 "); } - value_out.append(lnav::roles::comment(desc)); + value_out.append(desc); if (!mark_is_before) { value_out.append("\u25b2 "); } @@ -257,7 +263,10 @@ spectrogram_source::list_value_for_overlay(const listview_curses& lv, this->cache_bounds(); if (this->ss_cached_line_count == 0) { - value_out.append(lnav::roles::error("error: no log data")); + value_out + .append(lnav::roles::error("error: no data available, use the ")) + .append_quoted(lnav::roles::keyword(":spectrogram")) + .append(lnav::roles::error(" command to visualize numeric data")); return true; } @@ -433,12 +442,23 @@ spectrogram_source::text_attrs_for_line(textview_curses& tc, } } +void +spectrogram_source::reset_details_source() +{ + if (this->ss_details_view != nullptr) { + this->ss_details_view->set_sub_source(this->ss_no_details_source); + } + this->ss_details_source.reset(); +} + void spectrogram_source::cache_bounds() { if (this->ss_value_source == nullptr) { this->ss_cached_bounds.sb_count = 0; this->ss_cached_bounds.sb_begin_time = 0; + this->ss_cursor_column = nonstd::nullopt; + this->reset_details_source(); return; } @@ -454,6 +474,8 @@ spectrogram_source::cache_bounds() if (sb.sb_count == 0) { this->ss_cached_line_count = 0; + this->ss_cursor_column = nonstd::nullopt; + this->reset_details_source(); return; } @@ -533,7 +555,8 @@ spectrogram_source::text_is_row_selectable(textview_curses& tc, vis_line_t row) void spectrogram_source::text_selection_changed(textview_curses& tc) { - if (this->ss_value_source == nullptr) { + if (this->ss_value_source == nullptr || this->text_line_count() == 0) { + this->ss_cursor_column = nonstd::nullopt; return; } diff --git a/src/spectro_source.hh b/src/spectro_source.hh index cce9a372..c337ef1e 100644 --- a/src/spectro_source.hh +++ b/src/spectro_source.hh @@ -171,6 +171,8 @@ public: const spectrogram_row& load_row(const listview_curses& lv, int row); + void reset_details_source(); + textview_curses* ss_details_view; text_sub_source* ss_no_details_source; exec_context* ss_exec_context; diff --git a/src/view_curses.hh b/src/view_curses.hh index 5d873e49..0caabb40 100644 --- a/src/view_curses.hh +++ b/src/view_curses.hh @@ -291,6 +291,8 @@ private: /** Private constructor that initializes the member fields. */ view_colors(); + view_colors(const view_colors&) = delete; + struct dyn_pair { int dp_color_pair; }; diff --git a/src/view_helpers.cc b/src/view_helpers.cc index 6cf80b00..1111cbdf 100644 --- a/src/view_helpers.cc +++ b/src/view_helpers.cc @@ -1100,6 +1100,7 @@ void hist_index_delegate::index_complete(logfile_sub_source& lss) { this->hid_view.reload_data(); + lnav_data.ld_views[LNV_SPECTRO].reload_data(); } static std::vector diff --git a/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out b/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out index 1c6abf70..3d8822e1 100644 --- a/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out +++ b/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out @@ -1416,7 +1416,8 @@ lnav@googlegroups.com[1] support@lnav.org[2] :spectrogram field-name ══════════════════════════════════════════════════════════════════════ - Visualize the given message field using a spectrogram + Visualize the given message field or database column using a + spectrogram Parameter field-name The name of the numeric field to visualize. diff --git a/test/expected/test_sql.sh_26c0d94d7837792144f2d0f866fb3c12a0bd410d.err b/test/expected/test_sql.sh_26c0d94d7837792144f2d0f866fb3c12a0bd410d.err index 4233a661..78e48373 100644 --- a/test/expected/test_sql.sh_26c0d94d7837792144f2d0f866fb3c12a0bd410d.err +++ b/test/expected/test_sql.sh_26c0d94d7837792144f2d0f866fb3c12a0bd410d.err @@ -1,6 +1,6 @@ -✘ error: no 'log_time' column found or not in ascending order, unable to create spectrogram +✘ error: Cannot generate spectrogram for database results + reason: No “log_time” column found in the result set  --> command-option:2  | :spectrogram sc_bytes  - = help: :spectrogram field-name - ══════════════════════════════════════════════════════════════════════ - Visualize the given message field using a spectrogram + = note: An ascending “log_time” column is needed to render a spectrogram + = help: Include a “log_time” column in your statement. Use an AS directive to alias a computed timestamp diff --git a/test/expected/test_sql.sh_4090f96ea11a344c1e2939211da778992dab47d8.err b/test/expected/test_sql.sh_4090f96ea11a344c1e2939211da778992dab47d8.err index 66291e24..f6c8d4b0 100644 --- a/test/expected/test_sql.sh_4090f96ea11a344c1e2939211da778992dab47d8.err +++ b/test/expected/test_sql.sh_4090f96ea11a344c1e2939211da778992dab47d8.err @@ -1,6 +1,5 @@ -✘ error: unknown column -- sc_byes +✘ error: Cannot generate spectrogram for database results + reason: unknown column -- “sc_byes”  --> command-option:2  | :spectrogram sc_byes  - = help: :spectrogram field-name - ══════════════════════════════════════════════════════════════════════ - Visualize the given message field using a spectrogram + = help: Expecting a numeric column to visualize diff --git a/test/expected/test_sql.sh_57427f3c4b4ec785ffff7c5802c10db0d3e547cf.err b/test/expected/test_sql.sh_57427f3c4b4ec785ffff7c5802c10db0d3e547cf.err index 057ca08c..95fe71a3 100644 --- a/test/expected/test_sql.sh_57427f3c4b4ec785ffff7c5802c10db0d3e547cf.err +++ b/test/expected/test_sql.sh_57427f3c4b4ec785ffff7c5802c10db0d3e547cf.err @@ -1,6 +1,5 @@ -✘ error: column is not numeric -- c_ip +✘ error: Cannot generate spectrogram for database results + reason: “c_ip” is not a numeric column  --> command-option:2  | :spectrogram c_ip  - = help: :spectrogram field-name - ══════════════════════════════════════════════════════════════════════ - Visualize the given message field using a spectrogram + = help: Only numeric columns can be visualized diff --git a/test/expected/test_sql.sh_9b03e9f7a1bc35e408b3a17ee90cfdadea164df6.out b/test/expected/test_sql.sh_9b03e9f7a1bc35e408b3a17ee90cfdadea164df6.out index 4518a348..24b33acc 100644 --- a/test/expected/test_sql.sh_9b03e9f7a1bc35e408b3a17ee90cfdadea164df6.out +++ b/test/expected/test_sql.sh_9b03e9f7a1bc35e408b3a17ee90cfdadea164df6.out @@ -1,4 +1,4 @@ Min: 0   1-23   24-48   49+ Max: 291690  Thu Nov 03 00:15:00                -▲ 70 values in the range 0-3788.18  +▲ 70 values in the range 0.00-3788.18  Thu Nov 03 00:20:00 diff --git a/test/expected/test_sql.sh_9ceccab07fbf7130bffe3c201c710719e4a3e9af.err b/test/expected/test_sql.sh_9ceccab07fbf7130bffe3c201c710719e4a3e9af.err index 4233a661..b4340e4e 100644 --- a/test/expected/test_sql.sh_9ceccab07fbf7130bffe3c201c710719e4a3e9af.err +++ b/test/expected/test_sql.sh_9ceccab07fbf7130bffe3c201c710719e4a3e9af.err @@ -1,6 +1,6 @@ -✘ error: no 'log_time' column found or not in ascending order, unable to create spectrogram +✘ error: Cannot generate spectrogram for database results + reason: The “log_time” column is not in ascending order between rows 1 and 2  --> command-option:2  | :spectrogram sc_bytes  - = help: :spectrogram field-name - ══════════════════════════════════════════════════════════════════════ - Visualize the given message field using a spectrogram + = note: An ascending “log_time” column is needed to render a spectrogram + = help: Add an “ORDER BY log_time ASC” clause to your SELECT statement