[prql] limit stats.hist to top 20 by default and order results

master
Tim Stack 1 month ago
parent 5a744f0c66
commit 84000f46f7

@ -407,7 +407,18 @@ string_fragment::sub_cell_range(int cell_start, int cell_end) const
if (read_res.isErr()) {
byte_index += 1;
} else {
cell_index += wcwidth(read_res.unwrap());
auto ch = read_res.unwrap();
switch (ch) {
case '\t':
do {
cell_index += 1;
} while (cell_index % 8);
break;
default:
cell_index += wcwidth(read_res.unwrap());
break;
}
}
}
if (cell_start == cell_index) {
@ -436,7 +447,18 @@ string_fragment::column_width() const
if (read_res.isErr()) {
retval += 1;
} else {
retval += wcwidth(read_res.unwrap());
auto ch = read_res.unwrap();
switch (ch) {
case '\t':
do {
retval += 1;
} while (retval % 8);
break;
default:
retval += wcwidth(read_res.unwrap());
break;
}
}
}

@ -1156,6 +1156,28 @@ exec_context::exec_context(logline_value_vector* line_values,
void
exec_context::execute(const std::string& cmdline)
{
if (this->get_provenance<mouse_input>()) {
require(!lnav_data.ld_rl_view->is_active());
int context = 0;
switch (cmdline[0]) {
case '/':
context = lnav::enums::to_underlying(ln_mode_t::SEARCH);
break;
case ':':
context = lnav::enums::to_underlying(ln_mode_t::COMMAND);
break;
case ';':
context = lnav::enums::to_underlying(ln_mode_t::SQL);
break;
case '|':
context = lnav::enums::to_underlying(ln_mode_t::EXEC);
break;
}
lnav_data.ld_rl_view->append_to_history(context, cmdline.substr(1));
}
auto exec_res = execute_any(*this, cmdline);
if (exec_res.isErr()) {
this->ec_error_callback_stack.back()(exec_res.unwrapErr());

@ -121,12 +121,13 @@ struct exec_context {
void clear_output();
struct mouse_input {};
struct user {};
struct file_open {
std::string fo_name;
};
using provenance_t = mapbox::util::variant<user, file_open>;
using provenance_t = mapbox::util::variant<user, mouse_input, file_open>;
struct provenance_guard {
explicit provenance_guard(exec_context* context, provenance_t prov)
@ -149,9 +150,16 @@ struct exec_context {
}
}
exec_context* operator->() { return this->pg_context; }
exec_context* pg_context;
};
provenance_guard with_provenance(provenance_t prov)
{
return provenance_guard{this, prov};
}
struct source_guard {
source_guard(exec_context* context) : sg_context(context) {}

@ -55,7 +55,7 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
auto& vc = view_colors::singleton();
this->fos_lines.clear();
this->fos_row_to_field_name.clear();
this->fos_row_to_field_meta.clear();
if (lss.text_line_count() == 0) {
this->fos_log_helper.clear();
@ -342,13 +342,23 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
al.append("\u25c6"_ok);
}
al.append(" ");
switch (meta.to_chart_type()) {
case chart_type_t::none:
al.append(" ");
break;
case chart_type_t::hist:
case chart_type_t::spectro:
al.append(":bar_chart:"_emoji).append(" ");
break;
}
auto prefix_len = al.utf8_length_or_length();
hl_range.lr_start = al.get_string().length();
al.append(field_name);
hl_range.lr_end = al.get_string().length();
al.pad_to(prefix_len + this->fos_known_key_size);
this->fos_row_to_field_name[this->fos_lines.size()] = meta.lvm_name;
this->fos_row_to_field_meta.emplace(this->fos_lines.size(), meta);
} else {
auto jget_str = lnav::sql::mprintf("jget(%s, '/%q')",
meta.lvm_struct_name.get(),
@ -436,7 +446,8 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
this->fos_lines.emplace_back(" No discovered message fields");
} else {
this->fos_lines.emplace_back(
" Discovered fields for logline table from message format: ");
" Discovered fields for logline table from message "
"format: ");
this->fos_lines.back().with_attr(
string_attr(line_range(23, 23 + 7),
VC_STYLE.value(vc.attrs_for_ident("logline"))));
@ -688,7 +699,9 @@ field_overlay_source::list_overlay_menu(const listview_curses& lv,
[this](const std::string& value) {
auto cmd = fmt::format(FMT_STRING(":filter-in {}"),
lnav::pcre2pp::quote(value));
execute_any(*this->fos_lss.get_exec_context(), cmd);
this->fos_lss.get_exec_context()
->with_provenance(exec_context::mouse_input{})
->execute(cmd);
});
start += al.length();
al.append(":mag_right:"_emoji)
@ -700,7 +713,9 @@ field_overlay_source::list_overlay_menu(const listview_curses& lv,
[this](const std::string& value) {
auto cmd = fmt::format(FMT_STRING("/{}"),
lnav::pcre2pp::quote(value));
execute_any(*this->fos_lss.get_exec_context(), cmd);
this->fos_lss.get_exec_context()
->with_provenance(exec_context::mouse_input{})
->execute(cmd);
});
retval.emplace_back(attr_line_t().pad_to(left).append(al));
}
@ -715,7 +730,9 @@ field_overlay_source::list_overlay_menu(const listview_curses& lv,
[this](const std::string& value) {
auto cmd = fmt::format(FMT_STRING(":filter-out {}"),
lnav::pcre2pp::quote(value));
execute_any(*this->fos_lss.get_exec_context(), cmd);
this->fos_lss.get_exec_context()
->with_provenance(exec_context::mouse_input{})
->execute(cmd);
});
start += al.length();
al.append(":clipboard:"_emoji)
@ -725,8 +742,8 @@ field_overlay_source::list_overlay_menu(const listview_curses& lv,
2_vl,
line_range{start, start + (int) al.length()},
[this](const std::string& value) {
execute_any(*this->fos_lss.get_exec_context(),
"|lnav-copy-text");
this->fos_lss.get_exec_context()->execute(
"|lnav-copy-text");
});
retval.emplace_back(attr_line_t().pad_to(left).append(al));
}

@ -105,7 +105,7 @@ public:
std::vector<attr_line_t> fos_lines;
vis_line_t fos_meta_lines_row{0_vl};
std::vector<attr_line_t> fos_meta_lines;
std::map<size_t, intern_string_t> fos_row_to_field_name;
std::map<size_t, logline_value_meta> fos_row_to_field_meta;
struct menu_item {
menu_item(vis_line_t line,

@ -4868,14 +4868,15 @@ stats.count_by *column*
.. _stats_hist:
stats.hist *col* *\[slice:'5m'\]*
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
stats.hist *col* *\[slice:'1h'\]* *\[top:10\]*
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Count values per bucket of time
Count the top values per bucket of time
**Parameters**
* **col\*** --- The column to count
* **slice** --- The time slice
* **top** --- The limit on the number of values to report
**Examples**
To chart the values of ex_procname over time:
@ -4884,9 +4885,7 @@ stats.hist *col* *\[slice:'5m'\]*
;from lnav_example_log | stats.hist ex_procname
tslice v
2017-02⋯:00.000 {"gw":1,"hw":1}
2017-02⋯:00.000 {"gw":1}
2017-02⋯:00.000 {"gw":1}
2017-02⋯:00.000 {"gw":3,"hw":1}
**See Also**
:ref:`prql_aggregate`, :ref:`prql_append`, :ref:`prql_derive`, :ref:`prql_filter`, :ref:`prql_from`, :ref:`prql_group`, :ref:`prql_join`, :ref:`prql_select`, :ref:`prql_sort`, :ref:`prql_take`, :ref:`stats_average_of`, :ref:`stats_by`, :ref:`stats_count_by`, :ref:`stats_sum_of`, :ref:`utils_distinct`

@ -491,7 +491,7 @@ listview_curses::do_update()
y,
this->vc_x,
ov_menu_line,
lr,
line_range{0, (int) wrap_width},
role_t::VCR_ALT_ROW);
ov_menu_row += 1_vl;
++y;

@ -236,6 +236,35 @@ log_format::opid_descriptors::to_string(
return retval;
}
chart_type_t
logline_value_meta::to_chart_type() const
{
auto retval = chart_type_t::hist;
switch (this->lvm_kind) {
case value_kind_t::VALUE_NULL:
retval = chart_type_t::none;
break;
case value_kind_t::VALUE_INTEGER:
if (!this->lvm_identifier) {
retval = chart_type_t::spectro;
}
break;
case value_kind_t::VALUE_FLOAT:
retval = chart_type_t::spectro;
break;
case value_kind_t::VALUE_XML:
case value_kind_t::VALUE_JSON:
case value_kind_t::VALUE_BOOLEAN:
case value_kind_t::VALUE_TIMESTAMP:
retval = chart_type_t::none;
break;
default:
break;
}
return retval;
}
struct line_range
logline_value::origin_in_full_msg(const char* msg, ssize_t len) const
{

@ -108,6 +108,12 @@ enum class value_kind_t : int {
VALUE__MAX
};
enum class chart_type_t {
none,
hist,
spectro,
};
struct logline_value_meta {
struct internal_column {
bool operator==(const internal_column&) const { return true; }
@ -150,6 +156,8 @@ struct logline_value_meta {
return *this;
}
chart_type_t to_chart_type() const;
intern_string_t lvm_name;
value_kind_t lvm_kind;
column_t lvm_column{external_column{}};

@ -48,6 +48,7 @@
#include "log_accel.hh"
#include "md2attr_line.hh"
#include "ptimec.hh"
#include "shlex.hh"
#include "sql_util.hh"
#include "vtab_module.hh"
#include "yajlpp/yajlpp.hh"
@ -1388,16 +1389,16 @@ logfile_sub_source::list_input_handle_key(listview_curses& lv, int ch)
if (ov_vl) {
auto* fos = dynamic_cast<field_overlay_source*>(
lv.get_overlay_source());
auto iter = fos->fos_row_to_field_name.find(ov_vl.value());
if (iter != fos->fos_row_to_field_name.end()) {
auto iter = fos->fos_row_to_field_meta.find(ov_vl.value());
if (iter != fos->fos_row_to_field_meta.end()) {
auto find_res = this->find_line_with_file(lv.get_top());
if (find_res) {
auto file_and_line = find_res.value();
auto* format = file_and_line.first->get_format_ptr();
auto fstates = format->get_field_states();
auto state_iter = fstates.find(iter->second);
auto state_iter = fstates.find(iter->second.lvm_name);
if (state_iter != fstates.end()) {
format->hide_field(iter->second,
format->hide_field(iter->second.lvm_name,
!state_iter->second.is_hidden());
lv.set_needs_update();
}
@ -1407,6 +1408,44 @@ logfile_sub_source::list_input_handle_key(listview_curses& lv, int ch)
}
return false;
}
case '#': {
auto ov_vl = lv.get_overlay_selection();
if (ov_vl) {
auto* fos = dynamic_cast<field_overlay_source*>(
lv.get_overlay_source());
auto iter = fos->fos_row_to_field_meta.find(ov_vl.value());
if (iter != fos->fos_row_to_field_meta.end()) {
const auto& meta = iter->second;
std::string cmd;
switch (meta.to_chart_type()) {
case chart_type_t::none:
break;
case chart_type_t::hist: {
auto prql = fmt::format(
FMT_STRING(
"from {} | stats.hist {} slice:'1h'"),
meta.lvm_format.value()->get_name(),
meta.lvm_name);
cmd = fmt::format(FMT_STRING(":prompt sql ; '{}'"),
shlex::escape(prql));
break;
}
case chart_type_t::spectro:
cmd = fmt::format(FMT_STRING(":spectrogram {}"),
meta.lvm_name);
break;
}
if (!cmd.empty()) {
this->lss_exec_context
->with_provenance(exec_context::mouse_input{})
->execute(cmd);
}
}
return true;
}
return false;
}
case 'h':
case 'H':
case KEY_SLEFT:
@ -3043,10 +3082,12 @@ logfile_sub_source::text_handle_mouse(
}
}
if (tc.get_overlay_selection()
&& me.is_click_in(mouse_button_t::BUTTON_LEFT, 2, 4))
{
this->list_input_handle_key(tc, ' ');
if (tc.get_overlay_selection()) {
if (me.is_click_in(mouse_button_t::BUTTON_LEFT, 2, 4)) {
this->list_input_handle_key(tc, ' ');
} else if (me.is_click_in(mouse_button_t::BUTTON_LEFT, 5, 6)) {
this->list_input_handle_key(tc, '#');
}
}
return true;
}

@ -18,12 +18,16 @@ let by = func column values rel <relation> -> <relation> (
group {column} (aggregate values)
)
let hist = func column slice:'5m' rel <relation> -> (
let hist = func column slice:'1h' top:10 rel <relation> -> (
rel
group { tslice = (time.slice log_time_msecs slice), column } (
aggregate { total = count this }
)
group { tslice } (
aggregate { v = json.group_object column total }
window (
sort {-total}
take top
aggregate { v = json.group_object column total }
)
)
)

@ -107,6 +107,8 @@ public:
void load();
void set_history();
void save();
void add_possibility(const std::string& type, const std::string& value)

@ -684,6 +684,12 @@ readline_context::readline_context(std::string name,
this->rc_append_character = ' ';
}
void
readline_context::set_history()
{
history_set_history_state(&this->rc_history);
}
void
readline_context::load()
{
@ -1160,6 +1166,11 @@ readline_curses::start()
this->rc_contexts[context]->rem_possibility(
std::string(type), std::string(&msg[prompt_start]));
} else if (sscanf(msg, "ah:%d:%n", &context, &prompt_start)
== 1)
{
this->rc_contexts[context]->set_history();
add_history(&msg[prompt_start]);
} else if (sscanf(msg, "cpre:%d", &context) == 1) {
this->rc_contexts[context]->rc_prefixes.clear();
} else if (sscanf(msg, "cp:%d:%s", &context, type)) {
@ -1292,6 +1303,23 @@ readline_curses::line_ready(const char* line)
}
}
void
readline_curses::append_to_history(int context, const std::string& line)
{
if (line.empty()) {
return;
}
char buffer[2048];
snprintf(buffer, sizeof(buffer), "ah:%d:%s", context, line.c_str());
if (sendstring(
this->rc_command_pipe[RCF_MASTER], buffer, strlen(buffer) + 1)
== -1)
{
perror("add_possibility: write failed");
}
}
void
readline_curses::check_poll_set(const std::vector<struct pollfd>& pollfds)
{

@ -173,6 +173,8 @@ public:
void set_suggestion(const std::string& value);
bool is_active() const { return this->rc_active_context != -1; }
readline_context* get_active_context() const
{
require(this->rc_active_context != -1);
@ -276,6 +278,8 @@ public:
this->clear_possibilities(lnav::enums::to_underlying(context), args...);
}
void append_to_history(int context, const std::string& line);
const std::vector<std::string>& get_matches() const
{
return this->rc_matches;

@ -670,13 +670,17 @@ static readline_context::command_t sql_commands[] = {
{
"stats.hist",
prql_cmd_sort,
help_text("stats.hist", "Count values per bucket of time")
help_text("stats.hist", "Count the top values per bucket of time")
.prql_function()
.with_tags({"prql"})
.with_parameter(help_text{"col", "The column to count"})
.with_parameter(help_text{"slice", "The time slice"}
.optional()
.with_default_value("'5m'"))
.with_default_value("'1h'"))
.with_parameter(
help_text{"top", "The limit on the number of values to report"}
.optional()
.with_default_value("10"))
.with_example({
"To chart the values of ex_procname over time",
"from lnav_example_log | stats.hist ex_procname",

@ -417,10 +417,6 @@ textview_curses::handle_mouse(mouse_event& me)
return true;
}
if (me.me_button != mouse_button_t::BUTTON_LEFT) {
return false;
}
auto mouse_line = (me.me_y < 0 || me.me_y >= this->lv_display_lines.size())
? empty_space{}
: this->lv_display_lines[me.me_y];
@ -472,6 +468,7 @@ textview_curses::handle_mouse(mouse_event& me)
[this, &me, sub_delegate, &mouse_line](const main_content& mc) {
if (this->vc_enabled) {
if (this->tc_supports_marks
&& me.me_button == mouse_button_t::BUTTON_LEFT
&& me.is_modifier_pressed(
mouse_event::modifier_t::shift))
{
@ -506,7 +503,9 @@ textview_curses::handle_mouse(mouse_event& me)
mouse_line.match(
[this, &me, &mouse_line, sub_delegate](const main_content& mc) {
if (this->vc_enabled) {
if (this->tc_supports_marks) {
if (this->tc_supports_marks
&& me.me_button == mouse_button_t::BUTTON_LEFT)
{
attr_line_t al;
this->textview_value_for_row(mc.mc_line, al);
@ -525,7 +524,9 @@ textview_curses::handle_mouse(mouse_event& me)
auto tok = tok_res.value();
auto tok_sf = tok.inner_string_fragment();
if (tok_sf.contains(cursor_sf)) {
if (tok_sf.contains(cursor_sf)
&& tok.tr_token != data_token_t::DT_WHITE)
{
this->tc_selected_text = selected_text_info{
me.me_x,
mc.mc_line,
@ -575,19 +576,22 @@ textview_curses::handle_mouse(mouse_event& me)
this->lv_left + me.me_press_x);
this->set_selection_without_context(mc.mc_line);
this->textview_value_for_row(mc.mc_line, al);
auto line_sf = string_fragment::from_str(al.get_string());
auto cursor_sf = line_sf.sub_cell_range(low_x, high_x);
if (!cursor_sf.empty()) {
this->tc_selected_text = {
me.me_press_x,
mc.mc_line,
line_range{
cursor_sf.sf_begin,
cursor_sf.sf_end,
},
cursor_sf.to_string(),
};
if (me.me_button == mouse_button_t::BUTTON_LEFT) {
this->textview_value_for_row(mc.mc_line, al);
auto line_sf
= string_fragment::from_str(al.get_string());
auto cursor_sf = line_sf.sub_cell_range(low_x, high_x);
if (!cursor_sf.empty()) {
this->tc_selected_text = {
me.me_press_x,
mc.mc_line,
line_range{
cursor_sf.sf_begin,
cursor_sf.sf_end,
},
cursor_sf.to_string(),
};
}
}
}
} else {
@ -611,7 +615,8 @@ textview_curses::handle_mouse(mouse_event& me)
- 1_vl);
} else if (me.me_y >= overlay_content_max_y.value()) {
this->set_overlay_selection(
this->get_overlay_selection().value() + 1_vl);
this->get_overlay_selection().value_or(0_vl)
+ 1_vl);
} else if (mouse_line.is<overlay_content>()) {
this->set_overlay_selection(
mouse_line.get<overlay_content>().oc_line);
@ -622,6 +627,13 @@ textview_curses::handle_mouse(mouse_event& me)
}
case mouse_button_state_t::BUTTON_STATE_RELEASED: {
this->tc_text_selection_active = false;
if (me.is_click_in(mouse_button_t::BUTTON_RIGHT, 0, INT_MAX)) {
auto* lov = this->get_overlay_source();
if (lov != nullptr) {
lov->set_show_details_in_overlay(
!lov->get_show_details_in_overlay());
}
}
if (this->vc_enabled) {
if (this->tc_selection_start) {
this->toggle_user_mark(&BM_USER,

@ -1532,6 +1532,19 @@ lnav_behavior::mouse_event(int button, bool release, int x, int y)
me.me_press_y = me.me_y - tc->get_y();
me.me_press_x = me.me_x - tc->get_x();
this->lb_last_view = tc;
switch (lnav_data.ld_mode) {
case ln_mode_t::PAGING:
break;
case ln_mode_t::FILES:
case ln_mode_t::FILTER:
// Clicking on the main view when the config panels are
// open should return us to paging.
set_view_mode(ln_mode_t::PAGING);
break;
default:
break;
}
} else {
for (auto* vc : VIEWS) {
if (vc->contains(me.me_x, me.me_y)) {

@ -1,11 +1,11 @@
2023-03-24T14:26:16.243Z renovate[7] INFO Dependency extraction complete
Received Time: 2023-03-24T14:26:16.243 — in the future Format: %Y-%m-%dT%H:%M:%S.%L%z
Known message fields for table bunyan_log:
| ◆ name = renovate
| ◇ hostname = renovate-gitlab-67c4bcb5-9ggbv
| ◆ pid = 7
| ◇ level = 30
| ◇ v = 0
| ◆ 📊 name = renovate
| ◇ 📊 hostname = renovate-gitlab-67c4bcb5-9ggbv
| ◆ 📊 pid = 7
| ◇ 📊 level = 30
| ◇ 📊 v = 0
JSON fields:
jget(log_raw_text, '/baseBranch') = main
jget(log_raw_text, '/logContext') = qjifsaDDI

@ -2,9 +2,9 @@
Received Time: 2020-12-10T06:56:41.092 — in the future Format: %Y-%m-%d %H:%M:%S,%L
Pattern: /xml_msg_log/regex/std = ^\[(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]\s+(?<level>\w+)\s+\[(?<module>[^:]*):(?<line>\d+)\]\s*(?<body>[^\n]*)\n?(?<msg_data>.*)
Known message fields for table xml_msg_log:
| ◆ module = connect.client
| ◆ line = 69
| ◆ msg_data = <?xml version='1.0' encoding='iso-8859-2'?>␊<a-request>␊ <head>␊ x␊ </head>␊ <source>␊ x␊ </source>␊ <request id="1">␊ <name>␊ x␊ </name>␊ </request>␊</a-request>␊
| ◆ 📊 module = connect.client
| ◆ 📊 line = 69
| ◆ msg_data = <?xml version='1.0' encoding='iso-8859-2'?>␊<a-request>␊ <head>␊ x␊ </head>␊ <source>␊ x␊ </source>␊ <request id="1">␊ <name>␊ x␊ </name>␊ </request>␊</a-request>␊
XML fields:
xpath('/a-request/head/text()', xml_msg_log.msg_data) = x
xpath('/a-request/request/@id', xml_msg_log.msg_data) = 1

Loading…
Cancel
Save