[format] move foreign_key flag to value meta

Fix some chart issues as well.
master
Tim Stack 4 weeks ago
parent e805ca8a8a
commit f69878eaf5

@ -101,6 +101,7 @@ Bug Fixes:
* A crash during initialization on Apple Silicon and MacOS 12
has been fixed.
* A crash when previewing non-text files.
* Optimized ANSI-escape processing.
* Various fixes to make lnav usable as a `PAGER`.
## lnav v0.12.1

@ -286,6 +286,7 @@ db_label_source::push_column(const scoped_value_t& sv)
}
}
}
hm.hm_chart.next_row();
}
void

@ -359,6 +359,7 @@ files_sub_source::text_value_for_line(textview_curses& tc,
.append(start_time)
.append(" \u2014 ")
.append(end_time)
.append(" ")
.appendf(FMT_STRING("{}"), fmt::join(file_notes, "; "));
if (selected) {
al.with_attr_for_all(VC_ROLE.value(role_t::VCR_FOCUSED));

@ -67,9 +67,13 @@ hist_source2::text_value_for_line(textview_curses& tc,
std::string& value_out,
text_sub_source::line_flags_t flags)
{
bucket_t& bucket = this->find_bucket(row);
auto& bucket = this->find_bucket(row);
struct tm bucket_tm;
if (this->hs_needs_flush) {
this->end_of_row();
}
value_out.clear();
if (gmtime_r(&bucket.b_time, &bucket_tm) != nullptr) {
fmt::format_to(std::back_inserter(value_out),
@ -130,12 +134,14 @@ hist_source2::add_value(time_t row,
auto& bucket = this->find_bucket(this->hs_last_bucket);
bucket.b_time = row;
bucket.b_values[htype].hv_value += value;
this->hs_needs_flush = true;
}
void
hist_source2::init()
{
view_colors& vc = view_colors::singleton();
auto& vc = view_colors::singleton();
this->hs_chart.with_show_state(stacked_bar_chart_base::show_all{})
.with_attrs_for_ident(HT_NORMAL, vc.attrs_for_role(role_t::VCR_TEXT))
@ -160,12 +166,13 @@ void
hist_source2::end_of_row()
{
if (this->hs_last_bucket >= 0) {
bucket_t& last_bucket = this->find_bucket(this->hs_last_bucket);
auto& last_bucket = this->find_bucket(this->hs_last_bucket);
for (int lpc = 0; lpc < HT__MAX; lpc++) {
this->hs_chart.add_value((const hist_type_t) lpc,
last_bucket.b_values[lpc].hv_value);
}
this->hs_chart.next_row();
}
}

@ -204,18 +204,23 @@ public:
if (this->sbc_show_state.template is<show_all>()
|| lpc == (size_t) ident_to_show)
{
overall_stats.merge(this->sbc_idents[lpc].ci_stats,
this->sbc_do_stacking);
overall_stats.merge(this->sbc_idents[lpc].ci_stats);
}
}
if (this->sbc_max_row_value > overall_stats.bs_max_value) {
overall_stats.bs_max_value = this->sbc_max_row_value;
}
if (this->sbc_row_sum > overall_stats.bs_max_value) {
overall_stats.bs_max_value = this->sbc_row_sum;
}
if (this->sbc_show_state.template is<show_all>()) {
if (this->sbc_idents.size() == 1) {
avail_width = width;
} else if (width < this->sbc_idents.size()) {
} else if (width < this->sbc_max_row_items) {
avail_width = 0;
} else {
avail_width = width - this->sbc_idents.size();
avail_width = width;
}
} else {
avail_width = width - 1;
@ -256,12 +261,34 @@ public:
this->sbc_idents.clear();
this->sbc_ident_lookup.clear();
this->sbc_show_state = show_none();
this->sbc_row_sum = 0;
this->sbc_row_items = 0;
this->sbc_max_row_value = 0;
this->sbc_max_row_items = 0;
}
void add_value(const T& ident, double amount = 1.0)
{
struct chart_ident& ci = this->find_ident(ident);
ci.ci_stats.update(amount);
this->sbc_row_sum += amount;
if (ci.ci_last_seen_row != this->sbc_row_counter) {
ci.ci_last_seen_row = this->sbc_row_counter;
this->sbc_row_items += 1;
}
}
void next_row()
{
if (this->sbc_row_sum > this->sbc_max_row_value) {
this->sbc_max_row_value = this->sbc_row_sum;
}
if (this->sbc_row_items > this->sbc_max_row_items) {
this->sbc_max_row_items = this->sbc_row_items;
}
this->sbc_row_sum = 0;
this->sbc_row_items = 0;
this->sbc_row_counter += 1;
}
struct bucket_stats_t {
@ -270,15 +297,10 @@ public:
{
}
void merge(const bucket_stats_t& rhs, bool do_stacking)
void merge(const bucket_stats_t& rhs)
{
this->bs_min_value = std::min(this->bs_min_value, rhs.bs_min_value);
if (do_stacking) {
this->bs_max_value += rhs.bs_max_value;
} else {
this->bs_max_value
= std::max(this->bs_max_value, rhs.bs_max_value);
}
this->bs_max_value = std::max(this->bs_max_value, rhs.bs_max_value);
}
double width() const
@ -310,6 +332,7 @@ protected:
T ci_ident;
text_attrs ci_attrs;
bucket_stats_t ci_stats;
ssize_t ci_last_seen_row{-1};
};
struct chart_ident& find_ident(const T& ident)
@ -328,6 +351,12 @@ protected:
std::vector<struct chart_ident> sbc_idents;
std::unordered_map<T, unsigned int> sbc_ident_lookup;
show_state sbc_show_state{show_none()};
ssize_t sbc_row_counter{0};
double sbc_row_sum{0};
size_t sbc_row_items{0};
double sbc_max_row_value{0};
size_t sbc_max_row_items{0};
};
class hist_source2
@ -417,6 +446,7 @@ private:
time_t hs_last_row;
std::map<int64_t, struct bucket_block> hs_blocks;
stacked_bar_chart<hist_type_t> hs_chart;
bool hs_needs_flush{false};
};
#endif

@ -267,6 +267,8 @@ listview_curses::handle_key(int ch)
if (this->lv_selection <= top_for_last) {
this->set_selection(top_for_last + 1_vl);
}
} else if (this->lv_top > top_for_last) {
this->set_top(top_for_last);
} else {
this->shift_top(rows_avail);

@ -245,7 +245,7 @@ logline_value_meta::to_chart_type() const
retval = chart_type_t::none;
break;
case value_kind_t::VALUE_INTEGER:
if (!this->lvm_identifier) {
if (!this->lvm_identifier && !this->lvm_foreign_key) {
retval = chart_type_t::spectro;
}
break;
@ -2843,7 +2843,7 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
auto& ivd = pat.p_value_by_index[lpc];
auto vd = ivd.ivd_value_def;
if (!vd->vd_foreign_key && !vd->vd_meta.lvm_identifier) {
if (!vd->vd_meta.lvm_foreign_key && !vd->vd_meta.lvm_identifier) {
switch (vd->vd_meta.lvm_kind) {
case value_kind_t::VALUE_INTEGER:
case value_kind_t::VALUE_FLOAT:
@ -3606,7 +3606,7 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
elf_value_def->vd_meta.lvm_values_index
= nonstd::make_optional(value_def_index++);
if (elf_value_def->vd_foreign_key
if (elf_value_def->vd_meta.lvm_foreign_key
|| elf_value_def->vd_meta.lvm_identifier)
{
continue;
@ -3856,7 +3856,7 @@ public:
log_vtab_impl::get_foreign_keys(keys_inout);
for (const auto& elf_value_def : this->elt_format.elf_value_defs) {
if (elf_value_def.second->vd_foreign_key) {
if (elf_value_def.second->vd_meta.lvm_foreign_key) {
keys_inout.emplace_back(elf_value_def.first.to_string());
}
}

@ -163,6 +163,7 @@ struct logline_value_meta {
column_t lvm_column{external_column{}};
nonstd::optional<size_t> lvm_values_index;
bool lvm_identifier{false};
bool lvm_foreign_key{false};
bool lvm_hidden{false};
nonstd::optional<bool> lvm_user_hidden;
bool lvm_from_module{false};

@ -68,7 +68,6 @@ public:
logline_value_meta vd_meta;
std::string vd_collate;
bool vd_foreign_key{false};
intern_string_t vd_unit_field;
std::map<const intern_string_t, scaling_factor> vd_unit_scaling;
bool vd_internal{false};

@ -489,10 +489,12 @@ public:
field_def& with_kind(value_kind_t kind,
bool identifier = false,
bool foreign_key = false,
const std::string& collator = "")
{
this->fd_meta.lvm_kind = kind;
this->fd_meta.lvm_identifier = identifier;
this->fd_meta.lvm_foreign_key = foreign_key;
this->fd_collator = collator;
return *this;
}
@ -768,12 +770,14 @@ public:
"bro_referrer",
"bro_resp_fuids",
"bro_service",
"bro_status_code",
"bro_uid",
"bro_uri",
"bro_user_agent",
"bro_username",
};
static const char* KNOWN_FOREIGN[] = {
"bro_status_code",
};
int numeric_count = 0;
@ -792,7 +796,12 @@ public:
bool ident = std::binary_search(std::begin(KNOWN_IDS),
std::end(KNOWN_IDS),
fd.fd_meta.lvm_name);
fd.with_kind(value_kind_t::VALUE_INTEGER, ident)
bool foreign
= std::binary_search(std::begin(KNOWN_FOREIGN),
std::end(KNOWN_FOREIGN),
fd.fd_meta.lvm_name);
fd.with_kind(
value_kind_t::VALUE_INTEGER, ident, foreign)
.with_numeric_index(numeric_count);
numeric_count += 1;
} else if (field_type == "bool") {
@ -950,7 +959,7 @@ public:
this->log_vtab_impl::get_foreign_keys(keys_inout);
for (const auto& fd : this->blt_format.blf_field_defs) {
if (fd.fd_meta.lvm_identifier) {
if (fd.fd_meta.lvm_identifier || fd.fd_meta.lvm_foreign_key) {
keys_inout.push_back(fd.fd_meta.lvm_name.to_string());
}
}
@ -1124,6 +1133,7 @@ public:
const char* name,
value_kind_t kind,
bool ident = false,
bool foreign_key = false,
std::string coll = "")
: fd_name(intern_string::lookup(name)),
fd_meta(
@ -1133,6 +1143,7 @@ public:
fd_collator(std::move(coll))
{
this->fd_meta.lvm_identifier = ident;
this->fd_meta.lvm_foreign_key = foreign_key;
}
field_def& with_kind(value_kind_t kind,
@ -1625,7 +1636,7 @@ public:
this->log_vtab_impl::get_foreign_keys(keys_inout);
for (const auto& fd : KNOWN_FIELDS) {
if (fd.fd_meta.lvm_identifier) {
if (fd.fd_meta.lvm_identifier || fd.fd_meta.lvm_foreign_key) {
keys_inout.push_back(fd.fd_meta.lvm_name.to_string());
}
}
@ -1687,6 +1698,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
"c-ip",
value_kind_t::VALUE_TEXT,
true,
false,
"ipaddress",
},
{
@ -1706,6 +1718,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
"cs-uri-stem",
value_kind_t::VALUE_TEXT,
true,
false,
"naturalnocase",
},
{
@ -1731,6 +1744,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
"s-ip",
value_kind_t::VALUE_TEXT,
true,
false,
"ipaddress",
},
{
@ -1762,6 +1776,7 @@ const std::vector<w3c_log_format::field_def> w3c_log_format::KNOWN_FIELDS = {
"sc-status",
value_kind_t::VALUE_INTEGER,
false,
true,
},
{
KNOWN_FIELD_INDEX++,

@ -456,7 +456,8 @@ static const json_path_handler_base::enum_value_t OVERFLOW_ENUM[] = {
{"truncate",
external_log_format::json_format_element::overflow_t::TRUNCATE},
{"dot-dot", external_log_format::json_format_element::overflow_t::DOTDOT},
{"last-word", external_log_format::json_format_element::overflow_t::LASTWORD},
{"last-word",
external_log_format::json_format_element::overflow_t::LASTWORD},
json_path_handler_base::ENUM_TERMINATOR,
};
@ -616,7 +617,8 @@ static const struct json_path_container value_def_handlers = {
.with_synopsis("<bool>")
.with_description("Indicates whether or not this field should be "
"treated as a foreign key for row in another table")
.for_field(&external_log_format::value_def::vd_foreign_key),
.for_field(&external_log_format::value_def::vd_meta,
&logline_value_meta::lvm_foreign_key),
yajlpp::property_handler("hidden")
.with_synopsis("<bool>")

@ -25,9 +25,24 @@ let hist = func column slice:'1h' top:10 rel <relation> -> (
)
group { tslice } (
window (
sort {-total}
take top
aggregate { v = json.group_object column total }
sort {-total}
derive {rn = row_number column}
)
)
derive top_value = case [
rn < top => column,
rn >= top => 'Other',
]
group { tslice, top_value } (
aggregate { total2 = sum total }
)
group { tslice } (
window (
sort {-total2}
# XXX The `take` here is necessary to workaround a
# PRQL issue where the above sort is dropped entirely
take top
aggregate { v = json.group_object top_value total2 }
)
)
)

@ -595,7 +595,9 @@ textview_curses::handle_mouse(mouse_event& me)
(int) this->lv_left + me.me_x);
this->set_selection_without_context(mc.mc_line);
if (me.me_button == mouse_button_t::BUTTON_LEFT) {
if (this->tc_supports_marks
&& me.me_button == mouse_button_t::BUTTON_LEFT)
{
this->textview_value_for_row(mc.mc_line, al);
auto line_sf
= string_fragment::from_str(al.get_string());

@ -474,10 +474,8 @@ EXPECTED_FILES = \
$(srcdir)/%reldir%/test_prql.sh_06900fac5c2e854b1208320b753fcd43d4ba63a3.out \
$(srcdir)/%reldir%/test_prql.sh_15ebcd913f56cde35af26c6300533d6e76bf1b55.err \
$(srcdir)/%reldir%/test_prql.sh_15ebcd913f56cde35af26c6300533d6e76bf1b55.out \
$(srcdir)/%reldir%/test_prql.sh_451e242cdfa2db9005d4fe752a7b05d1ab5cba29.err \
$(srcdir)/%reldir%/test_prql.sh_451e242cdfa2db9005d4fe752a7b05d1ab5cba29.out \
$(srcdir)/%reldir%/test_prql.sh_45d57a042092ffdcd28ea35a892f02859e78f33d.err \
$(srcdir)/%reldir%/test_prql.sh_45d57a042092ffdcd28ea35a892f02859e78f33d.out \
$(srcdir)/%reldir%/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.err \
$(srcdir)/%reldir%/test_prql.sh_5aea925b5ad95f55ce6b1b07b7046bc3d7310137.out \
$(srcdir)/%reldir%/test_regex101.sh_0fa3663a45aca6a328cb728872af7ed7ee896f1c.err \
$(srcdir)/%reldir%/test_regex101.sh_0fa3663a45aca6a328cb728872af7ed7ee896f1c.out \
$(srcdir)/%reldir%/test_regex101.sh_182ae9244db314a953af2bee969726e381bc5a32.err \

@ -1,4 +1,4 @@
 Thu Nov 03 09:20:00  1 normal 2 errors 0 warnings  0 marks
 Thu Nov 03 09:45:00  1 normal 0 errors 0 warnings 0 marks
 Fri Feb 03 09:20:00  0 normal 1 errors 0 warnings 0 marks
 Wed Jan 03 09:20:00  1 normal 0 errors 0 warnings 0 marks
 Thu Nov 03 09:20:00  1 normal 2 errors 0 warnings  0 marks
 Thu Nov 03 09:45:00  1 normal 0 errors 0 warnings 0 marks
 Fri Feb 03 09:20:00  0 normal 1 errors 0 warnings 0 marks
 Wed Jan 03 09:20:00  1 normal 0 errors 0 warnings 0 marks

@ -1 +1 @@
 Sat Nov 03 08:00:00 1 normal 0 errors 0 warnings 0 marks
 Sat Nov 03 08:00:00 1 normal 0 errors 0 warnings  0 marks

@ -1,2 +1,2 @@
 Sat Nov 03 09:20:00 1 normal 2 errors  0 warnings  1 marks
 Sat Nov 03 09:45:00 1 normal 0 errors 0 warnings 0 marks
 Sat Nov 03 09:20:00 1 normal 2 errors  0 warnings  1 marks
 Sat Nov 03 09:45:00 1 normal 0 errors 0 warnings 0 marks

@ -1 +1 @@
 Sat Nov 03 00:00:00 2 normal 2 errors 0 warnings 0 marks
 Sat Nov 03 00:00:00 2 normal  2 errors 0 warnings  0 marks

@ -1 +1 @@
 Sat Nov 03 08:00:00 2 normal 2 errors 0 warnings 0 marks
 Sat Nov 03 08:00:00 2 normal  2 errors 0 warnings  0 marks

@ -0,0 +1,4 @@
 tslice   v  
2011-11-03 00:19:00.000 {"a.fsdn.com":33,"Other":31,"ad.doubleclick.net":10,"www.goo⋯"\xaf"dia.com":6,"s0.2mdn.net":5,"googleads.g.doubleclick.net":3}
2011-11-03 00:20:00.000 {"www.bro-ids.org":24,"www.google.com":15,"search.twitter.c⋯"\x8b""\xaf"ds.org":2,"ad.doubleclick.net":1,"cont-sjl-1.pandora.com":1}
2011-11-03 00:21:00.000 {"www.google.com":2,"ajax.googleapis.com":1,"api.twitter.com":1,"search.twitter.com":1,"www.bro-ids.org":1} 

@ -5,6 +5,6 @@
 * FEATURES: network-sandbox preserve-libs sandbox test userpriv usersandbox
* Checking whether python3_11 is suitable ...
* >=dev-lang/python-3.11.1-r1:3.11 ...
[ ok ]
[ ok ]
* Using python3.11 to build (via PYTHON_COMPAT iteration)
>>> Unpacking source...

@ -11,3 +11,7 @@ run_cap_test ${lnav_test} -n \
run_cap_test ${lnav_test} -n \
-c ";from access_log | take abc" \
${test_dir}/logfile_access_log.0
run_cap_test ${lnav_test} -n \
-c ";from bro_http_log | stats.hist bro_host slice:'1m'" \
${test_dir}/logfile_bro_http.log.0

Loading…
Cancel
Save