Archives/lnav
2
0
Fork 0
mirror of https://github.com/tstack/lnav synced 2024-11-01 21:40:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

[format] allow json log fields to be hidden

Browse Source 
Fixes #303
This commit is contained in:
Timothy Stack 2016-10-29 06:52:12 -07:00
parent b392886f0c
commit a59e0b290e
9 changed files with 206 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
NEWS
View File

@ -6,6 +6,8 @@ lnav v0.8.2:
* Added "min-width", "max-width", "align", and "overflow" options to the
"line-format" in format definitions for JSON log files. These options
give you more control over how the displayed line looks.
* Added a "hidden" option to log format values so that you can hide JSON
log fields from being displayed if they are not in the line format.
Interface Changes:
* The color used for text colored via ":highlight" is now based on the

5
docs/source/formats.rst
View File

@ -131,6 +131,9 @@ fields:
determine the "sub" format. This module identifier is used to help
**lnav** quickly identify the format to use when parsing message bodies.
:hide-extra: A boolean for JSON logs that indicates whether fields not
present in the line-format should be displayed on their own lines.
:level: A mapping of error levels to regular expressions. During scanning
the contents of the capture group specified by *level-field* will be
checked against each of these regexes. Once a match is found, the log
@ -153,6 +156,8 @@ fields:
an identifier and should be syntax colored.
:foreign-key: A boolean that indicates that this field is a key and should
not be graphed. This should only need to be set for integer fields.
:hidden: A boolean for JSON log fields that indicates whether they should
be displayed if they are not present in the line-format.
:sample: A list of objects that contain sample log messages. All formats
must include at least one sample and it must be matched by one of the

3
src/default-log-formats.json
View File

@ -771,6 +771,9 @@
"timestamp-field" : "generated_at",
"body-field" : "message",
"value" : {
"display_received_at" : {
"kind" : "string"
},
"program" : {
"kind" : "string",
"identifier" : true

160
src/log_format.cc
View File

@ -358,39 +358,14 @@ struct json_log_userdata {
shared_buffer_ref &jlu_shared_buffer;
};
struct json_field_cmp {
json_field_cmp(external_log_format::json_log_field type,
const intern_string_t name)
: jfc_type(type), jfc_field_name(name) {
};
bool operator()(const external_log_format::json_format_element &jfe) const {
return (this->jfc_type == jfe.jfe_type &&
this->jfc_field_name == jfe.jfe_value);
};
external_log_format::json_log_field jfc_type;
const intern_string_t jfc_field_name;
};
static int read_json_field(yajlpp_parse_context *ypc, const unsigned char *str, size_t len);
static int read_json_null(yajlpp_parse_context *ypc)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
vector<external_log_format::json_format_element> &line_format =
jlu->jlu_format->jlf_line_format;
const intern_string_t field_name = ypc->get_path();
if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) {
return 1;
}
if (!jlu->jlu_format->jlf_hide_extra &&
find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
}
jlu->jlu_sub_line_count += jlu->jlu_format->value_line_count(field_name);
return 1;
}
@ -398,19 +373,9 @@ static int read_json_null(yajlpp_parse_context *ypc)
static int read_json_bool(yajlpp_parse_context *ypc, int val)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
vector<external_log_format::json_format_element> &line_format =
jlu->jlu_format->jlf_line_format;
const intern_string_t field_name = ypc->get_path();
if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) {
return 1;
}
if (!jlu->jlu_format->jlf_hide_extra &&
find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
}
jlu->jlu_sub_line_count += jlu->jlu_format->value_line_count(field_name);
return 1;
}
@ -418,13 +383,8 @@ static int read_json_bool(yajlpp_parse_context *ypc, int val)
static int read_json_int(yajlpp_parse_context *ypc, long long val)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
vector<external_log_format::json_format_element> &line_format =
jlu->jlu_format->jlf_line_format;
const intern_string_t field_name = ypc->get_path();
if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) {
return 1;
}
if (jlu->jlu_format->lf_timestamp_field == field_name) {
long long divisor = jlu->jlu_format->elf_timestamp_divisor;
struct timeval tv;
@ -445,12 +405,8 @@ static int read_json_int(yajlpp_parse_context *ypc, long long val)
}
}
}
else if (!jlu->jlu_format->jlf_hide_extra &&
find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
}
jlu->jlu_sub_line_count += jlu->jlu_format->value_line_count(field_name);
return 1;
}
@ -458,13 +414,8 @@ static int read_json_int(yajlpp_parse_context *ypc, long long val)
static int read_json_double(yajlpp_parse_context *ypc, double val)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
vector<external_log_format::json_format_element> &line_format =
jlu->jlu_format->jlf_line_format;
const intern_string_t field_name = ypc->get_path();
if (!ypc->is_level(1) && !jlu->jlu_format->has_value_def(field_name)) {
return 1;
}
if (jlu->jlu_format->lf_timestamp_field == field_name) {
double divisor = jlu->jlu_format->elf_timestamp_divisor;
struct timeval tv;
@ -473,12 +424,8 @@ static int read_json_double(yajlpp_parse_context *ypc, double val)
tv.tv_usec = fmod(val, divisor) * (1000000.0 / divisor);
jlu->jlu_base_line->set_time(tv);
}
else if (!jlu->jlu_format->jlf_hide_extra &&
find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
}
jlu->jlu_sub_line_count += jlu->jlu_format->value_line_count(field_name);
return 1;
}
@ -495,8 +442,9 @@ static int json_array_start(void *ctx)
if (!jlu->jlu_format->jlf_hide_extra &&
find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
external_log_format::json_field_cmp(
external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
}
@ -974,8 +922,6 @@ void external_log_format::annotate(shared_buffer_ref &line,
static int read_json_field(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
vector<external_log_format::json_format_element> &line_format =
jlu->jlu_format->jlf_line_format;
const intern_string_t field_name = ypc->get_path();
struct exttm tm_out;
struct timeval tv_out;
@ -992,17 +938,8 @@ static int read_json_field(yajlpp_parse_context *ypc, const unsigned char *str,
uint8_t opid = hash_str((const char *) str, len);
jlu->jlu_base_line->set_opid(opid);
}
else if (ypc->is_level(1) || jlu->jlu_format->elf_value_defs.find(field_name) !=
jlu->jlu_format->elf_value_defs.end()) {
if (!jlu->jlu_format->jlf_hide_extra) {
if (find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
}
jlu->jlu_sub_line_count += std::count(&str[0], &str[len], '\n');
}
}
jlu->jlu_sub_line_count += jlu->jlu_format->value_line_count(field_name, str, len);
return 1;
}
@ -1118,6 +1055,7 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
if (vd_iter != this->elf_value_defs.end()) {
lv_iter->lv_identifier = vd_iter->second.vd_identifier;
lv_iter->lv_column = vd_iter->second.vd_column;
lv_iter->lv_hidden = vd_iter->second.vd_hidden;
}
}
@ -1243,10 +1181,10 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
"body", -1);
logline_value &lv = this->jlf_line_values[lpc];
if (used_values[lpc] ||
if (lv.lv_hidden ||
used_values[lpc] ||
lv.lv_name == this->lf_timestamp_field ||
lv.lv_name == body_name ||
lv.lv_name == this->elf_level_field) {
lv.lv_name == body_name) {
continue;
}
@ -1297,7 +1235,12 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
if (this->jlf_cached_line[this_off] == '\n') {
this_off += 1;
}
next_off = this->jlf_line_offsets[ll.get_sub_offset() + 1];
if ((ll.get_sub_offset() + 1) < this->jlf_line_offsets.size()) {
next_off = this->jlf_line_offsets[ll.get_sub_offset() + 1];
}
else {
next_off = this_off;
}
}
if (full_message) {
@ -1313,6 +1256,26 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
}
void external_log_format::build(std::vector<std::string> &errors) {
if (!this->lf_timestamp_field.empty()) {
value_def &vd = this->elf_value_defs[this->lf_timestamp_field];
vd.vd_name = this->lf_timestamp_field;
vd.vd_kind = logline_value::VALUE_TEXT;
vd.vd_internal = true;
}
if (!this->elf_level_field.empty() && this->elf_value_defs.
find(this->elf_level_field) == this->elf_value_defs.end()) {
value_def &vd = this->elf_value_defs[this->elf_level_field];
vd.vd_name = this->elf_level_field;
vd.vd_kind = logline_value::VALUE_TEXT;
vd.vd_internal = true;
}
if (!this->elf_body_field.empty()) {
value_def &vd = this->elf_value_defs[this->elf_body_field];
vd.vd_name = this->elf_body_field;
vd.vd_kind = logline_value::VALUE_TEXT;
vd.vd_internal = true;
}
if (!this->lf_timestamp_format.empty()) {
this->lf_timestamp_format.push_back(NULL);
}
@ -1388,7 +1351,7 @@ void external_log_format::build(std::vector<std::string> &errors) {
else {
vd.vd_unit_field_index = -1;
}
if (vd.vd_column == -1) {
if (!vd.vd_internal && vd.vd_column == -1) {
vd.vd_column = this->elf_column_count++;
}
pat.p_value_by_index.push_back(vd);
@ -1462,10 +1425,14 @@ void external_log_format::build(std::vector<std::string> &errors) {
++iter) {
std::vector<std::string>::iterator act_iter;
if (iter->second.vd_column == -1) {
if (!iter->second.vd_internal && iter->second.vd_column == -1) {
iter->second.vd_column = this->elf_column_count++;
}
if (iter->second.vd_kind == logline_value::VALUE_UNKNOWN) {
iter->second.vd_kind = logline_value::VALUE_TEXT;
}
for (act_iter = iter->second.vd_action_list.begin();
act_iter != iter->second.vd_action_list.end();
++act_iter) {
@ -1621,13 +1588,36 @@ void external_log_format::build(std::vector<std::string> &errors) {
this->lf_value_stats.resize(this->elf_numeric_value_defs.size());
int format_index = 0;
for (vector<json_format_element>::iterator iter = this->jlf_line_format.begin();
iter != this->jlf_line_format.end();
++iter) {
++iter, format_index++) {
static const intern_string_t ts = intern_string::lookup("__timestamp__");
map<const intern_string_t, value_def>::iterator vd_iter;
json_format_element &jfe = *iter;
if (jfe.jfe_value.empty() && !jfe.jfe_ts_format.empty()) {
jfe.jfe_value = intern_string::lookup("__timestamp__");
jfe.jfe_value = ts;
}
switch (jfe.jfe_type) {
case JLF_VARIABLE:
vd_iter = this->elf_value_defs.find(jfe.jfe_value);
if (jfe.jfe_value != ts &&
vd_iter == this->elf_value_defs.end()) {
char index_str[32];
snprintf(index_str, sizeof(index_str), "%d", format_index);
errors.push_back("error:" +
this->elf_name.to_string() +
":line_format[" +
index_str +
"]:line format variable is not defined -- " +
jfe.jfe_value.to_string());
}
break;
default:
break;
}
}
}
@ -1705,13 +1695,17 @@ public:
std::map<const intern_string_t, external_log_format::value_def>::const_iterator iter;
const external_log_format &elf = this->elt_format;
cols.resize(elf.elf_value_defs.size());
cols.resize(elf.elf_column_count);
for (iter = elf.elf_value_defs.begin();
iter != elf.elf_value_defs.end();
++iter) {
const external_log_format::value_def &vd = iter->second;
int type = 0;
if (vd.vd_column == -1) {
continue;
}
switch (vd.vd_kind) {
case logline_value::VALUE_NULL:
case logline_value::VALUE_TEXT:

136
src/log_format.hh
View File

@ -362,28 +362,29 @@ public:
logline_value(const intern_string_t name)
: lv_name(name), lv_kind(VALUE_NULL), lv_identifier(), lv_column(-1),
lv_from_module(false), lv_format(NULL) { };
lv_hidden(false), lv_from_module(false), lv_format(NULL) { };
logline_value(const intern_string_t name, bool b)
: lv_name(name),
lv_kind(VALUE_BOOLEAN),
lv_value((int64_t)(b ? 1 : 0)),
lv_identifier(),
lv_column(-1),
lv_hidden(false),
lv_from_module(false), lv_format(NULL) { };
logline_value(const intern_string_t name, int64_t i)
: lv_name(name), lv_kind(VALUE_INTEGER), lv_value(i), lv_identifier(), lv_column(-1),
lv_from_module(false), lv_format(NULL) { };
lv_hidden(false), lv_from_module(false), lv_format(NULL) { };
logline_value(const intern_string_t name, double i)
: lv_name(name), lv_kind(VALUE_FLOAT), lv_value(i), lv_identifier(), lv_column(-1),
lv_from_module(false), lv_format(NULL) { };
lv_hidden(false), lv_from_module(false), lv_format(NULL) { };
logline_value(const intern_string_t name, shared_buffer_ref &sbr, int column = -1)
: lv_name(name), lv_kind(VALUE_TEXT), lv_sbr(sbr),
lv_identifier(), lv_column(column),
lv_from_module(false), lv_format(NULL) {
lv_hidden(false), lv_from_module(false), lv_format(NULL) {
};
logline_value(const intern_string_t name, const intern_string_t val, int column = -1)
: lv_name(name), lv_kind(VALUE_TEXT), lv_intern_string(val), lv_identifier(),
lv_column(column), lv_from_module(false), lv_format(NULL) {
lv_column(column), lv_hidden(false), lv_from_module(false), lv_format(NULL) {
};
logline_value(const intern_string_t name, kind_t kind, shared_buffer_ref &sbr,
@ -391,7 +392,7 @@ public:
int col=-1, int start=-1, int end=-1, bool from_module=false,
const log_format *format=NULL)
: lv_name(name), lv_kind(kind),
lv_identifier(ident), lv_column(col),
lv_identifier(ident), lv_column(col), lv_hidden(false),
lv_origin(start, end),
lv_from_module(from_module),
lv_format(format)
@ -536,6 +537,7 @@ public:
intern_string_t lv_intern_string;
bool lv_identifier;
int lv_column;
bool lv_hidden;
struct line_range lv_origin;
bool lv_from_module;
const log_format *lv_format;
@ -826,7 +828,8 @@ public:
vd_foreign_key(false),
vd_unit_field_index(-1),
vd_column(-1),
vd_hidden(false) {
vd_hidden(false),
vd_internal(false) {
};
@ -841,6 +844,7 @@ public:
std::map<const intern_string_t, scaling_factor> vd_unit_scaling;
int vd_column;
bool vd_hidden;
bool vd_internal;
std::vector<std::string> vd_action_list;
bool operator<(const value_def &rhs) const {
@ -987,12 +991,85 @@ public:
return this->elf_source_path;
};
bool has_value_def(const intern_string_t &ist) const {
return (ist == this->lf_timestamp_field ||
ist == this->elf_level_field ||
ist == this->elf_body_field ||
this->elf_value_defs.find(ist) != this->elf_value_defs.end());
}
enum json_log_field {
JLF_CONSTANT,
JLF_VARIABLE
};
struct json_format_element {
enum align_t {
LEFT,
RIGHT,
};
enum overflow_t {
ABBREV,
TRUNCATE,
DOTDOT,
};
json_format_element()
: jfe_type(JLF_CONSTANT), jfe_default_value("-"), jfe_min_width(0),
jfe_max_width(LLONG_MAX), jfe_align(LEFT),
jfe_overflow(ABBREV)
{ };
json_log_field jfe_type;
intern_string_t jfe_value;
std::string jfe_default_value;
long long jfe_min_width;
long long jfe_max_width;
align_t jfe_align;
overflow_t jfe_overflow;
std::string jfe_ts_format;
};
struct json_field_cmp {
json_field_cmp(json_log_field type,
const intern_string_t name)
: jfc_type(type), jfc_field_name(name) {
};
bool operator()(const json_format_element &jfe) const {
return (this->jfc_type == jfe.jfe_type &&
this->jfc_field_name == jfe.jfe_value);
};
json_log_field jfc_type;
const intern_string_t jfc_field_name;
};
long value_line_count(const intern_string_t ist,
const unsigned char *str = NULL,
ssize_t len = -1) const {
std::map<const intern_string_t, value_def>::const_iterator iter =
this->elf_value_defs.find(ist);
long line_count = (str != NULL) ? std::count(&str[0], &str[len], '\n') + 1 : 1;
if (iter == this->elf_value_defs.end()) {
return this->jlf_hide_extra ? 0 : line_count;
}
if (iter->second.vd_hidden) {
return 0;
}
if (std::find_if(this->jlf_line_format.begin(),
this->jlf_line_format.end(),
json_field_cmp(JLF_VARIABLE, ist)) !=
this->jlf_line_format.end()) {
return line_count - 1;
}
return line_count;
};
bool has_value_def(const intern_string_t ist) const {
std::map<const intern_string_t, value_def>::const_iterator iter =
this->elf_value_defs.find(ist);
return iter != this->elf_value_defs.end();
};
std::string get_pattern_name() const {
if (this->jlf_json) {
@ -1035,39 +1112,6 @@ public:
bool elf_builtin_format;
std::vector<std::pair<intern_string_t, std::string> > elf_search_tables;
enum json_log_field {
JLF_CONSTANT,
JLF_VARIABLE
};
struct json_format_element {
enum align_t {
LEFT,
RIGHT,
};
enum overflow_t {
ABBREV,
TRUNCATE,
DOTDOT,
};
json_format_element()
: jfe_type(JLF_CONSTANT), jfe_default_value("-"), jfe_min_width(0),
jfe_max_width(LLONG_MAX), jfe_align(LEFT),
jfe_overflow(ABBREV)
{ };
json_log_field jfe_type;
intern_string_t jfe_value;
std::string jfe_default_value;
long long jfe_min_width;
long long jfe_max_width;
align_t jfe_align;
overflow_t jfe_overflow;
std::string jfe_ts_format;
};
void json_append_to_cache(const char *value, size_t len) {
size_t old_size = this->jlf_cached_line.size();
this->jlf_cached_line.resize(old_size + len);

2
src/shared_buffer.cc
View File

@ -57,6 +57,8 @@ void shared_buffer_ref::share(shared_buffer &sb, char *data, size_t len)
this->sb_owner = &sb;
this->sb_data = data;
this->sb_length = len;
ensure(this->sb_length < (5 * 1024 * 1024));
}
bool shared_buffer_ref::subset(shared_buffer_ref &other, off_t offset, size_t len)

30
test/formats/jsontest/lnav-logstash.json
View File

@ -8,7 +8,7 @@
"file-pattern" : "\\.clog.*",
"multiline": false,
"line-format" :
[
[
{ "field" : "@timestamp" },
" ",
{ "field" : "ipaddress" },
@ -21,21 +21,27 @@
"body-field" : "message",
"level-field" : "level",
"level" : {
"trace" : "TRACE",
"trace" : "TRACE",
"debug" : "DEBUG",
"info" : "INFO",
"error" : "ERROR",
"warning" : "WARN"
},
"value" : {
"logger_name" : {
"kind" : "string",
"identifier" : true
},
"ipaddress" : {
"kind" : "string",
"identifier" : true
}
}
"value" : {
"logger_name" : {
"kind" : "string",
"identifier" : true
},
"ipaddress" : {
"kind" : "string",
"identifier" : true
},
"level_value" : {
"hidden": true
},
"stack_trace" : {
"kind" : "string"
}
}
}
}

3
test/formats/jsontest2/format.json
View File

@ -26,6 +26,9 @@
"user" : {
"kind" : "string",
"identifier" : true
},
"cl" : {
"kind" : "string"
}
}
}

12
test/test_json_format.sh
View File

@ -72,13 +72,13 @@ Caused by: java.lang.ClassNotFoundException: javax.el.StaticFieldELResolver
@version: 1
logger_name: org.apache.jasper.runtime.JspFactoryImpl
thread_name: http-bio-0.0.0.0-8081-exec-198
level_value: 40000
level: ERROR
customer: foobaz
2016-08-03T12:06:31.009 - ;Exception initializing page context;
@version: 1
logger_name: org.apache.jasper.runtime.JspFactoryImpl
thread_name: http-bio-0.0.0.0-8081-exec-198
level_value: 40000
level: ERROR
customer: foobaz
EOF
@ -125,10 +125,10 @@ run_test ${lnav_test} -n -d /tmp/lnav.err \
${test_dir}/logfile_json2.json
check_output "log levels not working" <<EOF
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,user
0,<NULL>,2013-09-06 20:00:49.124,0,info,0,<NULL>
1,<NULL>,2013-09-06 22:00:49.124,7200000,info,0,steve@example.com
3,<NULL>,2013-09-06 22:01:49.124,60000,error,0,<NULL>
log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,cl,user
0,<NULL>,2013-09-06 20:00:49.124,0,info,0,com.exmaple.foo,<NULL>
1,<NULL>,2013-09-06 22:00:49.124,7200000,info,0,com.exmaple.foo,steve@example.com
3,<NULL>,2013-09-06 22:01:49.124,60000,error,0,com.exmaple.foo,<NULL>
EOF