[log-format] abbreviate long fields when formatting JSON logs

Fixes #340
pull/356/head
Timothy Stack 8 years ago
parent 9ee18c26d3
commit c05dfafae8

@ -3,9 +3,9 @@ lnav v0.8.2:
Features:
* The timestamp format for JSON log files can be specified with the
"timestamp-format" option in the "line-format" array.
* Added "min-width" and "align" options to the "line-format" in format
definitions for JSON log files. These options give you more control
over how the displayed line looks.
* Added "min-width", "max-width", ""align", and "overflow" options to the
"line-format" in format definitions for JSON log files. These options
give you more control over how the displayed line looks.
lnav v0.8.1:
Features:

@ -77,9 +77,16 @@ fields:
:min-width: The minimum width for the field. If the value for the field
in a given log message is shorter, padding will be added as needed to
meet the minimum-width requirement. (v0.8.2+)
:max-width: The maximum width for the field. If the value for the field
in a given log message is longer, the overflow algorithm will be applied
to try and shorten the field. (v0.8.2+)
:align: Specifies the alignment for the field, either "left" or "right".
If "left", padding to meet the minimum-width will be added on the right.
If "right", padding will be added on the left. (v0.8.2+)
:overflow: The algorithm used to shorten a field that is longer than
"max-width". The only option at the moment is "abbrev", which removes
all but the first letter in dotted text. For example, "com.example.foo"
would be shortened to "c.e.foo". (v0.8.2+)
:timestamp-format: The timestamp format to use when displaying the time
for this log message. (v0.8.2+)
:default-value: The default value to use if the field could not be found

@ -716,3 +716,27 @@ bool wordexperr(int rc, string &msg)
return true;
}
size_t abbreviate_str(char *str, size_t len, size_t max_len)
{
size_t last_start = 1;
if (len < max_len) {
return len;
}
for (size_t index = 0; index < len; index++) {
if (str[index] == '.' || str[index] == '-') {
memmove(&str[last_start], &str[index], len - index);
len -= (index - last_start);
index = last_start + 1;
last_start = index + 1;
if (len < max_len) {
return len;
}
}
}
return len;
}

@ -424,4 +424,6 @@ inline void rusageadd(const struct rusage &left, const struct rusage &right, str
diff_out.ru_nivcsw = left.ru_nivcsw + right.ru_nivcsw;
}
size_t abbreviate_str(char *str, size_t len, size_t max_len);
#endif

@ -71,6 +71,9 @@ const intern_string_t external_log_format::json_format_element::ALIGN_LEFT =
const intern_string_t external_log_format::json_format_element::ALIGN_RIGHT =
intern_string::lookup("right");
const intern_string_t external_log_format::json_format_element::OVERFLOW_ABBREV =
intern_string::lookup("abbrev");
const char *logline::level_names[LEVEL__MAX + 1] = {
"unknown",
"trace",
@ -1163,6 +1166,20 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
}
}
size_t actual_size = this->jlf_cached_line.size() -
lr.lr_start;
if (actual_size > jfe.jfe_max_width) {
if (jfe.jfe_overflow == json_format_element::OVERFLOW_ABBREV) {
size_t new_size = abbreviate_str(
&this->jlf_cached_line[lr.lr_start],
actual_size,
jfe.jfe_max_width);
this->jlf_cached_line.resize(lr.lr_start + new_size);
}
}
if (nl_pos == string::npos) {
lr.lr_end = this->jlf_cached_line.size();
}

@ -1044,16 +1044,21 @@ public:
static const intern_string_t ALIGN_LEFT;
static const intern_string_t ALIGN_RIGHT;
static const intern_string_t OVERFLOW_ABBREV;
json_format_element()
: jfe_type(JLF_CONSTANT), jfe_default_value("-"), jfe_min_width(0),
jfe_align(ALIGN_LEFT)
jfe_max_width(LLONG_MAX), jfe_align(ALIGN_LEFT),
jfe_overflow(OVERFLOW_ABBREV)
{ };
json_log_field jfe_type;
intern_string_t jfe_value;
std::string jfe_default_value;
long long jfe_min_width;
long long jfe_max_width;
intern_string_t jfe_align;
intern_string_t jfe_overflow;
std::string jfe_ts_format;
};

@ -431,6 +431,12 @@ static const intern_string_t ALIGN_ENUM[] = {
intern_string_t()
};
static const intern_string_t OVERFLOW_ENUM[] = {
external_log_format::json_format_element::OVERFLOW_ABBREV,
intern_string_t()
};
static struct json_path_handler line_format_handlers[] = {
json_path_handler("field")
.with_synopsis("<field-name>")
@ -455,12 +461,24 @@ static struct json_path_handler line_format_handlers[] = {
.with_description("The minimum width of the field")
.for_field(&nullobj<external_log_format::json_format_element>()->jfe_min_width),
json_path_handler("max-width")
.with_min_value(0)
.with_synopsis("<size>")
.with_description("The maximum width of the field")
.for_field(&nullobj<external_log_format::json_format_element>()->jfe_max_width),
json_path_handler("align")
.with_synopsis("left|right")
.with_description("Align the text in the column to the left or right side")
.with_enum_values(ALIGN_ENUM)
.for_field(&nullobj<external_log_format::json_format_element>()->jfe_align),
json_path_handler("overflow")
.with_synopsis("abbrev")
.with_description("Overflow style")
.with_enum_values(OVERFLOW_ENUM)
.for_field(&nullobj<external_log_format::json_format_element>()->jfe_overflow),
json_path_handler()
};

@ -14,7 +14,15 @@ add_executable(test_date_time_scanner test_date_time_scanner.cc
../src/pcrepp.cc
../src/lnav_log.cc
../src/spookyhash/SpookyV2.cpp)
add_executable(test_abbrev test_abbrev.cc
../src/lnav_util.cc
../../lbuild/src/time_fmts.cc
../src/ptimec_rt.cc
../src/pcrepp.cc
../src/lnav_log.cc
../src/spookyhash/SpookyV2.cpp)
link_directories(/opt/local/lib)
target_link_libraries(test_pcrepp /opt/local/lib/libpcre.a)
target_link_libraries(test_reltime /opt/local/lib/libpcre.a)
target_link_libraries(test_date_time_scanner /opt/local/lib/libpcre.a)
target_link_libraries(test_abbrev /opt/local/lib/libpcre.a)

@ -32,6 +32,7 @@ check_PROGRAMS = \
drive_readline_curses \
slicer \
scripty \
test_abbrev \
test_ansi_scrubber \
test_auto_fd \
test_auto_mem \
@ -104,6 +105,9 @@ test_top_status_LDADD = \
test_yajlpp_SOURCES = test_yajlpp.cc
test_yajlpp_LDADD = ../src/libdiag.a
test_abbrev_SOURCES = test_abbrev.cc
test_abbrev_LDADD = ../src/libdiag.a
test_concise_SOURCES = test_concise.cc
test_concise_LDADD = ../src/libdiag.a
@ -319,6 +323,7 @@ dist_noinst_DATA = \
log-samples/sample-ad31f12d2adabd07e3ddda3ad5b0dbf6b49c4c99.txt
TESTS = \
test_abbrev \
test_ansi_scrubber \
test_auto_fd \
test_auto_mem \

@ -11,6 +11,8 @@
" ",
{ "field" : "lvl", "min-width": 5 },
" ",
{ "field" : "cl", "max-width": 5},
" ",
{ "field" : "msg" }
],
"level-field" : "lvl",

@ -1,3 +1,3 @@
{"ts": "2013-09-06T20:00:49.124817Z", "lvl": 0, "msg": "Starting up service"}
{"ts": "2013-09-06T22:00:49.124817Z", "lvl": 0, "msg": "Shutting down service", "user": "steve@example.com"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": 10, "msg": "looking bad"}
{"ts": "2013-09-06T20:00:49.124817Z", "lvl": 0, "msg": "Starting up service", "cl": "com.exmaple.foo"}
{"ts": "2013-09-06T22:00:49.124817Z", "lvl": 0, "msg": "Shutting down service", "user": "steve@example.com", "cl": "com.exmaple.foo"}
{"ts": "2013-09-06T22:01:49.124817Z", "lvl": 10, "msg": "looking bad", "cl": "com.exmaple.foo"}

@ -0,0 +1,63 @@
/**
* Copyright (c) 2016, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include <assert.h>
#include "lnav_util.hh"
static struct test_data {
const char *str;
const char *abbrev_str;
size_t max_len;
} TEST_DATA[] = {
{ "abc", "abc", 5 },
{ "com.example.foo.bar", "c.e.f.bar", 5 },
{ "com.example.foo.bar", "c.e.foo.bar", 15 },
{ "no dots in here", "no dots in here", 5 },
{ NULL }
};
int main(int argc, char *argv[])
{
for (int lpc = 0; TEST_DATA[lpc].str; lpc++) {
test_data &td = TEST_DATA[lpc];
char buffer[1024];
strcpy(buffer, td.str);
size_t actual = abbreviate_str(buffer, strlen(td.str), td.max_len);
buffer[actual] = '\0';
printf("orig: %s\n", td.str);
printf(" act: %s\n", buffer);
assert(strcmp(buffer, td.abbrev_str) == 0);
}
}

@ -54,10 +54,10 @@ run_test ${lnav_test} -n \
${test_dir}/logfile_json2.json
check_output "timestamp-format not working" <<EOF
2013-09-06T20:00:49.124 abc 49 def 0 Starting up service
2013-09-06T22:00:49.124 abc 49 def 0 Shutting down service
2013-09-06T20:00:49.124 abc 49 def 0 c.e.foo Starting up service
2013-09-06T22:00:49.124 abc 49 def 0 c.e.foo Shutting down service
user: steve@example.com
2013-09-06T22:01:49.124 abc 49 def 10 looking bad
2013-09-06T22:01:49.124 abc 49 def 10 c.e.foo looking bad
EOF
run_test ${lnav_test} -n -d /tmp/lnav.err \

Loading…
Cancel
Save