[pretty-print] first pass at a pretty-printer

Defect Number:
    Reviewed By:
   Testing Done:
pull/129/head
Timothy Stack 9 years ago
parent 1d3481c3fa
commit c670a86d04

@ -3,6 +3,8 @@ lnav v0.7.3:
* Add 'pipe-to' and 'pipe-line-to' commands that pipe the currently
marked lines or the current log message to a shell command,
respectively.
* Added a "pretty-print" view (P hotkey) that tries to reformat log
messages so that they are easier to read.
lnav v0.7.2:
* Added log formats for vdsm, openstack, and the vmkernel.

@ -174,6 +174,8 @@ Display
- View/leave builtin help
* - |ks| q |ke|
- Return to the previous view/quit
* - |ks| Shift |ke| + |ks| p |ke|
- Switch to/from the pretty-printed view of the top log message
* - |ks| Shift |ke| + |ks| t |ke|
- Display elapsed time between lines
* - |ks| t |ke|

@ -84,6 +84,7 @@ set(diag_STAT_SRCS
log_data_table.hh
log_format_impls.cc
logfile_stats.hh
pretty_printer.hh
ptimec.hh
sequence_sink.hh
status_controllers.hh

@ -680,6 +680,9 @@ private:
case DT_IPV6_ADDRESS:
case DT_MAC_ADDRESS:
case DT_HEX_DUMP:
case DT_XML_OPEN_TAG:
case DT_XML_CLOSE_TAG:
case DT_XML_EMPTY_TAG:
case DT_UUID:
case DT_URL:
case DT_PATH:

@ -63,6 +63,17 @@ static struct {
{ "hexd", pcrepp(
"\\A([0-9a-fA-F][0-9a-fA-F](?::[0-9a-fA-F][0-9a-fA-F])+)"), },
{ "xmlt", pcrepp(
"\\A(<\\??[\\w:]+\\s*(?:[\\w:]+(?:\\s*=\\s*"
"(?:\"((?:\\\\.|[^\"])+)\"|'((?:\\\\.|[^'])+)'|[^>]+)"
"))*\\s*(?:/|\\?)>)"), },
{ "xmlo", pcrepp(
"\\A(<[\\w:]+\\s*(?:[\\w:]+(?:\\s*=\\s*"
"(?:\"((?:\\\\.|[^\"])+)\"|'((?:\\\\.|[^'])+)'|[^>]+)"
"))*\\s*>)"), },
{ "xmlc", pcrepp("\\A(</[\\w:]+\\s*>)"), },
{ "coln", pcrepp("\\A(:)"),
},
{ "eq", pcrepp("\\A(=)"),

@ -46,6 +46,9 @@ enum data_token_t {
DT_TIME,
DT_IPV6_ADDRESS,
DT_HEX_DUMP,
DT_XML_EMPTY_TAG,
DT_XML_OPEN_TAG,
DT_XML_CLOSE_TAG,
/* DT_QUALIFIED_NAME, */
DT_COLON,
@ -116,8 +119,9 @@ public:
};
data_scanner(shared_buffer_ref &line, size_t off = 0, size_t len = (size_t) -1)
: ds_sbr(line), ds_pcre_input(line.get_data(), off, len)
: ds_sbr(line), ds_pcre_input(line.get_data(), off, len == -1 ? line.length() : len)
{
require(len == -1 || len <= line.length());
if (line.length() > 0 && line.get_data()[line.length() - 1] == '.') {
this->ds_pcre_input.pi_length -= 1;
}
@ -130,6 +134,7 @@ public:
private:
std::string ds_line;
shared_buffer_ref ds_sbr;
pcre_input ds_pcre_input;
pcre_input ds_pcre_input;
};
#endif

@ -11,7 +11,7 @@
"pattern" : "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?<c_ip>[^ ]+) (?<cs_username>[^ ]+) (?<cs_method>[A-Z]+) \"(?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))?\" (?:-1|\\d+) (?<sc_status>\\d+) \\d+"
},
"std" : {
"pattern" : "^(?<c_ip>[\\w\\.:\\-]+) [\\w\\.\\-]+ (?<cs_username>\\S+) \\[(?<timestamp>[^\\]]+)\\] \"(?:\\-|(?<cs_method>\\w+) (?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))? (?<cs_version>[\\w/\\.]+))\" (?<sc_status>\\d+) (?<sc_bytes>\\d+|-)(?: \"(?<cs_referer>[^\"]+)\" \"(?<cs_user_agent>[^\"]+)\")?.*"
"pattern" : "^(?<c_ip>[\\w\\.:\\-]+)\\s+[\\w\\.\\-]+\\s+(?<cs_username>\\S+)\\s+\\[(?<timestamp>[^\\]]+)\\] \"(?:\\-|(?<cs_method>\\w+) (?<cs_uri_stem>[^ \\?]+)(?:\\?(?<cs_uri_query>[^ ]*))? (?<cs_version>[\\w/\\.]+))\" (?<sc_status>\\d+) (?<sc_bytes>\\d+|-)(?: \"(?<cs_referer>[^\"]+)\" \"(?<cs_user_agent>[^\"]+)\")?(?<body>.*)"
}
},
"level-field": "sc_status",
@ -62,6 +62,9 @@
"sample" : [
{
"line" : "10.112.72.172 - - [11/Feb/2013:06:43:36 +0000] \"GET /client/ HTTP/1.1\" 200 5778 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17\""
},
{
"line" : "10.1.10.51 - - [23/Dec/2014:21:20:35 +0000] \"POST /api/1/rest/foo/bar HTTP/1.1\" 200 - \"-\" \"-\" 293"
}
]
},

@ -161,6 +161,10 @@ through the file.
>/< Move horizontally to the next/previous search hit.
P Switch to/from the pretty-printed view of the top log
message. In this view, structured data, such as XML,
will be reformatted to make it easier to read.
t Switch to/from the text file view. The text file view is
for any files that are not recognized as log files.

@ -115,6 +115,7 @@
#include "log_data_helper.hh"
#include "readline_highlighters.hh"
#include "environ_vtab.hh"
#include "pretty_printer.hh"
#include "yajlpp.hh"
@ -152,6 +153,7 @@ const char *lnav_view_strings[LNV__MAX + 1] = {
"db",
"example",
"schema",
"pretty",
NULL
};
@ -165,6 +167,7 @@ static const char *view_titles[LNV__MAX] = {
"DB",
"EXAMPLE",
"SCHEMA",
"PRETTY",
};
static bool rescan_files(bool required = false);
@ -948,8 +951,8 @@ public:
}
};
plain_text_source(const vector<string> &lines) {
this->tds_lines = lines;
plain_text_source(const vector<string> &text_lines) {
this->tds_lines = text_lines;
};
size_t text_line_count()
@ -1108,6 +1111,52 @@ static void update_view_name(void)
A_REVERSE | view_colors::ansi_color_pair(COLOR_BLUE, COLOR_WHITE)));
}
static void open_schema_view(void)
{
textview_curses *schema_tc = &lnav_data.ld_views[LNV_SCHEMA];
string schema;
dump_sqlite_schema(lnav_data.ld_db, schema);
schema += "\n\n-- Virtual Table Definitions --\n\n";
schema += ENVIRON_CREATE_STMT;
for (log_vtab_manager::iterator vtab_iter =
lnav_data.ld_vtab_manager->begin();
vtab_iter != lnav_data.ld_vtab_manager->end();
++vtab_iter) {
schema += vtab_iter->second->get_table_statement();
}
if (schema_tc->get_sub_source() != NULL) {
delete schema_tc->get_sub_source();
}
schema_tc->set_sub_source(new plain_text_source(schema));
}
static void open_pretty_view(void)
{
textview_curses *log_tc = &lnav_data.ld_views[LNV_LOG];
textview_curses *pretty_tc = &lnav_data.ld_views[LNV_PRETTY];
logfile_sub_source &lss = lnav_data.ld_log_source;
if (lss.text_line_count() > 0) {
content_line_t cl = lss.at(log_tc->get_top());
logfile *lf = lss.find(cl);
logfile::iterator ll = lf->message_start(lf->begin() + cl);
shared_buffer_ref sbr;
lf->read_full_message(ll, sbr);
data_scanner ds(sbr);
pretty_printer pp(&ds);
if (pretty_tc->get_sub_source() != NULL) {
delete pretty_tc->get_sub_source();
}
string pretty_text = pp.print();
pretty_tc->set_sub_source(new plain_text_source(pretty_text));
}
}
bool toggle_view(textview_curses *toggle_tc)
{
textview_curses *tc = lnav_data.ld_view_stack.top();
@ -1117,6 +1166,12 @@ bool toggle_view(textview_curses *toggle_tc)
lnav_data.ld_view_stack.pop();
}
else {
if (toggle_tc == &lnav_data.ld_views[LNV_SCHEMA]) {
open_schema_view();
}
else if (toggle_tc == &lnav_data.ld_views[LNV_PRETTY]) {
open_pretty_view();
}
lnav_data.ld_view_stack.push(toggle_tc);
retval = true;
}
@ -1145,29 +1200,6 @@ void redo_search(lnav_view_t view_index)
lnav_data.ld_scroll_broadcaster.invoke(tc);
}
static void open_schema_view(void)
{
textview_curses *schema_tc = &lnav_data.ld_views[LNV_SCHEMA];
string schema;
dump_sqlite_schema(lnav_data.ld_db, schema);
schema += "\n\n-- Virtual Table Definitions --\n\n";
schema += ENVIRON_CREATE_STMT;
for (log_vtab_manager::iterator vtab_iter =
lnav_data.ld_vtab_manager->begin();
vtab_iter != lnav_data.ld_vtab_manager->end();
++vtab_iter) {
schema += vtab_iter->second->get_table_statement();
}
if (schema_tc->get_sub_source() != NULL) {
delete schema_tc->get_sub_source();
}
schema_tc->set_sub_source(new plain_text_source(schema));
}
/**
* Ensure that the view is on the top of the view stack.
*
@ -1178,9 +1210,6 @@ void ensure_view(textview_curses *expected_tc)
textview_curses *tc = lnav_data.ld_view_stack.top();
if (tc != expected_tc) {
if (expected_tc == &lnav_data.ld_views[LNV_SCHEMA]) {
open_schema_view();
}
toggle_view(expected_tc);
}
@ -1968,6 +1997,16 @@ static void handle_paging_key(int ch)
tc->reload_data();
break;
case 'P':
if (tc == &lnav_data.ld_views[LNV_PRETTY] ||
(lss && lss->text_line_count() > 0)) {
toggle_view(&lnav_data.ld_views[LNV_PRETTY]);
}
else {
lnav_data.ld_rl_view->set_value("Pretty-printed only works with log messages");
}
break;
case 't':
if (lnav_data.ld_text_source.current_file() == NULL) {
flash();
@ -4264,6 +4303,7 @@ int main(int argc, char *argv[])
setup_highlights(lnav_data.ld_views[LNV_LOG].get_highlights());
setup_highlights(lnav_data.ld_views[LNV_TEXT].get_highlights());
setup_highlights(lnav_data.ld_views[LNV_SCHEMA].get_highlights());
setup_highlights(lnav_data.ld_views[LNV_PRETTY].get_highlights());
}
{

@ -108,6 +108,7 @@ typedef enum {
LNV_DB,
LNV_EXAMPLE,
LNV_SCHEMA,
LNV_PRETTY,
LNV__MAX
} lnav_view_t;

@ -0,0 +1,170 @@
/**
* Copyright (c) 2015, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __pretty_printer_hh
#define __pretty_printer_hh
#include <deque>
#include <strstream>
#include "data_scanner.hh"
class pretty_printer {
public:
struct element {
element(data_token_t token, pcre_context &pc)
: e_token(token), e_capture(*pc.all()) {
};
data_token_t e_token;
pcre_context::capture_t e_capture;
};
pretty_printer(data_scanner *ds)
: pp_depth(0), pp_line_length(0), pp_scanner(ds) {
};
std::string print() {
pcre_context_static<30> pc;
data_token_t dt;
while (this->pp_scanner->tokenize(pc, dt)) {
element el(dt, pc);
switch (dt) {
case DT_XML_EMPTY_TAG:
this->start_new_line();
this->pp_values.push_back(el);
this->start_new_line();
continue;
case DT_XML_OPEN_TAG:
this->start_new_line();
this->write_element(el);
this->pp_depth += 1;
continue;
case DT_XML_CLOSE_TAG:
this->flush_values();
this->pp_depth -= 1;
this->write_element(el);
this->start_new_line();
continue;
case DT_LCURLY:
case DT_LSQUARE:
this->flush_values(true);
this->pp_values.push_back(el);
this->pp_depth += 1;
continue;
case DT_RCURLY:
case DT_RSQUARE:
this->flush_values();
this->pp_depth -= 1;
this->write_element(el);
continue;
case DT_COMMA:
this->flush_values(true);
this->write_element(el);
this->start_new_line();
continue;
}
this->pp_values.push_back(el);
}
this->flush_values();
this->pp_stream << std::endl << std::ends;
std::string retval = this->pp_stream.str();
this->pp_stream.freeze(false);
return retval;
};
private:
void start_new_line() {
bool has_output;
if (this->pp_line_length > 0) {
this->pp_stream << std::endl;
}
has_output = this->flush_values();
if (has_output) {
this->pp_stream << std::endl;
}
this->pp_line_length = 0;
}
bool flush_values(bool start_on_depth = false) {
bool retval = false;
while (!this->pp_values.empty()) {
{
element &el = this->pp_values.front();
this->write_element(this->pp_values.front());
if (start_on_depth &&
(el.e_token == DT_LSQUARE ||
el.e_token == DT_LCURLY)) {
this->pp_stream << std::endl;
this->pp_line_length = 0;
}
}
this->pp_values.pop_front();
retval = true;
}
return retval;
}
void append_indent() {
for (int lpc = 0; lpc < this->pp_depth; lpc++) {
this->pp_stream << " ";
}
}
void write_element(const element &el) {
if (this->pp_line_length == 0 && el.e_token == DT_WHITE) {
return;
}
pcre_input &pi = this->pp_scanner->get_input();
if (this->pp_line_length == 0) {
this->append_indent();
}
this->pp_stream << pi.get_substr(&el.e_capture);
this->pp_line_length += el.e_capture.length();
}
int pp_depth;
int pp_line_length;
data_scanner *pp_scanner;
std::strstream pp_stream;
std::deque<element> pp_values;
};
#endif

@ -200,6 +200,7 @@ dist_noinst_DATA = \
datafile_simple.11 \
datafile_simple.12 \
datafile_simple.13 \
datafile_xml.0 \
listview_output.0 \
listview_output.1 \
listview_output.2 \

@ -794,6 +794,7 @@ dist_noinst_DATA = \
datafile_simple.11 \
datafile_simple.12 \
datafile_simple.13 \
datafile_xml.0 \
listview_output.0 \
listview_output.1 \
listview_output.2 \

@ -0,0 +1,21 @@
<ns1:foo> <elem attr1=xyz attr2="123"> </elem> <closed />
key 0:0
xmlo 0:9 ^-------^ <ns1:foo>
pair 0:9 ^-------^ <ns1:foo>
key 12:12 ^
xmlo 12:40 ^--------------------------^ <elem attr1=xyz attr2="123">
pair 12:40 ^--------------------------^ <elem attr1=xyz attr2="123">
key 42:42 ^
xmlc 42:49 ^-----^ </elem>
pair 42:49 ^-----^ </elem>
key 51:51 ^
xmlt 51:61 ^--------^ <closed />
pair 51:61 ^--------^ <closed />
--
<ns1:foo>
<elem attr1=xyz attr2="123"> </elem>
<closed />

@ -41,7 +41,8 @@
#include "data_parser.hh"
#include "log_format.hh"
#include "log_format_loader.hh"
#include "../src/shared_buffer.hh"
#include "pretty_printer.hh"
#include "shared_buffer.hh"
using namespace std;
@ -50,7 +51,7 @@ const char *TMP_NAME = "scanned.tmp";
int main(int argc, char *argv[])
{
int c, retval = EXIT_SUCCESS;
bool prompt = false, is_log = false;
bool prompt = false, is_log = false, pretty_print = false;
{
std::vector<std::string> paths, errors;
@ -58,12 +59,16 @@ int main(int argc, char *argv[])
load_formats(paths, errors);
}
while ((c = getopt(argc, argv, "pl")) != -1) {
while ((c = getopt(argc, argv, "pPl")) != -1) {
switch (c) {
case 'p':
prompt = true;
break;
case 'P':
pretty_print = true;
break;
case 'l':
is_log = true;
break;
@ -159,6 +164,14 @@ int main(int argc, char *argv[])
dp.parse();
dp.print(out, dp.dp_pairs);
if (pretty_print) {
data_scanner ds2(sub_line, body.lr_start, sub_line.length());
pretty_printer pp(&ds2);
string pretty_out = pp.print();
fprintf(out, "\n--\n%s", pretty_out.c_str());
}
fclose(out);
sprintf(cmd, "diff -u %s %s", argv[lpc], TMP_NAME);

@ -5,6 +5,11 @@ for fn in ${top_srcdir}/test/datafile_simple.*; do
on_error_fail_with "$fn does not match"
done
for fn in ${top_srcdir}/test/datafile_xml.*; do
run_test ./drive_data_scanner -P $fn
on_error_fail_with "$fn does not match"
done
for fn in ${top_srcdir}/test/log-samples/*.txt; do
run_test ./drive_data_scanner -l $fn
on_error_fail_with "$fn does not match"

Loading…
Cancel
Save