[pretty-print] check if lines are XML before indenting them as such

Also, update the scanner for changes to re2c
pull/509/head
Timothy Stack 6 years ago
parent 5268872d1e
commit 1ed0ff7e0f

@ -62,6 +62,7 @@ data_format_state_t dfs_prefix_next(data_format_state_t state,
case DT_RSQUARE:
case DT_LANGLE:
case DT_RANGLE:
case DT_EMPTY_CONTAINER:
break;
default:
@ -182,6 +183,10 @@ data_format_state_t dfs_comma_next(data_format_state_t state,
retval = DFS_VALUE;
break;
case DT_EMPTY_CONTAINER:
retval = DFS_INIT;
break;
case DT_COMMA:
case DT_SEMI:
retval = DFS_ERROR;

@ -978,6 +978,22 @@ private:
state_stack.push(discover_format_state());
break;
case DT_EMPTY_CONTAINER: {
auto &curr_group = this->dp_group_stack.back();
auto empty_list = element_list_t("_anon_", __FILE__, __LINE__);
discover_format_state dfs;
dfs.finalize();
empty_list.el_format = dfs.dfs_format;
curr_group.PUSH_BACK(element(empty_list, DNT_GROUP));
auto &empty = curr_group.back();
empty.e_capture.c_begin = elem.e_capture.c_begin + 1;
empty.e_capture.c_end = elem.e_capture.c_begin + 1;
break;
}
case DT_RPAREN:
case DT_RANGLE:
case DT_RCURLY:

@ -83,6 +83,9 @@ static struct {
{ "semi", pcrepp("\\A(;)"),
},
{ "empt", pcrepp("\\A(\\(\\)|\\{\\}|\\[\\])"),
},
{ "lcurly", pcrepp("\\A({)"),
},
{ "rcurly", pcrepp("\\A(})"),

@ -56,6 +56,8 @@ enum data_token_t {
DT_COMMA,
DT_SEMI,
DT_EMPTY_CONTAINER,
DT_LCURLY,
DT_RCURLY,
@ -132,6 +134,14 @@ public:
pcre_input &get_input() { return this->ds_pcre_input; };
void reset() {
this->ds_pcre_input.reset_next_offset();
if (!this->ds_line.empty() &&
this->ds_line[this->ds_line.length() - 1] == '.') {
this->ds_pcre_input.pi_length -= 1;
}
}
private:
std::string ds_line;
shared_buffer_ref ds_sbr;

File diff suppressed because it is too large Load Diff

@ -48,6 +48,7 @@ bool data_scanner::tokenize2(pcre_context &pc, data_token_t &token_out)
CAPTURE(tok); \
return true; \
}
static const char *EMPTY = "";
pcre_input &pi = this->ds_pcre_input;
struct _YYCURSOR {
const YYCTYPE operator*() const {
@ -58,7 +59,10 @@ bool data_scanner::tokenize2(pcre_context &pc, data_token_t &token_out)
}
operator const YYCTYPE *() const {
return this->val;
if (this->val < this->lim) {
return this->val;
}
return EMPTY;
}
const YYCTYPE *operator=(const YYCTYPE *rhs) {
@ -70,6 +74,11 @@ bool data_scanner::tokenize2(pcre_context &pc, data_token_t &token_out)
return this->val + rhs;
}
const _YYCURSOR *operator-=(int rhs) {
this->val -= rhs;
return this;
}
_YYCURSOR& operator++() {
this->val += 1;
return *this;
@ -79,9 +88,10 @@ bool data_scanner::tokenize2(pcre_context &pc, data_token_t &token_out)
const YYCTYPE *lim;
} YYCURSOR;
YYCURSOR = pi.get_string() + pi.pi_next_offset;
_YYCURSOR yyt1;
_YYCURSOR yyt2;
const YYCTYPE *YYLIMIT = pi.get_string() + pi.pi_length;
const YYCTYPE *YYMARKER = YYCURSOR;
const YYCTYPE *YYCTXMARKER = YYCURSOR;
pcre_context::capture_t *cap = pc.all();
YYCURSOR.lim = YYLIMIT;
@ -92,6 +102,7 @@ bool data_scanner::tokenize2(pcre_context &pc, data_token_t &token_out)
/*!re2c
re2c:yyfill:enable = 0;
re2c:flags:tags = 1;
SPACE = [ \t\r];
ALPHA = [a-zA-Z];
@ -177,6 +188,7 @@ bool data_scanner::tokenize2(pcre_context &pc, data_token_t &token_out)
"=" { RET(DT_EQUALS); }
"," { RET(DT_COMMA); }
";" { RET(DT_SEMI); }
"()" | "{}" | "[]" { RET(DT_EMPTY_CONTAINER); }
"{" { RET(DT_LCURLY); }
"}" { RET(DT_RCURLY); }
"[" { RET(DT_LSQUARE); }

@ -250,6 +250,10 @@ public:
dst[iter->length()] = '\0';
};
void reset_next_offset() {
this->pi_next_offset = this->pi_offset;
};
void reset(const char *str, size_t off = 0, size_t len = -1)
{
this->pi_string = str;

@ -40,3 +40,263 @@ void sigalrm_handler(int sig)
reverse_lookup_enabled = 0;
}
}
std::string pretty_printer::print() {
pcre_context_static<30> pc;
data_token_t dt;
this->pp_scanner->reset();
while (this->pp_scanner->tokenize2(pc, dt)) {
element el(dt, pc);
switch (dt) {
case DT_XML_EMPTY_TAG:
if (is_xml) {
this->start_new_line();
}
this->pp_values.push_back(el);
if (is_xml) {
this->start_new_line();
}
continue;
case DT_XML_OPEN_TAG:
if (is_xml) {
this->start_new_line();
this->write_element(el);
this->descend();
} else {
this->pp_values.push_back(el);
}
continue;
case DT_XML_CLOSE_TAG:
this->flush_values();
this->ascend();
this->write_element(el);
this->start_new_line();
continue;
case DT_LCURLY:
case DT_LSQUARE:
case DT_LPAREN:
this->flush_values(true);
this->pp_values.push_back(el);
this->descend();
continue;
case DT_RCURLY:
case DT_RSQUARE:
case DT_RPAREN:
this->flush_values();
if (this->pp_body_lines.top()) {
this->start_new_line();
}
this->ascend();
this->write_element(el);
continue;
case DT_COMMA:
if (this->pp_depth > 0) {
this->flush_values(true);
this->write_element(el);
this->start_new_line();
continue;
}
break;
case DT_WHITE:
if (this->pp_values.empty() && this->pp_depth == 0) {
// this->pp_leading_indent = el.e_capture.length();
continue;
}
break;
default:
break;
}
this->pp_values.push_back(el);
}
while (this->pp_depth > 0) {
this->ascend();
}
this->flush_values();
return this->pp_stream.str();
}
void pretty_printer::convert_ip_address(const pretty_printer::element &el) {
union {
struct sockaddr_in sin;
struct sockaddr_in6 sin6;
} sa;
pcre_input &pi = this->pp_scanner->get_input();
std::string ipstr = pi.get_substr(&el.e_capture);
std::string result = "unknown";
char buffer[NI_MAXHOST];
int socklen, rc;
if (ipstr == "::") {
return;
}
switch (el.e_token) {
case DT_IPV4_ADDRESS:
sa.sin.sin_family = AF_INET;
rc = inet_pton(AF_INET, ipstr.c_str(), &sa.sin.sin_addr);
socklen = sizeof(struct sockaddr_in);
break;
case DT_IPV6_ADDRESS:
sa.sin6.sin6_family = AF_INET6;
rc = inet_pton(AF_INET6, ipstr.c_str(), &sa.sin6.sin6_addr);
socklen = sizeof(struct sockaddr_in6);
break;
default:
require(0);
break;
}
if (rc == 1 && reverse_lookup_enabled) {
const struct timeval timeout = {0, 500 * 1000};
{
timer::interrupt_timer t(timeout, sigalrm_handler);
if (t.arm_timer() == 0) {
rc = getnameinfo((struct sockaddr *)&sa, socklen,
buffer, sizeof(buffer), NULL, 0,
NI_NAMEREQD);
if (rc == 0) {
result = buffer;
}
}
else {
log_error("Unable to set timer, disabling reverse lookup");
reverse_lookup_enabled = 0;
}
}
if (!reverse_lookup_enabled) {
log_info("Reverse lookup in pretty-print view disabled");
}
}
this->pp_stream << " " << ANSI_UNDERLINE_START <<
"(" << result << ")" <<
ANSI_NORM;
}
void pretty_printer::write_element(const pretty_printer::element &el) {
if (this->pp_leading_indent == 0 &&
this->pp_line_length == 0 &&
el.e_token == DT_WHITE) {
return;
}
if (this->pp_line_length == 0 && el.e_token == DT_LINE) {
return;
}
pcre_input &pi = this->pp_scanner->get_input();
if (this->pp_line_length == 0) {
this->append_indent();
}
if (el.e_token == DT_QUOTED_STRING) {
auto_mem<char> unquoted_str((char *)malloc(el.e_capture.length() + 1));
const char *start = pi.get_substr_start(&el.e_capture);
unquote(unquoted_str.in(), start, el.e_capture.length());
data_scanner ds(unquoted_str.in());
pretty_printer str_pp(&ds,
this->pp_leading_indent + this->pp_depth * 4);
std::string result = str_pp.print();
if (result.find('\n') != std::string::npos) {
switch (start[0]) {
case 'r':
case 'u':
this->pp_stream << start[0];
this->pp_stream << start[1] << start[1];
break;
default:
this->pp_stream << start[0] << start[0];
break;
}
this->pp_stream
<< std::endl
<< result;
if (!endswith(result.c_str(), "\n")) {
this->pp_stream << std::endl;
}
this->pp_stream
<< start[el.e_capture.length() - 1]
<< start[el.e_capture.length() - 1];
} else {
this->pp_stream << pi.get_substr(&el.e_capture);
}
} else {
this->pp_stream << pi.get_substr(&el.e_capture);
switch (el.e_token) {
case DT_IPV4_ADDRESS:
case DT_IPV6_ADDRESS:
this->convert_ip_address(el);
break;
default:
break;
}
}
this->pp_line_length += el.e_capture.length();
if (el.e_token == DT_LINE) {
this->pp_line_length = 0;
this->pp_body_lines.top() += 1;
}
}
void pretty_printer::append_indent() {
this->pp_stream << std::string(this->pp_leading_indent, ' ');
if (this->pp_stream.tellp() == this->pp_leading_indent) {
return;
}
for (int lpc = 0; lpc < this->pp_depth; lpc++) {
this->pp_stream << " ";
}
}
bool pretty_printer::flush_values(bool start_on_depth) {
bool retval = false;
while (!this->pp_values.empty()) {
{
element &el = this->pp_values.front();
this->write_element(this->pp_values.front());
if (start_on_depth &&
(el.e_token == DT_LSQUARE ||
el.e_token == DT_LCURLY)) {
if (this->pp_line_length > 0) {
this->pp_stream << std::endl;
}
this->pp_line_length = 0;
}
}
this->pp_values.pop_front();
retval = true;
}
return retval;
}
void pretty_printer::start_new_line() {
bool has_output;
if (this->pp_line_length > 0) {
this->pp_stream << std::endl;
this->pp_line_length = 0;
}
has_output = this->flush_values();
if (has_output && this->pp_line_length > 0) {
this->pp_stream << std::endl;
}
this->pp_line_length = 0;
this->pp_body_lines.top() += 1;
}
void pretty_printer::ascend() {
if (this->pp_depth > 0) {
int lines = this->pp_body_lines.top();
this->pp_depth -= 1;
this->pp_body_lines.pop();
this->pp_body_lines.top() += lines;
}
else {
this->pp_body_lines.top() = 0;
}
}
void pretty_printer::descend() {
this->pp_depth += 1;
this->pp_body_lines.push(0);
}

@ -70,259 +70,38 @@ public:
: pp_leading_indent(leading_indent),
pp_depth(0),
pp_line_length(0),
pp_scanner(ds) {
pp_scanner(ds),
is_xml(false) {
this->pp_body_lines.push(0);
};
std::string print() {
pcre_context_static<30> pc;
data_token_t dt;
this->pp_scanner->reset();
while (this->pp_scanner->tokenize2(pc, dt)) {
element el(dt, pc);
switch (dt) {
case DT_XML_EMPTY_TAG:
this->start_new_line();
this->pp_values.push_back(el);
this->start_new_line();
continue;
case DT_XML_OPEN_TAG:
this->start_new_line();
this->write_element(el);
this->descend();
continue;
case DT_XML_CLOSE_TAG:
this->flush_values();
this->ascend();
this->write_element(el);
this->start_new_line();
continue;
case DT_LCURLY:
case DT_LSQUARE:
case DT_LPAREN:
this->flush_values(true);
this->pp_values.push_back(el);
this->descend();
continue;
case DT_RCURLY:
case DT_RSQUARE:
case DT_RPAREN:
this->flush_values();
if (this->pp_body_lines.top()) {
this->start_new_line();
}
this->ascend();
this->write_element(el);
continue;
case DT_COMMA:
if (this->pp_depth > 0) {
this->flush_values(true);
this->write_element(el);
this->start_new_line();
continue;
}
break;
case DT_WHITE:
if (this->pp_values.empty() && this->pp_depth == 0) {
// this->pp_leading_indent = el.e_capture.length();
continue;
}
break;
default:
break;
if (dt == DT_XML_CLOSE_TAG) {
is_xml = true;
}
this->pp_values.push_back(el);
}
while (this->pp_depth > 0) {
this->ascend();
}
this->flush_values();
return this->pp_stream.str();
};
private:
void convert_ip_address(const element &el) {
union {
struct sockaddr_in sin;
struct sockaddr_in6 sin6;
} sa;
pcre_input &pi = this->pp_scanner->get_input();
std::string ipstr = pi.get_substr(&el.e_capture);
std::string result = "unknown";
char buffer[NI_MAXHOST];
int socklen, rc;
switch (el.e_token) {
case DT_IPV4_ADDRESS:
sa.sin.sin_family = AF_INET;
rc = inet_pton(AF_INET, ipstr.c_str(), &sa.sin.sin_addr);
socklen = sizeof(struct sockaddr_in);
break;
case DT_IPV6_ADDRESS:
sa.sin6.sin6_family = AF_INET6;
rc = inet_pton(AF_INET6, ipstr.c_str(), &sa.sin6.sin6_addr);
socklen = sizeof(struct sockaddr_in6);
break;
default:
require(0);
break;
}
if (rc == 1 && reverse_lookup_enabled) {
const struct timeval timeout = {0, 500 * 1000};
std::string print();
{
timer::interrupt_timer t(timeout, sigalrm_handler);
if (t.arm_timer() == 0) {
rc = getnameinfo((struct sockaddr *)&sa, socklen,
buffer, sizeof(buffer), NULL, 0,
NI_NAMEREQD);
if (rc == 0) {
result = buffer;
}
}
else {
log_error("Unable to set timer, disabling reverse lookup");
reverse_lookup_enabled = 0;
}
}
if (!reverse_lookup_enabled) {
log_info("Reverse lookup in pretty-print view disabled");
}
}
this->pp_stream << " " << ANSI_UNDERLINE_START <<
"(" << result << ")" <<
ANSI_NORM;
}
void descend() {
this->pp_depth += 1;
this->pp_body_lines.push(0);
}
private:
void ascend() {
if (this->pp_depth > 0) {
int lines = this->pp_body_lines.top();
this->pp_depth -= 1;
this->pp_body_lines.pop();
this->pp_body_lines.top() += lines;
}
else {
this->pp_body_lines.top() = 0;
}
}
void convert_ip_address(const element &el);
void start_new_line() {
bool has_output;
void descend();
if (this->pp_line_length > 0) {
this->pp_stream << std::endl;
this->pp_line_length = 0;
}
has_output = this->flush_values();
if (has_output && this->pp_line_length > 0) {
this->pp_stream << std::endl;
}
this->pp_line_length = 0;
this->pp_body_lines.top() += 1;
}
void ascend();
bool flush_values(bool start_on_depth = false) {
bool retval = false;
void start_new_line();
while (!this->pp_values.empty()) {
{
element &el = this->pp_values.front();
this->write_element(this->pp_values.front());
if (start_on_depth &&
(el.e_token == DT_LSQUARE ||
el.e_token == DT_LCURLY)) {
if (this->pp_line_length > 0) {
this->pp_stream << std::endl;
}
this->pp_line_length = 0;
}
}
this->pp_values.pop_front();
retval = true;
}
return retval;
}
bool flush_values(bool start_on_depth = false);
void append_indent() {
this->pp_stream << std::string(this->pp_leading_indent, ' ');
if (this->pp_stream.tellp() == this->pp_leading_indent) {
return;
}
for (int lpc = 0; lpc < this->pp_depth; lpc++) {
this->pp_stream << " ";
}
}
void append_indent();
void write_element(const element &el) {
if (this->pp_leading_indent == 0 &&
this->pp_line_length == 0 &&
el.e_token == DT_WHITE) {
return;
}
if (this->pp_line_length == 0 && el.e_token == DT_LINE) {
return;
}
pcre_input &pi = this->pp_scanner->get_input();
if (this->pp_line_length == 0) {
this->append_indent();
}
if (el.e_token == DT_QUOTED_STRING) {
pcre_input &pi = this->pp_scanner->get_input();
auto_mem<char> unquoted_str((char *)malloc(el.e_capture.length() + 1));
const char *start = pi.get_substr_start(&el.e_capture);
unquote(unquoted_str.in(), start, el.e_capture.length());
data_scanner ds(unquoted_str.in());
pretty_printer str_pp(&ds,
this->pp_leading_indent + this->pp_depth * 4);
std::string result = str_pp.print();
if (result.find('\n') != std::string::npos) {
switch (start[0]) {
case 'r':
case 'u':
this->pp_stream << start[0];
this->pp_stream << start[1] << start[1];
break;
default:
this->pp_stream << start[0] << start[0];
break;
}
this->pp_stream
<< std::endl
<< result;
if (!endswith(result.c_str(), "\n")) {
this->pp_stream << std::endl;
}
this->pp_stream
<< start[el.e_capture.length() - 1]
<< start[el.e_capture.length() - 1];
} else {
this->pp_stream << pi.get_substr(&el.e_capture);
}
} else {
this->pp_stream << pi.get_substr(&el.e_capture);
switch (el.e_token) {
case DT_IPV4_ADDRESS:
case DT_IPV6_ADDRESS:
this->convert_ip_address(el);
break;
default:
break;
}
}
this->pp_line_length += el.e_capture.length();
if (el.e_token == DT_LINE) {
this->pp_line_length = 0;
this->pp_body_lines.top() += 1;
}
}
void write_element(const element &el);
int pp_leading_indent;
int pp_depth;
@ -331,7 +110,7 @@ private:
data_scanner *pp_scanner;
std::ostringstream pp_stream;
std::deque<element> pp_values;
bool is_xml;
};
#endif

@ -299,6 +299,7 @@ dist_noinst_DATA = \
logfile_bro_conn.log.0 \
logfile_bro_http.log.0 \
logfile_blued.0 \
logfile_cxx.0 \
logfile_empty.0 \
logfile_epoch.0 \
logfile_epoch.1 \

@ -0,0 +1 @@
Mar 24 15:17:38.999 000000000264F I shmem.res 262144 262144 1 chassis_msg_svc/osenv::req_blocking<osenv::req_lambda<tcp_messaging_impl::register_app(svc::messaging_port, defs::atom*, defs::borrowed<svc::messaging_session>, defs::owned<svc::connection_eviction_strategy>&&, svc::messaging::connection_type, svc::messaging::app_param)::{lambda()#1}>, osenv::aloc_dynamic_named<tcp_messaging_impl::register_app(svc::messaging_port, defs::atom*, defs::borrowed<svc::messaging_session>, defs::owned<svc::connection_eviction_strategy>&&, svc::messaginconnection_type, svc::messaging::app_param)::{lambda()#1}, osenv::temporal, tcp_messaging_impl::register_app(svc::messaging_port, defs::atom*, defs::borrowed<svc::messaging_session>, des::owned<svc::connection_eviction_strategy>&&, svc::messaging::connection_type, svc::messaging::app_param)::{lambda()#1}>, osenv::req>->fiber stacks

@ -1,5 +1,30 @@
#! /bin/bash
run_test ${lnav_test} -n \
-c ":switch-to-view pretty" \
${test_dir}/logfile_cxx.0
check_output "pretty-printer is not working for C++ dumps" <<EOF
Mar 24 15:17:38.999 000000000264F I shmem.res 262144 262144 1 chassis_msg_svc/osenv::req_blocking<osenv::req_lambda<tcp_messaging_impl::register_app(svc::messaging_port,
defs::atom*,
defs::borrowed<svc::messaging_session>,
defs::owned<svc::connection_eviction_strategy>&&,
svc::messaging::connection_type,
svc::messaging::app_param
)::{lambda()#1}>, osenv::aloc_dynamic_named<tcp_messaging_impl::register_app(svc::messaging_port,
defs::atom*,
defs::borrowed<svc::messaging_session>,
defs::owned<svc::connection_eviction_strategy>&&,
svc::messaginconnection_type,
svc::messaging::app_param
)::{lambda()#1}, osenv::temporal, tcp_messaging_impl::register_app(svc::messaging_port,
defs::atom*,
defs::borrowed<svc::messaging_session>,
des::owned<svc::connection_eviction_strategy>&&,
svc::messaging::connection_type,
svc::messaging::app_param
)::{lambda()#1}>, osenv::req>->fiber stacks
EOF
echo '2015-04-18T13:16:30.003 8.8.8.8 <foo>8.8.8.8</foo>9 8.8.8.8<1054 198.51.100.1546 544.9.8.7 98.542.241.99 19143.2.5.6' | \
run_test ${lnav_test} -n -c ":switch-to-view pretty"

Loading…
Cancel
Save