Archives/lnav
2
0
Fork 0
mirror of https://github.com/tstack/lnav synced 2024-11-08 01:10:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

[data_scanner] recognize toml/ini table headers

Browse Source
This commit is contained in:
Tim Stack 2023-10-03 09:04:45 -07:00
parent 9663b1f49f
commit 930748a013
20 changed files with 75550 additions and 29638 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Makefile.am
View File

@ -87,7 +87,7 @@ time_fmts.cc: ptimec$(BUILD_EXEEXT)
if HAVE_RE2C
%.cc: %.re
$(RE2C_V)$(RE2C_CMD) --bit-vectors -W -8 -o $@ $<
$(RE2C_V)$(RE2C_CMD) --bit-vectors -W -8 -c -o $@ $<
$(REC2_V)test $@ -ef $(srcdir)/$*.cc || cp $@ $(srcdir)/$*.cc
endif

16
src/base/string_util.tests.cc
View File

@ -87,4 +87,20 @@ TEST_CASE("strnatcmp")
CHECK(strnatcmp(strlen(n1), n1, strlen(n2), n2) < 0);
}
{
constexpr const char* n1 = "servers";
constexpr const char* n2 = "servers.alpha";
CHECK(strnatcasecmp(strlen(n1), n1, strlen(n2), n2) < 0);
}
{
static constexpr const char* TOKENS = "[](){}";
const std::string n1 = "[servers]";
const std::string n2 = "[servers.alpha]";
auto lhs = string_fragment::from_str(n1).trim(TOKENS);
auto rhs = string_fragment::from_str(n2).trim(TOKENS);
CHECK(strnatcasecmp(lhs.length(), lhs.data(), rhs.length(), rhs.data())
< 0);
}
}

12
src/breadcrumb.hh
View File

@ -53,6 +53,18 @@ struct possibility {
bool operator<=(const possibility& rhs) const { return !(rhs < *this); }
bool operator>=(const possibility& rhs) const { return !(*this < rhs); }
static bool sort_cmp(const possibility& lhs, const possibility& rhs)
{
static constexpr const char* TOKENS = "[](){}";
auto lhsf = string_fragment::from_str(lhs.p_key).trim(TOKENS);
auto rhsf = string_fragment::from_str(rhs.p_key).trim(TOKENS);
return strnatcasecmp(
lhsf.length(), lhsf.data(), rhsf.length(), rhsf.data())
< 0;
}
std::string p_key;
attr_line_t p_display_value;
};

8
src/breadcrumb_curses.cc
View File

@ -121,13 +121,7 @@ breadcrumb_curses::reload_data()
[](const auto& elem) { return elem.p_key; },
this->bc_current_search,
128)
| lnav::itertools::sort_with([](const auto& lhs, const auto& rhs) {
return strnatcasecmp(lhs.p_key.size(),
lhs.p_key.data(),
rhs.p_key.size(),
rhs.p_key.data())
< 0;
});
| lnav::itertools::sort_with(breadcrumb::possibility::sort_cmp);
if (selected_crumb_ref.c_key.is<std::string>()
&& selected_crumb_ref.c_expected_input
!= breadcrumb::crumb::expected_input_t::anything)

1
src/data_scanner.hh
View File

@ -220,6 +220,7 @@ private:
string_fragment ds_input;
int ds_init_offset{0};
int ds_next_offset{0};
bool ds_bol{true};
};
#endif

104592
src/data_scanner_re.cc
View File

File diff suppressed because it is too large Load Diff

167
src/data_scanner_re.re
View File

@ -35,6 +35,8 @@
#include "config.h"
#include "data_scanner.hh"
/*!conditions:re2c*/
nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_format_t tf)
{
data_token_t token_out = DT_INVALID;
@ -99,10 +101,26 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
_YYCURSOR yyt1;
_YYCURSOR yyt2;
_YYCURSOR yyt3;
_YYCURSOR yyt4;
_YYCURSOR hunk_heading;
const YYCTYPE *YYLIMIT = (const unsigned char *) this->ds_input.end();
const YYCTYPE *YYMARKER = YYCURSOR;
class _yycond {
public:
int operator()() const {
return this->val;
}
void operator=(int v) {
this->val = v;
}
int val{yycinit};
} c;
if (this->ds_bol) {
c = yycbol;
}
this->ds_bol = false;
YYCURSOR.lim = YYLIMIT;
@ -113,14 +131,18 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
/*!re2c
re2c:yyfill:enable = 0;
re2c:tags = 1;
re2c:sentinel = 0;
re2c:define:YYCTYPE = uint8_t;
re2c:define:YYGETCONDITION = "c";
re2c:define:YYSETCONDITION = "c = @@;";
re2c:tags = 1;
SPACE = [ \t\r];
ALPHA = [a-zA-Z];
ESC = "\x1b";
NUM = [0-9];
ALPHANUM = [a-zA-Z0-9_];
EOF = "\x00";
EOF = "";
SYN = "\x16";
IPV4SEG = ("25"[0-5]|("2"[0-4]|"1"{0,1}[0-9]){0,1}[0-9]);
IPV4ADDR = (IPV4SEG"."){3,3}IPV4SEG;
@ -140,13 +162,14 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
(IPV6SEG":"){1,4}":"IPV4ADDR
);
EOF { return nonstd::nullopt; }
SYN+ {
<init, bol> EOF { return nonstd::nullopt; }
<init, bol> [\x00] { return nonstd::nullopt; }
<*> * { return nonstd::nullopt; }
<init, bol> SYN+ {
RET(DT_ZERO_WIDTH_SPACE);
}
("f"|"u"|"r")?'"'('\\'.|[^\x00\x16\x1b\n"\\]|'""')*'"' {
<init, bol> ("f"|"u"|"r")?'"'('\\'[^\x00]|[^\x00\x16\x1b\n"\\]|'""')*'"' {
CAPTURE(DT_QUOTED_STRING);
switch (this->ds_input[cap_inner.c_begin]) {
case 'f':
@ -159,7 +182,7 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
cap_inner.c_end -= 1;
return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
}
("f"|"u"|"r")?'"""'[^\x00\x16\x1b]*'"""' {
<init, bol> ("f"|"u"|"r")?'"""'[^\x00\x16\x1b]*'"""' {
CAPTURE(DT_QUOTED_STRING);
switch (this->ds_input[cap_inner.c_begin]) {
case 'f':
@ -172,21 +195,21 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
cap_inner.c_end -= 1;
return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
}
"/*" ([^\x00*]|"*"+[^\x00/])* "*"+ "/" {
<init, bol> "/*" ([^\x00*]|"*"+[^\x00/])* "*"+ "/" {
RET(DT_COMMENT);
}
"<!--" ([^\x00*]|"-"+[^\x00>])* "-"{2,} ">" {
<init, bol> "<!--" ([^\x00*]|"-"+[^\x00>])* "-"{2,} ">" {
RET(DT_COMMENT);
}
"#[" "="* "[" ([^\x00\]]|"]" [^=\]])* "]" "="* "]" {
<init, bol> "#[" "="* "[" ([^\x00\]]|"]" [^\x00=\]])* "]" "="* "]" {
RET(DT_COMMENT);
}
[a-qstv-zA-QSTV-Z]"'" {
<init, bol> [a-qstv-zA-QSTV-Z]"'" {
CAPTURE(DT_WORD);
}
("f"|"u"|"r")?"'"('\\'.|"''"|[^\x00\x16\x1b\n'\\])*"'"/[^sS] {
<init, bol> ("f"|"u"|"r")?"'"('\\'[^\x00]|"''"|[^\x00\x16\x1b\n'\\])*"'"/[^sS] {
CAPTURE(DT_QUOTED_STRING);
if (tf == text_format_t::TF_RUST) {
auto sf = this->to_string_fragment(cap_all);
@ -207,96 +230,105 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
cap_inner.c_end -= 1;
return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
}
[a-zA-Z0-9]+":/""/"?[^\x00\x16\x1b\r\n\t '"[\](){}]+[/a-zA-Z0-9\-=&?%] { RET(DT_URL); }
("/"|"./"|"../"|[A-Z]":\\"|"\\\\")("Program Files"(" (x86)")?)?[a-zA-Z0-9_\.\-\~/\\!@#$%^&*()]* { RET(DT_PATH); }
(SPACE|NUM)NUM":"NUM{2}/[^:] { RET(DT_TIME); }
(SPACE|NUM)NUM?":"NUM{2}":"NUM{2}("."NUM{3,6})?/[^:] { RET(DT_TIME); }
[0-9a-fA-F][0-9a-fA-F]((":"|"-")[0-9a-fA-F][0-9a-fA-F])+ {
<init, bol> [a-zA-Z0-9]+":/""/"?[^\x00\x16\x1b\r\n\t '"[\](){}]+[/a-zA-Z0-9\-=&?%] { RET(DT_URL); }
<init, bol> ("/"|"./"|"../"|[A-Z]":\\"|"\\\\")("Program Files"(" (x86)")?)?[a-zA-Z0-9_\.\-\~/\\!@#$%^&*()]* { RET(DT_PATH); }
<init, bol> (SPACE|NUM)NUM":"NUM{2}/[^:] { RET(DT_TIME); }
<init, bol> (SPACE|NUM)NUM?":"NUM{2}":"NUM{2}("."NUM{3,6})?/[^:] { RET(DT_TIME); }
<init, bol> [0-9a-fA-F][0-9a-fA-F]((":"|"-")[0-9a-fA-F][0-9a-fA-F])+ {
if ((YYCURSOR.val - (this->ds_input.udata() + this->ds_next_offset)) == 17) {
RET(DT_MAC_ADDRESS);
} else {
RET(DT_HEX_DUMP);
}
}
(NUM{4}"/"NUM{1,2}"/"NUM{1,2}|NUM{4}"-"NUM{1,2}"-"NUM{1,2}|NUM{2}"/"ALPHA{3}"/"NUM{4})("T"|" ")NUM{2}":"NUM{2}(":"NUM{2}("."NUM{3,6})?)? {
<init, bol> (NUM{4}"/"NUM{1,2}"/"NUM{1,2}|NUM{4}"-"NUM{1,2}"-"NUM{1,2}|NUM{2}"/"ALPHA{3}"/"NUM{4})("T"|" ")NUM{2}":"NUM{2}(":"NUM{2}("."NUM{3,6})?)? {
RET(DT_DATE_TIME);
}
ALPHA{3}(" "NUM|" "NUM{2})" "NUM{2}":"NUM{2}(":"NUM{2}("."NUM{3,6})?)? {
<init, bol> ALPHA{3}(" "NUM|" "NUM{2})" "NUM{2}":"NUM{2}(":"NUM{2}("."NUM{3,6})?)? {
RET(DT_DATE_TIME);
}
(NUM{4}"/"NUM{1,2}"/"NUM{1,2}|NUM{4}"-"NUM{1,2}"-"NUM{1,2}|NUM{2}"/"ALPHA{3}"/"NUM{4}) {
<init, bol> (NUM{4}"/"NUM{1,2}"/"NUM{1,2}|NUM{4}"-"NUM{1,2}"-"NUM{1,2}|NUM{2}"/"ALPHA{3}"/"NUM{4}) {
RET(DT_DATE);
}
IPV6ADDR/[^:a-zA-Z0-9] { RET(DT_IPV6_ADDRESS); }
<init, bol> IPV6ADDR/[^:a-zA-Z0-9] { RET(DT_IPV6_ADDRESS); }
"<!"[a-zA-Z0-9_:\-]+SPACE*([a-zA-Z0-9_:\-]+(SPACE*'='SPACE*('"'(('\\'.|[^\x00"\\])+)'"'|"'"(('\\'.|[^\x00'\\])+)"'"|[^\x00>]+))?|SPACE*('"'(('\\'.|[^\x00"\\])+)'"'|"'"(('\\'.|[^\x00'\\])+)"'"))*SPACE*">" {
<init, bol> "<!"[a-zA-Z0-9_:\-]+SPACE*([a-zA-Z0-9_:\-]+(SPACE*'='SPACE*('"'(('\\'[^\x00]|[^\x00"\\])+)'"'|"'"(('\\'[^\x00]|[^\x00'\\])+)"'"|[^\x00>]+))?|SPACE*('"'(('\\'[^\x00]|[^\x00"\\])+)'"'|"'"(('\\'[^\x00]|[^\x00'\\])+)"'"))*SPACE*">" {
RET(DT_XML_DECL_TAG);
}
"<""?"?[a-zA-Z0-9_:\-]+SPACE*([a-zA-Z0-9_:\-]+(SPACE*'='SPACE*('"'(('\\'.|[^\x00"\\])+)'"'|"'"(('\\'.|[^\x00'\\])+)"'"|[^\x00>]+))?)*SPACE*("/"|"?")">" {
<init, bol> "<""?"?[a-zA-Z0-9_:\-]+SPACE*([a-zA-Z0-9_:\-]+(SPACE*'='SPACE*('"'(('\\'[^\x00]|[^\x00"\\])+)'"'|"'"(('\\'[^\x00]|[^\x00'\\])+)"'"|[^\x00>]+))?)*SPACE*("/"|"?")">" {
RET(DT_XML_EMPTY_TAG);
}
"<"[a-zA-Z0-9_:\-]+SPACE*([a-zA-Z0-9_:\-]+(SPACE*"="SPACE*('"'(('\\'.|[^\x00"\\])+)'"'|"'"(('\\'.|[^\x00'\\])+)"'"|[^\x00>]+))?)*SPACE*">" {
<init, bol> "<"[a-zA-Z0-9_:\-]+SPACE*([a-zA-Z0-9_:\-]+(SPACE*"="SPACE*('"'(('\\'[^\x00]|[^\x00"\\])+)'"'|"'"(('\\'[^\x00]|[^\x00'\\])+)"'"|[^\x00>]+))?)*SPACE*">" {
RET(DT_XML_OPEN_TAG);
}
"</"[a-zA-Z0-9_:\-]+SPACE*">" {
<init, bol> "</"[a-zA-Z0-9_:\-]+SPACE*">" {
RET(DT_XML_CLOSE_TAG);
}
"\n"[A-Z][A-Z _\-0-9]+"\n" {
<bol> [A-Z][A-Z _\-0-9]+"\n" {
CAPTURE(DT_H1);
cap_inner.c_begin += 1;
cap_inner.c_end -= 1;
this->ds_bol = true;
return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
}
"\ndiff --git "[^\n\x00]+"\n" {
<bol> "["[^\x00\n]+"]\n" {
CAPTURE(DT_H1);
cap_inner.c_begin += 1;
cap_inner.c_end = cap_inner.c_begin;
cap_inner.c_end -= 1;
this->ds_bol = true;
return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
}
"--- "[^\n\x00]+"\n+++ "[^\n\x00]+"\n" {
<bol> "diff --git "[^\x00\n]+"\n" {
CAPTURE(DT_H1);
cap_inner.c_end = cap_inner.c_begin;
this->ds_bol = true;
return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
}
<bol> "--- "[^\x00\n]+"\n+++ "[^\x00\n]+"\n" {
CAPTURE(DT_DIFF_FILE_HEADER);
cap_inner.c_end -= 1;
this->ds_bol = true;
return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
}
"@@ -"[0-9]+","[0-9]+" +"[0-9]+","[0-9]+" @@ " @hunk_heading ([^\n\x00]+)"\n" {
<init, bol> "@@ -"[0-9]+","[0-9]+" +"[0-9]+","[0-9]+" @@ " @hunk_heading ([^\x00\n]+)"\n" {
CAPTURE(DT_DIFF_HUNK_HEADING);
cap_inner.c_begin = hunk_heading.val - this->ds_input.udata();
cap_inner.c_end -= 1;
this->ds_bol = true;
return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
}
ESC"["[0-9=;?]*[a-zA-Z] {
<init, bol> ESC"["[0-9=;?]*[a-zA-Z] {
RET(DT_CSI);
}
":" { RET(DT_COLON); }
"=" { RET(DT_EQUALS); }
"," { RET(DT_COMMA); }
";" { RET(DT_SEMI); }
"()" | "{}" | "[]" { RET(DT_EMPTY_CONTAINER); }
"{" { RET(DT_LCURLY); }
"}" { RET(DT_RCURLY); }
"[" { RET(DT_LSQUARE); }
"]" { RET(DT_RSQUARE); }
"(" { RET(DT_LPAREN); }
")" { RET(DT_RPAREN); }
"<" { RET(DT_LANGLE); }
">" { RET(DT_RANGLE); }
<init, bol> ":" { RET(DT_COLON); }
<init, bol> "=" { RET(DT_EQUALS); }
<init, bol> "," { RET(DT_COMMA); }
<init, bol> ";" { RET(DT_SEMI); }
<init, bol> "()" | "{}" | "[]" { RET(DT_EMPTY_CONTAINER); }
<init, bol> "{" { RET(DT_LCURLY); }
<init, bol> "}" { RET(DT_RCURLY); }
<init, bol> "[" { RET(DT_LSQUARE); }
<init, bol> "]" { RET(DT_RSQUARE); }
<init, bol> "(" { RET(DT_LPAREN); }
<init, bol> ")" { RET(DT_RPAREN); }
<init, bol> "<" { RET(DT_LANGLE); }
<init, bol> ">" { RET(DT_RANGLE); }
IPV4ADDR/[^0-9] {
<init, bol> IPV4ADDR/[^0-9] {
RET(DT_IPV4_ADDRESS);
}
[0-9a-fA-F]{8}("-"[0-9a-fA-F]{4}){3}"-"[0-9a-fA-F]{12} { RET(DT_UUID); }
<init, bol> [0-9a-fA-F]{8}("-"[0-9a-fA-F]{4}){3}"-"[0-9a-fA-F]{12} { RET(DT_UUID); }
(NUM{4}" "NUM{4}" "NUM{4}" "NUM{4}|NUM{16})/[^0-9] {
<init, bol> (NUM{4}" "NUM{4}" "NUM{4}" "NUM{4}|NUM{16})/[^0-9] {
CAPTURE(DT_CREDIT_CARD_NUMBER);
if (!this->is_credit_card(this->to_string_fragment(cap_all))) {
if (cap_all.length() > 16) {
@ -309,32 +341,37 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
}
[0-9]"."[0-9]+'e'[\-\+][0-9]+ { RET(DT_NUMBER); }
<init, bol> [0-9]"."[0-9]+'e'[\-\+][0-9]+ { RET(DT_NUMBER); }
[0-9]+("."[0-9]+[a-zA-Z0-9_]*){2,}("-"[a-zA-Z0-9_]+)?|[0-9]+("."[0-9]+[a-zA-Z0-9_]*)+"-"[a-zA-Z0-9_]+ {
<init, bol> [0-9]+("."[0-9]+[a-zA-Z0-9_]*){2,}("-"[a-zA-Z0-9_]+)?|[0-9]+("."[0-9]+[a-zA-Z0-9_]*)+"-"[a-zA-Z0-9_]+ {
RET(DT_VERSION_NUMBER);
}
"-"?"0"[0-7]+ { RET(DT_OCTAL_NUMBER); }
"-"?[0-9]+("."[0-9]+)?[ ]*"%" { RET(DT_PERCENTAGE); }
"-"?[0-9]+("."[0-9]+)?([eE][\-+][0-9]+)? { RET(DT_NUMBER); }
"-"?("0x"|[0-9])[0-9a-fA-F]+ { RET(DT_HEX_NUMBER); }
<init, bol> "-"?"0"[0-7]+ { RET(DT_OCTAL_NUMBER); }
<init, bol> "-"?[0-9]+("."[0-9]+)?[ ]*"%" { RET(DT_PERCENTAGE); }
<init, bol> "-"?[0-9]+("."[0-9]+)?([eE][\-+][0-9]+)? { RET(DT_NUMBER); }
<init, bol> "-"?("0x"|[0-9])[0-9a-fA-F]+ { RET(DT_HEX_NUMBER); }
[a-zA-Z0-9\._%+-]+"@"[a-zA-Z0-9\.-]+"."[a-zA-Z]+ { RET(DT_EMAIL); }
<init, bol> [a-zA-Z0-9\._%+-]+"@"[a-zA-Z0-9\.-]+"."[a-zA-Z]+ { RET(DT_EMAIL); }
"true"|"True"|"TRUE"|"false"|"False"|"FALSE"|"None"|"null"|"NULL"/([\r\n\t \(\)!\*:;'\"\?,]|[\.\!,\?]SPACE|EOF) { RET(DT_CONSTANT); }
<init, bol> "true"|"True"|"TRUE"|"false"|"False"|"FALSE"|"None"|"null"|"NULL"/([\r\n\t \(\)!\*:;'\"\?,]|[\.\!,\?]SPACE|EOF) { RET(DT_CONSTANT); }
("re-")?[a-zA-Z][a-z']+/([\r\n\t \(\)!\*:;'\"\?,]|[\.\!,\?]SPACE|EOF) { RET(DT_WORD); }
<init, bol> ("re-")?[a-zA-Z][a-z']+/([\r\n\t \(\)!\*:;'\"\?,]|[\.\!,\?]SPACE|EOF) { RET(DT_WORD); }
[^\x00\x16\x1b"; \t\r\n:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\.\\][^\x00\x16\x1b"; \t\r\n:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\\]*("::"[^\x00\x16\x1b"; \r\n\t:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\\]+)* {
<init, bol> [^\x00\x16\x1b"; \t\r\n:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\.\\][^\x00\x16\x1b"; \t\r\n:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\\]*("::"[^\x00\x16\x1b"; \r\n\t:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\\]+)* {
RET(DT_SYMBOL);
}
("\r"?"\n"|"\\n") { RET(DT_LINE); }
SPACE+ { RET(DT_WHITE); }
"." { RET(DT_DOT); }
"\\". { RET(DT_ESCAPED_CHAR); }
. { RET(DT_GARBAGE); }
<init, bol> ("\r"?"\n"|"\\n") {
this->ds_bol = true;
RET(DT_LINE);
}
<init, bol> SPACE+ { RET(DT_WHITE); }
<init, bol> "." { RET(DT_DOT); }
<init, bol> "\\". { RET(DT_ESCAPED_CHAR); }
<init, bol> . { RET(DT_GARBAGE); }
*/
return nonstd::nullopt;
}

105
src/document.sections.cc
View File

@ -266,6 +266,20 @@ public:
this->sw_hier_nodes.push_back(std::make_unique<hier_node>());
}
bool is_structured_text() const
{
switch (this->sw_text_format) {
case text_format_t::TF_JSON:
case text_format_t::TF_YAML:
case text_format_t::TF_TOML:
case text_format_t::TF_LOG:
case text_format_t::TF_UNKNOWN:
return true;
default:
return false;
}
}
metadata walk()
{
metadata_builder mb;
@ -345,7 +359,7 @@ public:
this->sw_range.lr_start + inner_cap.c_end,
},
VC_ROLE.value(role_t::VCR_H1));
this->sw_line_number += 2;
this->sw_line_number += 1;
break;
}
case DT_DIFF_FILE_HEADER: {
@ -392,57 +406,68 @@ public:
case DT_LCURLY:
case DT_LSQUARE:
case DT_LPAREN: {
this->flush_values();
// this->append_child_node(term);
this->sw_depth += 1;
this->sw_interval_state.back().is_start
= el.e_capture.c_begin;
this->sw_interval_state.back().is_line_number
= this->sw_line_number;
this->sw_interval_state.resize(this->sw_depth + 1);
this->sw_hier_nodes.push_back(
std::make_unique<hier_node>());
if (this->is_structured_text()) {
this->flush_values();
// this->append_child_node(term);
this->sw_depth += 1;
this->sw_interval_state.back().is_start
= el.e_capture.c_begin;
this->sw_interval_state.back().is_line_number
= this->sw_line_number;
this->sw_interval_state.resize(this->sw_depth + 1);
this->sw_hier_nodes.push_back(
std::make_unique<hier_node>());
} else {
this->sw_values.emplace_back(el);
}
break;
}
case DT_RCURLY:
case DT_RSQUARE:
case DT_RPAREN: {
auto term = this->flush_values();
if (this->sw_depth > 0) {
this->append_child_node(term);
this->sw_depth -= 1;
this->sw_interval_state.pop_back();
this->sw_hier_stage
= std::move(this->sw_hier_nodes.back());
this->sw_hier_nodes.pop_back();
if (this->sw_interval_state.back().is_start) {
data_scanner::capture_t obj_cap = {
static_cast<int>(this->sw_interval_state.back()
.is_start.value()),
el.e_capture.c_end,
};
case DT_RPAREN:
if (this->is_structured_text()) {
auto term = this->flush_values();
if (this->sw_depth > 0) {
this->append_child_node(term);
this->sw_depth -= 1;
this->sw_interval_state.pop_back();
this->sw_hier_stage
= std::move(this->sw_hier_nodes.back());
this->sw_hier_nodes.pop_back();
if (this->sw_interval_state.back().is_start) {
data_scanner::capture_t obj_cap = {
static_cast<int>(
this->sw_interval_state.back()
.is_start.value()),
el.e_capture.c_end,
};
auto sf
= this->sw_scanner.to_string_fragment(obj_cap);
if (!sf.find('\n')) {
this->sw_hier_stage->hn_named_children.clear();
this->sw_hier_stage->hn_children.clear();
while (!this->sw_intervals.empty()
&& this->sw_intervals.back().start
> obj_cap.c_begin)
{
this->sw_intervals.pop_back();
auto sf = this->sw_scanner.to_string_fragment(
obj_cap);
if (!sf.find('\n')) {
this->sw_hier_stage->hn_named_children
.clear();
this->sw_hier_stage->hn_children.clear();
while (!this->sw_intervals.empty()
&& this->sw_intervals.back().start
> obj_cap.c_begin)
{
this->sw_intervals.pop_back();
}
}
}
}
}
this->sw_values.emplace_back(el);
break;
}
case DT_COMMA:
if (this->sw_depth > 0) {
auto term = this->flush_values();
this->append_child_node(term);
if (this->is_structured_text()) {
if (this->sw_depth > 0) {
auto term = this->flush_values();
this->append_child_node(term);
}
} else {
this->sw_values.emplace_back(el);
}
break;
case DT_LINE:

4
src/text_anonymizer.cc
View File

@ -508,6 +508,10 @@ text_anonymizer::next(string_fragment line)
break;
}
default: {
log_debug("tok_re %d %d:%d",
tok_res->tr_token,
tok_res->tr_capture.c_begin,
tok_res->tr_capture.c_end);
retval += tok_res->to_string();
break;
}

2
test/Makefile.am
View File

@ -275,6 +275,8 @@ dist_noinst_DATA = \
datafile_xml.0 \
dhcp.pcapng \
dhcp-trunc.pcapng \
example.patch \
example.toml \
expected_help.txt \
invalid-books.xml \
listview_output.0 \

32
test/datafile_simple.23
View File

@ -2,9 +2,6 @@
key 23:23 ^
sym 23:24 ^ Z
pair 23:24 ^ Z
key 30:30 ^
sym 30:34 ^--^ vpxd
pair 30:34 ^--^ vpxd
key 35:35 ^
num 35:40 ^---^ 47413
val 35:40 ^---^ 47413
@ -59,30 +56,29 @@ uuid 236:272
grp 236:272 ^----------------------------------^ 522e0475-8901-e8b8-1eb8-07ec729ac50c
pair 236:272 ^----------------------------------^ 522e0475-8901-e8b8-1eb8-07ec729ac50c
msg :2022-06-02T12:26:22.072Z info vpxd[47413] [Originator@6876 sub=vpxLro opID=21fa61e9-3e] [VpxLRO] -- BEGIN lro-954041 -- AuthorizationManager -- vim.AuthorizationManager.hasUserPrivilegeOnEntities -- 52768da7-4006-3d4a-4917-ee027373630f(522e0475-8901-e8b8-1eb8-07ec729ac50c)
format :2022-06-02T12:26:22.072# info #[#] [#] [#] # # # # # # # # #(#)
format :2022-06-02T12:26:22.072# info vpxd[#] [#] [#] # # # # # # # # #(#)
{
"col_0": "Z",
"col_1": "vpxd",
"col_2": [
"col_1": [
47413
],
"col_3": {
"col_2": {
"Originator@6876 sub": "vpxLro",
"opID": "21fa61e9-3e"
},
"col_4": [
"col_3": [
"VpxLRO"
],
"col_5": "--",
"col_6": "BEGIN",
"col_7": "lro-954041",
"col_8": "--",
"col_9": "AuthorizationManager",
"col_10": "--",
"col_11": "vim.AuthorizationManager.hasUserPrivilegeOnEntities",
"col_12": "--",
"col_13": "52768da7-4006-3d4a-4917-ee027373630f",
"col_14": [
"col_4": "--",
"col_5": "BEGIN",
"col_6": "lro-954041",
"col_7": "--",
"col_8": "AuthorizationManager",
"col_9": "--",
"col_10": "vim.AuthorizationManager.hasUserPrivilegeOnEntities",
"col_11": "--",
"col_12": "52768da7-4006-3d4a-4917-ee027373630f",
"col_13": [
"522e0475-8901-e8b8-1eb8-07ec729ac50c"
]
}

102
test/example.patch Normal file
View File

@ -0,0 +1,102 @@
[tests] fix export session test
diff --git a/test/Makefile.am b/test/Makefile.am
index aad96a87..07afe2d5 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -483,6 +483,7 @@ distclean-local:
$(RM_V)rm -rf .lnav
$(RM_V)rm -rf regex101-home
$(RM_V)rm -rf events-home
+ $(RM_V)rm -rf support-dump
$(RM_V)rm -rf ../installer-test-home
expected:
diff --git a/test/expected/expected.am b/test/expected/expected.am
index 99823624..2fa3d9fd 100644
--- a/test/expected/expected.am
+++ b/test/expected/expected.am
@@ -414,6 +414,8 @@ EXPECTED_FILES = \
$(srcdir)/%reldir%/test_sessions.sh_b932b33dd087b94d4306dd179c5d4f9ddd394960.out \
$(srcdir)/%reldir%/test_sessions.sh_ddf45811e9906de9f3930fe802ac7b2cc6e48106.err \
$(srcdir)/%reldir%/test_sessions.sh_ddf45811e9906de9f3930fe802ac7b2cc6e48106.out \
+ $(srcdir)/%reldir%/test_sessions.sh_e39648f425c3f291c9d1c0d14595a019abd0cb48.err \
+ $(srcdir)/%reldir%/test_sessions.sh_e39648f425c3f291c9d1c0d14595a019abd0cb48.out \
$(srcdir)/%reldir%/test_shlexer.sh_14dd967cb2af90899c9e5e45d00b676b5a3163aa.err \
$(srcdir)/%reldir%/test_shlexer.sh_14dd967cb2af90899c9e5e45d00b676b5a3163aa.out \
$(srcdir)/%reldir%/test_shlexer.sh_2781f5dd570580cbe746ad91b58a28b8371283b3.err \
diff --git a/test/expected/test_sessions.sh_8732dad5481be991ca7f291d9c5451c7b016cea7.out b/test/expected/test_sessions.sh_8732dad5481be991ca7f291d9c5451c7b016cea7.out
index 7b4c3036..a1e2e6f2 100644
--- a/test/expected/test_sessions.sh_8732dad5481be991ca7f291d9c5451c7b016cea7.out
+++ b/test/expected/test_sessions.sh_8732dad5481be991ca7f291d9c5451c7b016cea7.out
@@ -11,6 +11,7 @@
;INSERT OR IGNORE INTO environ (name, value) VALUES ('LOG_DIR_0', '{top_srcdir_parent}')
:open $LOG_DIR_0/lnav/test/logfile_access_log.0
+
:rebuild
diff --git a/test/expected/test_sessions.sh_e39648f425c3f291c9d1c0d14595a019abd0cb48.err b/test/expected/test_sessions.sh_e39648f425c3f291c9d1c0d14595a019abd0cb48.err
new file mode 100644
index 00000000..e69de29b
diff --git a/test/expected/test_sessions.sh_e39648f425c3f291c9d1c0d14595a019abd0cb48.out b/test/expected/test_sessions.sh_e39648f425c3f291c9d1c0d14595a019abd0cb48.out
new file mode 100644
index 00000000..6cc4bc8c
--- /dev/null
+++ b/test/expected/test_sessions.sh_e39648f425c3f291c9d1c0d14595a019abd0cb48.out
@@ -0,0 +1,33 @@
+#!lnav -Nf
+# This file is an export of an lnav session. You can type
+# '|/path/to/this/file' in lnav to execute this file and
+# restore the state of the session.
+
+;SELECT raise_error('This session export was made with a newer version of lnav, please upgrade to ' || '0.11.0' || ' or later')
+ WHERE lnav_version() < '0.11.0' COLLATE naturalcase
+
+# The files loaded into the session were:
+
+
+;INSERT OR IGNORE INTO environ (name, value) VALUES ('LOG_DIR_0', '{builddir}')
+:open $LOG_DIR_0/support-dump/logfile_access_log.0
+
+:rebuild
+
+
+# The following SQL statements will restore the bookmarks,
+# comments, and tags that were added in the session.
+
+;SELECT total_changes() AS before_mark_changes
+;UPDATE all_logs SET log_mark = 1, log_comment = NULL, log_tags = NULL WHERE log_time_msecs = 1248130769000 AND log_format = 'access_log' AND log_line_hash = 'v1:b05c1bdfe75cde41e151c89087e31951'
+
+;SELECT 1 - (total_changes() - $before_mark_changes) AS failed_mark_changes
+;SELECT echoln(printf('%sERROR%s: failed to restore %d bookmarks',
+ $ansi_red, $ansi_norm, $failed_mark_changes))
+ WHERE $failed_mark_changes != 0
+
+
+# The following commands will restore the state of the LOG view.
+
+:switch-to-view log
+:goto 1
diff --git a/test/test_sessions.sh b/test/test_sessions.sh
index a2c59bbe..31d6d376 100644
--- a/test/test_sessions.sh
+++ b/test/test_sessions.sh
@@ -26,11 +26,15 @@ run_cap_test ${lnav_test} -nq \
-c ":save-session" \
${test_dir}/logfile_access_log.0
+mkdir -p support-dump
+echo 'Hello' > support-dump/readme
+cp ${test_dir}/logfile_access_log.0 support-dump/
+
run_cap_test ${lnav_test} -nq \
-c ";update access_log set log_mark = 1 where sc_bytes > 60000" \
-c ":goto 1" \
-c ":export-session-to -" \
- ${test_dir}/logfile_access_log.0
+ support-dump/logfile_access_log.0
run_cap_test ${lnav_test} -nq \
-c ";update access_log set log_mark = 1 where sc_bytes > 60000" \

38
test/example.toml Normal file
View File

@ -0,0 +1,38 @@
# This is a TOML document
title = "TOML Example"
[owner]
name = "Tom Preston-Werner"
dob = 1979-05-27T07:32:00-08:00
[database]
enabled = true
ports = [
8000,
8001,
8002,
8003,
8004,
8005,
8006,
8007,
8008,
8009,
8010,
8011,
8012,
8013,
]
data = [["delta", "phi"], [3.14]]
temp_targets = { cpu = 79.5, case = 72.0 }
[servers]
[servers.alpha]
ip = "10.0.0.1"
role = "frontend"
[servers.beta]
ip = "10.0.0.2"
role = "backend"

4
test/expected/expected.am
View File

@ -1188,6 +1188,8 @@ EXPECTED_FILES = \
$(srcdir)/%reldir%/test_sql_xml_func.sh_fefeb387ae14d4171225ea06cbbff3ec43990cf0.out \
$(srcdir)/%reldir%/test_sql_yaml_func.sh_dc189d02e8979b7ed245d5d750f68b9965984699.err \
$(srcdir)/%reldir%/test_sql_yaml_func.sh_dc189d02e8979b7ed245d5d750f68b9965984699.out \
$(srcdir)/%reldir%/test_text_file.sh_1ce4056d72b871f8bb844c86aade2a9b1da58030.err \
$(srcdir)/%reldir%/test_text_file.sh_1ce4056d72b871f8bb844c86aade2a9b1da58030.out \
$(srcdir)/%reldir%/test_text_file.sh_4226123565a53b4e3f80e602c1f294721e8e07bf.err \
$(srcdir)/%reldir%/test_text_file.sh_4226123565a53b4e3f80e602c1f294721e8e07bf.out \
$(srcdir)/%reldir%/test_text_file.sh_5b51b55dff7332c5bee2c9b797c401c5614d574a.err \
@ -1214,6 +1216,8 @@ EXPECTED_FILES = \
$(srcdir)/%reldir%/test_text_file.sh_d59b67113864ef5e77267d7fd8ad4072f5aef0fc.out \
$(srcdir)/%reldir%/test_text_file.sh_e088ea61a5382458cc48a2607e2639e52b0be1da.err \
$(srcdir)/%reldir%/test_text_file.sh_e088ea61a5382458cc48a2607e2639e52b0be1da.out \
$(srcdir)/%reldir%/test_text_file.sh_e556fa91b91579df20d38540a5db9cedbaf68a62.err \
$(srcdir)/%reldir%/test_text_file.sh_e556fa91b91579df20d38540a5db9cedbaf68a62.out \
$(srcdir)/%reldir%/test_text_file.sh_f586ef080a86dfe1f981b345bcf8d7a279b2b247.err \
$(srcdir)/%reldir%/test_text_file.sh_f586ef080a86dfe1f981b345bcf8d7a279b2b247.out \
$()

0
test/expected/test_text_file.sh_1ce4056d72b871f8bb844c86aade2a9b1da58030.err Normal file
View File

49
test/expected/test_text_file.sh_1ce4056d72b871f8bb844c86aade2a9b1da58030.out Normal file
View File

@ -0,0 +1,49 @@
[
{
"top_meta": {
"file": "stdin",
"anchor": "#distclean-local-",
"breadcrumbs": [
{
"display_value": "stdin",
"search_placeholder": "",
"possibilities": [
{
"display_value": "stdin"
}
]
},
{
"display_value": "test/Makefile.am",
"search_placeholder": "",
"possibilities": [
{
"display_value": "test/Makefile.am"
},
{
"display_value": "test/expected/expected.am"
},
{
"display_value": "test/expected/test_sessions.sh_8732dad5481be991ca7f291d9c5451c7b016cea7.out"
},
{
"display_value": "test/expected/test_sessions.sh_e39648f425c3f291c9d1c0d14595a019abd0cb48.out"
},
{
"display_value": "test/test_sessions.sh"
}
]
},
{
"display_value": "distclean-local:",
"search_placeholder": "",
"possibilities": [
{
"display_value": "distclean-local:"
}
]
}
]
}
}
]

0
test/expected/test_text_file.sh_e556fa91b91579df20d38540a5db9cedbaf68a62.err Normal file
View File

40
test/expected/test_text_file.sh_e556fa91b91579df20d38540a5db9cedbaf68a62.out Normal file
View File

@ -0,0 +1,40 @@
[
{
"top_meta": {
"file": "stdin",
"anchor": "#-owner-",
"breadcrumbs": [
{
"display_value": "stdin",
"search_placeholder": "",
"possibilities": [
{
"display_value": "stdin"
}
]
},
{
"display_value": "[owner]",
"search_placeholder": "",
"possibilities": [
{
"display_value": "[database]"
},
{
"display_value": "[owner]"
},
{
"display_value": "[servers.alpha]"
},
{
"display_value": "[servers.beta]"
},
{
"display_value": "[servers]"
}
]
}
]
}
}
]

2
test/lnav_doctests.cc
View File

@ -285,7 +285,7 @@ TEST_CASE("data_scanner CSI")
CHECK(tok_res->tr_token == DT_CSI);
CHECK(tok_res->to_string() == "\x1b[32m");
tok_res = ds.tokenize2();
CHECK(tok_res->tr_token == DT_SYMBOL);
CHECK(tok_res->tr_token == DT_WORD);
CHECK(tok_res->to_string() == "Hello");
tok_res = ds.tokenize2();
CHECK(tok_res->tr_token == DT_CSI);

12
test/test_text_file.sh
View File

@ -54,3 +54,15 @@ run_cap_test ${lnav_test} -n \
-c ";SELECT top_meta FROM lnav_views WHERE name = 'text'" \
-c ':write-json-to -' \
< ${test_dir}/man_echo.txt
run_cap_test ${lnav_test} -n \
-c ':goto 6' \
-c ";SELECT top_meta FROM lnav_views WHERE name = 'text'" \
-c ':write-json-to -' \
< ${test_dir}/example.toml
run_cap_test ${lnav_test} -n \
-c ':goto 9' \
-c ";SELECT top_meta FROM lnav_views WHERE name = 'text'" \
-c ':write-json-to -' \
< ${test_dir}/example.patch