diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e8ecf7a..dec5cea2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,7 @@ set(lnav_LIBS ) add_subdirectory(src) -# add_subdirectory(test) +add_subdirectory(test) # ---- Install rules ---- diff --git a/src/data_parser.cc b/src/data_parser.cc index aaec953f..8d18d323 100644 --- a/src/data_parser.cc +++ b/src/data_parser.cc @@ -37,7 +37,8 @@ data_format data_parser::FORMAT_COMMA("comma", DT_INVALID, DT_COMMA); data_format data_parser::FORMAT_EMDASH("emdash", DT_INVALID, DT_EMDASH); data_format data_parser::FORMAT_PLAIN("plain", DT_INVALID, DT_INVALID); -data_parser::data_parser(data_scanner* ds) +data_parser:: +data_parser(data_scanner* ds) : dp_errors("dp_errors", __FILE__, __LINE__), dp_pairs("dp_pairs", __FILE__, __LINE__), dp_msg_format(nullptr), dp_msg_format_begin(ds->get_init_offset()), dp_scanner(ds) @@ -580,6 +581,7 @@ data_parser::discover_format() require(elem.e_capture.c_begin >= 0); require(elem.e_capture.c_end >= 0); + require(elem.e_capture.c_begin <= elem.e_capture.c_end); state_stack.top().update_for_element(elem); switch (elem.e_token) { @@ -1164,14 +1166,16 @@ dfs_comma_next(data_format_state_t state, data_token_t next_token) return retval; } -data_parser::element::element() +data_parser::element:: +element() : e_capture(-1, -1), e_token(DT_INVALID), e_sub_elements(nullptr) { } -data_parser::element::element(data_parser::element_list_t& subs, - data_token_t token, - bool assign_subs_elements) +data_parser::element:: +element(data_parser::element_list_t& subs, + data_token_t token, + bool assign_subs_elements) : e_capture(subs.front().e_capture.c_begin, subs.back().e_capture.c_end), e_token(token), e_sub_elements(nullptr) { @@ -1180,7 +1184,8 @@ data_parser::element::element(data_parser::element_list_t& subs, } } -data_parser::element::element(const data_parser::element& other) +data_parser::element:: +element(const data_parser::element& other) { /* require(other.e_sub_elements == nullptr); */ @@ -1192,7 +1197,8 @@ data_parser::element::element(const data_parser::element& other) } } -data_parser::element::~element() +data_parser::element::~ +element() { delete this->e_sub_elements; this->e_sub_elements = nullptr; @@ -1346,7 +1352,8 @@ data_parser::element::is_value() const } } -data_parser::discover_format_state::discover_format_state() +data_parser::discover_format_state:: +discover_format_state() : dfs_prefix_state(DFS_INIT), dfs_semi_state(DFS_INIT), dfs_comma_state(DFS_INIT) { diff --git a/src/data_scanner_re.cc b/src/data_scanner_re.cc index e3af69b2..1162fc13 100644 --- a/src/data_scanner_re.cc +++ b/src/data_scanner_re.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 3.1 on Thu Apr 18 13:48:53 2024 */ +/* Generated by re2c 3.1 on Tue Jul 2 09:17:01 2024 */ #line 1 "../../lnav/src/data_scanner_re.re" /** * Copyright (c) 2015, Timothy Stack @@ -837,7 +837,7 @@ yy1: yy2: ++YYCURSOR; yy3: -#line 476 "../../lnav/src/data_scanner_re.re" +#line 481 "../../lnav/src/data_scanner_re.re" { RET(DT_GARBAGE); } #line 843 "../../lnav/src/data_scanner_re.cc" yy4: @@ -848,12 +848,12 @@ yy4: if (yych <= ':') goto yy74; goto yy71; yy5: -#line 473 "../../lnav/src/data_scanner_re.re" +#line 478 "../../lnav/src/data_scanner_re.re" { RET(DT_WHITE); } #line 854 "../../lnav/src/data_scanner_re.cc" yy6: ++YYCURSOR; -#line 469 "../../lnav/src/data_scanner_re.re" +#line 474 "../../lnav/src/data_scanner_re.re" { this->ds_bol = true; RET(DT_LINE); @@ -965,12 +965,12 @@ yy13: yy14: yych = *++YYCURSOR; if (yych == ')') goto yy101; -#line 380 "../../lnav/src/data_scanner_re.re" +#line 385 "../../lnav/src/data_scanner_re.re" { RET(DT_LPAREN); } #line 971 "../../lnav/src/data_scanner_re.cc" yy15: ++YYCURSOR; -#line 381 "../../lnav/src/data_scanner_re.re" +#line 386 "../../lnav/src/data_scanner_re.re" { RET(DT_RPAREN); } #line 976 "../../lnav/src/data_scanner_re.cc" yy16: @@ -1001,7 +1001,7 @@ yy16: } yy17: ++YYCURSOR; -#line 372 "../../lnav/src/data_scanner_re.re" +#line 377 "../../lnav/src/data_scanner_re.re" { RET(DT_COMMA); } #line 1007 "../../lnav/src/data_scanner_re.cc" yy18: @@ -1059,7 +1059,7 @@ yy19: } } yy20: -#line 474 "../../lnav/src/data_scanner_re.re" +#line 479 "../../lnav/src/data_scanner_re.re" { RET(DT_DOT); } #line 1065 "../../lnav/src/data_scanner_re.cc" yy21: @@ -1068,7 +1068,7 @@ yy21: if (yych == 'P') goto yy116; goto yy114; yy22: -#line 293 "../../lnav/src/data_scanner_re.re" +#line 298 "../../lnav/src/data_scanner_re.re" { RET(DT_PATH); } #line 1074 "../../lnav/src/data_scanner_re.cc" yy23: @@ -1151,7 +1151,7 @@ yy23: default: goto yy24; } yy24: -#line 429 "../../lnav/src/data_scanner_re.re" +#line 434 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_NUMBER); auto sf = this->to_string_fragment(cap_all); @@ -1409,12 +1409,12 @@ yy28: yych = *(YYMARKER = ++YYCURSOR); if (yych == ':') goto yy146; yy29: -#line 370 "../../lnav/src/data_scanner_re.re" +#line 375 "../../lnav/src/data_scanner_re.re" { RET(DT_COLON); } #line 1415 "../../lnav/src/data_scanner_re.cc" yy30: ++YYCURSOR; -#line 373 "../../lnav/src/data_scanner_re.re" +#line 378 "../../lnav/src/data_scanner_re.re" { RET(DT_SEMI); } #line 1420 "../../lnav/src/data_scanner_re.cc" yy31: @@ -1440,17 +1440,17 @@ yy31: } } yy32: -#line 382 "../../lnav/src/data_scanner_re.re" +#line 387 "../../lnav/src/data_scanner_re.re" { RET(DT_LANGLE); } #line 1446 "../../lnav/src/data_scanner_re.cc" yy33: ++YYCURSOR; -#line 371 "../../lnav/src/data_scanner_re.re" +#line 376 "../../lnav/src/data_scanner_re.re" { RET(DT_EQUALS); } #line 1451 "../../lnav/src/data_scanner_re.cc" yy34: ++YYCURSOR; -#line 383 "../../lnav/src/data_scanner_re.re" +#line 388 "../../lnav/src/data_scanner_re.re" { RET(DT_RANGLE); } #line 1456 "../../lnav/src/data_scanner_re.cc" yy35: @@ -1497,7 +1497,7 @@ yy36: } } yy37: -#line 451 "../../lnav/src/data_scanner_re.re" +#line 456 "../../lnav/src/data_scanner_re.re" { RET(DT_WORD); } #line 1503 "../../lnav/src/data_scanner_re.cc" yy38: @@ -1525,7 +1525,7 @@ yy38: } } yy39: -#line 461 "../../lnav/src/data_scanner_re.re" +#line 466 "../../lnav/src/data_scanner_re.re" { RET(DT_SYMBOL); } @@ -1679,7 +1679,7 @@ yy45: } } yy46: -#line 378 "../../lnav/src/data_scanner_re.re" +#line 383 "../../lnav/src/data_scanner_re.re" { RET(DT_LSQUARE); } #line 1685 "../../lnav/src/data_scanner_re.cc" yy47: @@ -1709,7 +1709,7 @@ yy47: } yy48: ++YYCURSOR; -#line 379 "../../lnav/src/data_scanner_re.re" +#line 384 "../../lnav/src/data_scanner_re.re" { RET(DT_RSQUARE); } #line 1715 "../../lnav/src/data_scanner_re.cc" yy49: @@ -1943,12 +1943,12 @@ yy59: yy60: yych = *++YYCURSOR; if (yych == '}') goto yy101; -#line 376 "../../lnav/src/data_scanner_re.re" +#line 381 "../../lnav/src/data_scanner_re.re" { RET(DT_LCURLY); } #line 1949 "../../lnav/src/data_scanner_re.cc" yy61: ++YYCURSOR; -#line 377 "../../lnav/src/data_scanner_re.re" +#line 382 "../../lnav/src/data_scanner_re.re" { RET(DT_RCURLY); } #line 1954 "../../lnav/src/data_scanner_re.cc" yy62: @@ -2446,7 +2446,7 @@ yy100: yy101: ++YYCURSOR; yy102: -#line 375 "../../lnav/src/data_scanner_re.re" +#line 380 "../../lnav/src/data_scanner_re.re" { RET(DT_EMPTY_CONTAINER); } #line 2452 "../../lnav/src/data_scanner_re.cc" yy103: @@ -2762,7 +2762,7 @@ yy119: } } yy120: -#line 428 "../../lnav/src/data_scanner_re.re" +#line 433 "../../lnav/src/data_scanner_re.re" { RET(DT_PERCENTAGE); } #line 2768 "../../lnav/src/data_scanner_re.cc" yy121: @@ -2872,7 +2872,7 @@ yy124: } } yy125: -#line 427 "../../lnav/src/data_scanner_re.re" +#line 432 "../../lnav/src/data_scanner_re.re" { RET(DT_OCTAL_NUMBER); } #line 2878 "../../lnav/src/data_scanner_re.cc" yy126: @@ -2916,7 +2916,7 @@ yy126: } } yy127: -#line 443 "../../lnav/src/data_scanner_re.re" +#line 448 "../../lnav/src/data_scanner_re.re" { RET(DT_HEX_NUMBER); } #line 2922 "../../lnav/src/data_scanner_re.cc" yy128: @@ -3048,7 +3048,7 @@ yy132: goto yy135; } yy133: -#line 465 "../../lnav/src/data_scanner_re.re" +#line 470 "../../lnav/src/data_scanner_re.re" { RET(DT_ID); } @@ -3716,7 +3716,7 @@ yy155: } yy156: YYCURSOR = yyt3; -#line 449 "../../lnav/src/data_scanner_re.re" +#line 454 "../../lnav/src/data_scanner_re.re" { RET(DT_WORD); } #line 3722 "../../lnav/src/data_scanner_re.cc" yy157: @@ -4309,7 +4309,7 @@ yy184: goto yy73; yy185: ++YYCURSOR; -#line 475 "../../lnav/src/data_scanner_re.re" +#line 480 "../../lnav/src/data_scanner_re.re" { RET(DT_ESCAPED_CHAR); } #line 4315 "../../lnav/src/data_scanner_re.cc" yy186: @@ -4698,7 +4698,7 @@ yy208: goto yy73; yy209: ++YYCURSOR; -#line 366 "../../lnav/src/data_scanner_re.re" +#line 371 "../../lnav/src/data_scanner_re.re" { RET(DT_CSI); } @@ -4822,7 +4822,7 @@ yy217: yyt1 = yyt2; yy218: YYCURSOR = yyt1; -#line 271 "../../lnav/src/data_scanner_re.re" +#line 276 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_QUOTED_STRING); if (tf == text_format_t::TF_RUST) { @@ -4864,7 +4864,7 @@ yy219: } } yy220: -#line 214 "../../lnav/src/data_scanner_re.re" +#line 219 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_QUOTED_STRING); switch (this->ds_input[cap_inner.c_begin]) { @@ -4937,7 +4937,7 @@ yy228: } } yy229: -#line 453 "../../lnav/src/data_scanner_re.re" +#line 458 "../../lnav/src/data_scanner_re.re" { RET(DT_SYMBOL); } @@ -4946,7 +4946,7 @@ yy230: ++YYCURSOR; yy231: YYCURSOR -= 1; -#line 457 "../../lnav/src/data_scanner_re.re" +#line 462 "../../lnav/src/data_scanner_re.re" { RET(DT_SYMBOL); } @@ -5228,7 +5228,7 @@ yy240: ++YYCURSOR; yy241: YYCURSOR = yyt1; -#line 374 "../../lnav/src/data_scanner_re.re" +#line 379 "../../lnav/src/data_scanner_re.re" { RET(DT_EMDASH); } #line 5234 "../../lnav/src/data_scanner_re.cc" yy242: @@ -5838,7 +5838,7 @@ yy273: default: goto yy274; } yy274: -#line 404 "../../lnav/src/data_scanner_re.re" +#line 409 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_NUMBER); auto sf = this->to_string_fragment(cap_all); @@ -6777,7 +6777,7 @@ yy305: ++YYCURSOR; yy306: YYCURSOR = yyt2; -#line 312 "../../lnav/src/data_scanner_re.re" +#line 317 "../../lnav/src/data_scanner_re.re" { RET(DT_IPV6_ADDRESS); } #line 6783 "../../lnav/src/data_scanner_re.cc" yy307: @@ -7281,7 +7281,7 @@ yy324: yy325: ++YYCURSOR; yy326: -#line 322 "../../lnav/src/data_scanner_re.re" +#line 327 "../../lnav/src/data_scanner_re.re" { RET(DT_XML_OPEN_TAG); } @@ -7352,7 +7352,7 @@ yy330: goto yy73; yy331: ++YYCURSOR; -#line 330 "../../lnav/src/data_scanner_re.re" +#line 335 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_H1); cap_inner.c_end -= 1; @@ -8884,7 +8884,7 @@ yy390: goto yy113; } yy391: -#line 237 "../../lnav/src/data_scanner_re.re" +#line 242 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_COMMENT); if (tf == text_format_t::TF_DIFF) { @@ -11016,7 +11016,7 @@ yy456: yy457: ++YYCURSOR; yy458: -#line 314 "../../lnav/src/data_scanner_re.re" +#line 319 "../../lnav/src/data_scanner_re.re" { RET(DT_XML_DECL_TAG); } @@ -11155,7 +11155,7 @@ yy461: yy462: ++YYCURSOR; yy463: -#line 318 "../../lnav/src/data_scanner_re.re" +#line 323 "../../lnav/src/data_scanner_re.re" { RET(DT_XML_EMPTY_TAG); } @@ -11176,7 +11176,7 @@ yy464: } yy465: ++YYCURSOR; -#line 326 "../../lnav/src/data_scanner_re.re" +#line 331 "../../lnav/src/data_scanner_re.re" { RET(DT_XML_CLOSE_TAG); } @@ -12172,7 +12172,7 @@ yy494: } yy495: YYCURSOR = yyt1; -#line 447 "../../lnav/src/data_scanner_re.re" +#line 452 "../../lnav/src/data_scanner_re.re" { RET(DT_CONSTANT); } #line 12178 "../../lnav/src/data_scanner_re.cc" yy496: @@ -12280,7 +12280,7 @@ yy496: } yy497: ++YYCURSOR; -#line 337 "../../lnav/src/data_scanner_re.re" +#line 342 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_H1); cap_inner.c_end -= 1; @@ -12617,7 +12617,7 @@ yy506: if (yych != ']') goto yy73; yy507: ++YYCURSOR; -#line 259 "../../lnav/src/data_scanner_re.re" +#line 264 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_COMMENT); if (tf == text_format_t::TF_DIFF) { @@ -12648,7 +12648,7 @@ yy508: } } yy509: -#line 445 "../../lnav/src/data_scanner_re.re" +#line 450 "../../lnav/src/data_scanner_re.re" { RET(DT_EMAIL); } #line 12654 "../../lnav/src/data_scanner_re.cc" yy510: @@ -13060,7 +13060,7 @@ yy531: } } yy532: -#line 419 "../../lnav/src/data_scanner_re.re" +#line 424 "../../lnav/src/data_scanner_re.re" { RET(DT_VERSION_NUMBER); } @@ -13267,7 +13267,7 @@ yy543: } } yy544: -#line 296 "../../lnav/src/data_scanner_re.re" +#line 301 "../../lnav/src/data_scanner_re.re" { if ((YYCURSOR.val - (this->ds_input.udata() + this->ds_next_offset)) == 17) { RET(DT_MAC_ADDRESS); @@ -13639,7 +13639,7 @@ yy559: } } yy560: -#line 292 "../../lnav/src/data_scanner_re.re" +#line 297 "../../lnav/src/data_scanner_re.re" { RET(DT_URL); } #line 13645 "../../lnav/src/data_scanner_re.cc" yy561: @@ -16915,7 +16915,7 @@ yy678: ++YYCURSOR; yy679: YYCURSOR = yyt1; -#line 294 "../../lnav/src/data_scanner_re.re" +#line 299 "../../lnav/src/data_scanner_re.re" { RET(DT_TIME); } #line 16921 "../../lnav/src/data_scanner_re.cc" yy680: @@ -22661,7 +22661,7 @@ yy894: } } yy895: -#line 248 "../../lnav/src/data_scanner_re.re" +#line 253 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_COMMENT); if (tf == text_format_t::TF_DIFF) { @@ -24496,7 +24496,7 @@ yy987: yyt1 = yyt2; yy988: YYCURSOR = yyt1; -#line 295 "../../lnav/src/data_scanner_re.re" +#line 300 "../../lnav/src/data_scanner_re.re" { RET(DT_TIME); } #line 24502 "../../lnav/src/data_scanner_re.cc" yy989: @@ -24549,7 +24549,7 @@ yy998: ++YYCURSOR; yy999: YYCURSOR = yyt1; -#line 385 "../../lnav/src/data_scanner_re.re" +#line 390 "../../lnav/src/data_scanner_re.re" { RET(DT_IPV4_ADDRESS); } @@ -25002,7 +25002,7 @@ yy1020: } } yy1021: -#line 309 "../../lnav/src/data_scanner_re.re" +#line 314 "../../lnav/src/data_scanner_re.re" { RET(DT_DATE); } @@ -36291,7 +36291,7 @@ yy1360: } yy1361: ++YYCURSOR; -#line 351 "../../lnav/src/data_scanner_re.re" +#line 356 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_DIFF_FILE_HEADER); cap_inner.c_end -= 1; @@ -37792,7 +37792,7 @@ yy1399: yych = *(YYMARKER = ++YYCURSOR); if (yych == ':') goto yy1438; yy1400: -#line 306 "../../lnav/src/data_scanner_re.re" +#line 311 "../../lnav/src/data_scanner_re.re" { RET(DT_DATE_TIME); } @@ -38886,7 +38886,7 @@ yy1438: goto yy73; yy1439: ++YYCURSOR; -#line 344 "../../lnav/src/data_scanner_re.re" +#line 349 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_H1); cap_inner.c_end = cap_inner.c_begin; @@ -38951,7 +38951,7 @@ yy1443: yych = *(YYMARKER = ++YYCURSOR); if (yych == ':') goto yy1467; yy1444: -#line 303 "../../lnav/src/data_scanner_re.re" +#line 308 "../../lnav/src/data_scanner_re.re" { RET(DT_DATE_TIME); } @@ -41429,7 +41429,7 @@ yy1506: ++YYCURSOR; yy1507: YYCURSOR = yyt1; -#line 391 "../../lnav/src/data_scanner_re.re" +#line 396 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_CREDIT_CARD_NUMBER); if (!this->is_credit_card(this->to_string_fragment(cap_all))) { @@ -42420,7 +42420,7 @@ yy1551: yy1552: ++YYCURSOR; hunk_heading = yyt1; -#line 358 "../../lnav/src/data_scanner_re.re" +#line 363 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_DIFF_HUNK_HEADING); cap_inner.c_begin = hunk_heading.val - this->ds_input.udata(); @@ -43243,7 +43243,7 @@ yy1608: } } yy1609: -#line 389 "../../lnav/src/data_scanner_re.re" +#line 394 "../../lnav/src/data_scanner_re.re" { RET(DT_UUID); } #line 43249 "../../lnav/src/data_scanner_re.cc" yy1610: @@ -43440,7 +43440,7 @@ yy1612: yy1613: ++YYCURSOR; yy1614: -#line 476 "../../lnav/src/data_scanner_re.re" +#line 481 "../../lnav/src/data_scanner_re.re" { RET(DT_GARBAGE); } #line 43446 "../../lnav/src/data_scanner_re.cc" yy1615: @@ -43451,12 +43451,12 @@ yy1615: if (yych <= ':') goto yy1683; goto yy1680; yy1616: -#line 473 "../../lnav/src/data_scanner_re.re" +#line 478 "../../lnav/src/data_scanner_re.re" { RET(DT_WHITE); } #line 43457 "../../lnav/src/data_scanner_re.cc" yy1617: ++YYCURSOR; -#line 469 "../../lnav/src/data_scanner_re.re" +#line 474 "../../lnav/src/data_scanner_re.re" { this->ds_bol = true; RET(DT_LINE); @@ -43568,12 +43568,12 @@ yy1624: yy1625: yych = *++YYCURSOR; if (yych == ')') goto yy1710; -#line 380 "../../lnav/src/data_scanner_re.re" +#line 385 "../../lnav/src/data_scanner_re.re" { RET(DT_LPAREN); } #line 43574 "../../lnav/src/data_scanner_re.cc" yy1626: ++YYCURSOR; -#line 381 "../../lnav/src/data_scanner_re.re" +#line 386 "../../lnav/src/data_scanner_re.re" { RET(DT_RPAREN); } #line 43579 "../../lnav/src/data_scanner_re.cc" yy1627: @@ -43604,7 +43604,7 @@ yy1627: } yy1628: ++YYCURSOR; -#line 372 "../../lnav/src/data_scanner_re.re" +#line 377 "../../lnav/src/data_scanner_re.re" { RET(DT_COMMA); } #line 43610 "../../lnav/src/data_scanner_re.cc" yy1629: @@ -43662,7 +43662,7 @@ yy1630: } } yy1631: -#line 474 "../../lnav/src/data_scanner_re.re" +#line 479 "../../lnav/src/data_scanner_re.re" { RET(DT_DOT); } #line 43668 "../../lnav/src/data_scanner_re.cc" yy1632: @@ -43671,7 +43671,7 @@ yy1632: if (yych == 'P') goto yy1724; goto yy1722; yy1633: -#line 293 "../../lnav/src/data_scanner_re.re" +#line 298 "../../lnav/src/data_scanner_re.re" { RET(DT_PATH); } #line 43677 "../../lnav/src/data_scanner_re.cc" yy1634: @@ -43754,7 +43754,7 @@ yy1634: default: goto yy1635; } yy1635: -#line 429 "../../lnav/src/data_scanner_re.re" +#line 434 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_NUMBER); auto sf = this->to_string_fragment(cap_all); @@ -44012,12 +44012,12 @@ yy1639: yych = *(YYMARKER = ++YYCURSOR); if (yych == ':') goto yy1754; yy1640: -#line 370 "../../lnav/src/data_scanner_re.re" +#line 375 "../../lnav/src/data_scanner_re.re" { RET(DT_COLON); } #line 44018 "../../lnav/src/data_scanner_re.cc" yy1641: ++YYCURSOR; -#line 373 "../../lnav/src/data_scanner_re.re" +#line 378 "../../lnav/src/data_scanner_re.re" { RET(DT_SEMI); } #line 44023 "../../lnav/src/data_scanner_re.cc" yy1642: @@ -44043,17 +44043,17 @@ yy1642: } } yy1643: -#line 382 "../../lnav/src/data_scanner_re.re" +#line 387 "../../lnav/src/data_scanner_re.re" { RET(DT_LANGLE); } #line 44049 "../../lnav/src/data_scanner_re.cc" yy1644: ++YYCURSOR; -#line 371 "../../lnav/src/data_scanner_re.re" +#line 376 "../../lnav/src/data_scanner_re.re" { RET(DT_EQUALS); } #line 44054 "../../lnav/src/data_scanner_re.cc" yy1645: ++YYCURSOR; -#line 383 "../../lnav/src/data_scanner_re.re" +#line 388 "../../lnav/src/data_scanner_re.re" { RET(DT_RANGLE); } #line 44059 "../../lnav/src/data_scanner_re.cc" yy1646: @@ -44098,7 +44098,7 @@ yy1647: } } yy1648: -#line 451 "../../lnav/src/data_scanner_re.re" +#line 456 "../../lnav/src/data_scanner_re.re" { RET(DT_WORD); } #line 44104 "../../lnav/src/data_scanner_re.cc" yy1649: @@ -44125,7 +44125,7 @@ yy1649: } } yy1650: -#line 461 "../../lnav/src/data_scanner_re.re" +#line 466 "../../lnav/src/data_scanner_re.re" { RET(DT_SYMBOL); } @@ -44251,7 +44251,7 @@ yy1655: yy1656: yych = *++YYCURSOR; if (yych == ']') goto yy1710; -#line 378 "../../lnav/src/data_scanner_re.re" +#line 383 "../../lnav/src/data_scanner_re.re" { RET(DT_LSQUARE); } #line 44257 "../../lnav/src/data_scanner_re.cc" yy1657: @@ -44281,7 +44281,7 @@ yy1657: } yy1658: ++YYCURSOR; -#line 379 "../../lnav/src/data_scanner_re.re" +#line 384 "../../lnav/src/data_scanner_re.re" { RET(DT_RSQUARE); } #line 44287 "../../lnav/src/data_scanner_re.cc" yy1659: @@ -44491,12 +44491,12 @@ yy1668: yy1669: yych = *++YYCURSOR; if (yych == '}') goto yy1710; -#line 376 "../../lnav/src/data_scanner_re.re" +#line 381 "../../lnav/src/data_scanner_re.re" { RET(DT_LCURLY); } #line 44497 "../../lnav/src/data_scanner_re.cc" yy1670: ++YYCURSOR; -#line 377 "../../lnav/src/data_scanner_re.re" +#line 382 "../../lnav/src/data_scanner_re.re" { RET(DT_RCURLY); } #line 44502 "../../lnav/src/data_scanner_re.cc" yy1671: @@ -44985,7 +44985,7 @@ yy1709: goto yy1682; yy1710: ++YYCURSOR; -#line 375 "../../lnav/src/data_scanner_re.re" +#line 380 "../../lnav/src/data_scanner_re.re" { RET(DT_EMPTY_CONTAINER); } #line 44991 "../../lnav/src/data_scanner_re.cc" yy1711: @@ -45301,7 +45301,7 @@ yy1727: } } yy1728: -#line 428 "../../lnav/src/data_scanner_re.re" +#line 433 "../../lnav/src/data_scanner_re.re" { RET(DT_PERCENTAGE); } #line 45307 "../../lnav/src/data_scanner_re.cc" yy1729: @@ -45411,7 +45411,7 @@ yy1732: } } yy1733: -#line 427 "../../lnav/src/data_scanner_re.re" +#line 432 "../../lnav/src/data_scanner_re.re" { RET(DT_OCTAL_NUMBER); } #line 45417 "../../lnav/src/data_scanner_re.cc" yy1734: @@ -45455,7 +45455,7 @@ yy1734: } } yy1735: -#line 443 "../../lnav/src/data_scanner_re.re" +#line 448 "../../lnav/src/data_scanner_re.re" { RET(DT_HEX_NUMBER); } #line 45461 "../../lnav/src/data_scanner_re.cc" yy1736: @@ -45587,7 +45587,7 @@ yy1740: goto yy1743; } yy1741: -#line 465 "../../lnav/src/data_scanner_re.re" +#line 470 "../../lnav/src/data_scanner_re.re" { RET(DT_ID); } @@ -46247,7 +46247,7 @@ yy1761: } yy1762: YYCURSOR = yyt3; -#line 449 "../../lnav/src/data_scanner_re.re" +#line 454 "../../lnav/src/data_scanner_re.re" { RET(DT_WORD); } #line 46253 "../../lnav/src/data_scanner_re.cc" yy1763: @@ -46680,7 +46680,7 @@ yy1778: } yy1779: ++YYCURSOR; -#line 475 "../../lnav/src/data_scanner_re.re" +#line 480 "../../lnav/src/data_scanner_re.re" { RET(DT_ESCAPED_CHAR); } #line 46686 "../../lnav/src/data_scanner_re.cc" yy1780: @@ -46935,7 +46935,7 @@ yy1796: goto yy1682; yy1797: ++YYCURSOR; -#line 366 "../../lnav/src/data_scanner_re.re" +#line 371 "../../lnav/src/data_scanner_re.re" { RET(DT_CSI); } @@ -47059,7 +47059,7 @@ yy1805: yyt1 = yyt2; yy1806: YYCURSOR = yyt1; -#line 271 "../../lnav/src/data_scanner_re.re" +#line 276 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_QUOTED_STRING); if (tf == text_format_t::TF_RUST) { @@ -47101,7 +47101,7 @@ yy1807: } } yy1808: -#line 214 "../../lnav/src/data_scanner_re.re" +#line 219 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_QUOTED_STRING); switch (this->ds_input[cap_inner.c_begin]) { @@ -47174,7 +47174,7 @@ yy1816: } } yy1817: -#line 453 "../../lnav/src/data_scanner_re.re" +#line 458 "../../lnav/src/data_scanner_re.re" { RET(DT_SYMBOL); } @@ -47183,7 +47183,7 @@ yy1818: ++YYCURSOR; yy1819: YYCURSOR -= 1; -#line 457 "../../lnav/src/data_scanner_re.re" +#line 462 "../../lnav/src/data_scanner_re.re" { RET(DT_SYMBOL); } @@ -47465,7 +47465,7 @@ yy1828: ++YYCURSOR; yy1829: YYCURSOR = yyt1; -#line 374 "../../lnav/src/data_scanner_re.re" +#line 379 "../../lnav/src/data_scanner_re.re" { RET(DT_EMDASH); } #line 47471 "../../lnav/src/data_scanner_re.cc" yy1830: @@ -48071,7 +48071,7 @@ yy1860: default: goto yy1861; } yy1861: -#line 404 "../../lnav/src/data_scanner_re.re" +#line 409 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_NUMBER); auto sf = this->to_string_fragment(cap_all); @@ -49010,7 +49010,7 @@ yy1892: ++YYCURSOR; yy1893: YYCURSOR = yyt2; -#line 312 "../../lnav/src/data_scanner_re.re" +#line 317 "../../lnav/src/data_scanner_re.re" { RET(DT_IPV6_ADDRESS); } #line 49016 "../../lnav/src/data_scanner_re.cc" yy1894: @@ -49514,7 +49514,7 @@ yy1911: yy1912: ++YYCURSOR; yy1913: -#line 322 "../../lnav/src/data_scanner_re.re" +#line 327 "../../lnav/src/data_scanner_re.re" { RET(DT_XML_OPEN_TAG); } @@ -50868,7 +50868,7 @@ yy1966: goto yy1721; } yy1967: -#line 237 "../../lnav/src/data_scanner_re.re" +#line 242 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_COMMENT); if (tf == text_format_t::TF_DIFF) { @@ -53000,7 +53000,7 @@ yy2032: yy2033: ++YYCURSOR; yy2034: -#line 314 "../../lnav/src/data_scanner_re.re" +#line 319 "../../lnav/src/data_scanner_re.re" { RET(DT_XML_DECL_TAG); } @@ -53139,7 +53139,7 @@ yy2037: yy2038: ++YYCURSOR; yy2039: -#line 318 "../../lnav/src/data_scanner_re.re" +#line 323 "../../lnav/src/data_scanner_re.re" { RET(DT_XML_EMPTY_TAG); } @@ -53160,7 +53160,7 @@ yy2040: } yy2041: ++YYCURSOR; -#line 326 "../../lnav/src/data_scanner_re.re" +#line 331 "../../lnav/src/data_scanner_re.re" { RET(DT_XML_CLOSE_TAG); } @@ -54091,7 +54091,7 @@ yy2067: } yy2068: YYCURSOR = yyt1; -#line 447 "../../lnav/src/data_scanner_re.re" +#line 452 "../../lnav/src/data_scanner_re.re" { RET(DT_CONSTANT); } #line 54097 "../../lnav/src/data_scanner_re.cc" yy2069: @@ -54478,7 +54478,7 @@ yy2077: if (yych != ']') goto yy1682; yy2078: ++YYCURSOR; -#line 259 "../../lnav/src/data_scanner_re.re" +#line 264 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_COMMENT); if (tf == text_format_t::TF_DIFF) { @@ -54509,7 +54509,7 @@ yy2079: } } yy2080: -#line 445 "../../lnav/src/data_scanner_re.re" +#line 450 "../../lnav/src/data_scanner_re.re" { RET(DT_EMAIL); } #line 54515 "../../lnav/src/data_scanner_re.cc" yy2081: @@ -54867,7 +54867,7 @@ yy2094: } } yy2095: -#line 419 "../../lnav/src/data_scanner_re.re" +#line 424 "../../lnav/src/data_scanner_re.re" { RET(DT_VERSION_NUMBER); } @@ -55074,7 +55074,7 @@ yy2106: } } yy2107: -#line 296 "../../lnav/src/data_scanner_re.re" +#line 301 "../../lnav/src/data_scanner_re.re" { if ((YYCURSOR.val - (this->ds_input.udata() + this->ds_next_offset)) == 17) { RET(DT_MAC_ADDRESS); @@ -55446,7 +55446,7 @@ yy2122: } } yy2123: -#line 292 "../../lnav/src/data_scanner_re.re" +#line 297 "../../lnav/src/data_scanner_re.re" { RET(DT_URL); } #line 55452 "../../lnav/src/data_scanner_re.cc" yy2124: @@ -58646,7 +58646,7 @@ yy2236: ++YYCURSOR; yy2237: YYCURSOR = yyt1; -#line 294 "../../lnav/src/data_scanner_re.re" +#line 299 "../../lnav/src/data_scanner_re.re" { RET(DT_TIME); } #line 58652 "../../lnav/src/data_scanner_re.cc" yy2238: @@ -64363,7 +64363,7 @@ yy2447: } } yy2448: -#line 248 "../../lnav/src/data_scanner_re.re" +#line 253 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_COMMENT); if (tf == text_format_t::TF_DIFF) { @@ -66176,7 +66176,7 @@ yy2537: yyt1 = yyt2; yy2538: YYCURSOR = yyt1; -#line 295 "../../lnav/src/data_scanner_re.re" +#line 300 "../../lnav/src/data_scanner_re.re" { RET(DT_TIME); } #line 66182 "../../lnav/src/data_scanner_re.cc" yy2539: @@ -66225,7 +66225,7 @@ yy2547: ++YYCURSOR; yy2548: YYCURSOR = yyt1; -#line 385 "../../lnav/src/data_scanner_re.re" +#line 390 "../../lnav/src/data_scanner_re.re" { RET(DT_IPV4_ADDRESS); } @@ -66678,7 +66678,7 @@ yy2569: } } yy2570: -#line 309 "../../lnav/src/data_scanner_re.re" +#line 314 "../../lnav/src/data_scanner_re.re" { RET(DT_DATE); } @@ -79290,7 +79290,7 @@ yy2926: yych = *(YYMARKER = ++YYCURSOR); if (yych == ':') goto yy2956; yy2927: -#line 306 "../../lnav/src/data_scanner_re.re" +#line 311 "../../lnav/src/data_scanner_re.re" { RET(DT_DATE_TIME); } @@ -80381,7 +80381,7 @@ yy2960: yych = *(YYMARKER = ++YYCURSOR); if (yych == ':') goto yy2983; yy2961: -#line 303 "../../lnav/src/data_scanner_re.re" +#line 308 "../../lnav/src/data_scanner_re.re" { RET(DT_DATE_TIME); } @@ -82795,7 +82795,7 @@ yy3020: ++YYCURSOR; yy3021: YYCURSOR = yyt1; -#line 391 "../../lnav/src/data_scanner_re.re" +#line 396 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_CREDIT_CARD_NUMBER); if (!this->is_credit_card(this->to_string_fragment(cap_all))) { @@ -83774,7 +83774,7 @@ yy3064: yy3065: ++YYCURSOR; hunk_heading = yyt1; -#line 358 "../../lnav/src/data_scanner_re.re" +#line 363 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_DIFF_HUNK_HEADING); cap_inner.c_begin = hunk_heading.val - this->ds_input.udata(); @@ -84318,7 +84318,7 @@ yy3103: } } yy3104: -#line 389 "../../lnav/src/data_scanner_re.re" +#line 394 "../../lnav/src/data_scanner_re.re" { RET(DT_UUID); } #line 84324 "../../lnav/src/data_scanner_re.cc" /* *********************************** */ @@ -84329,58 +84329,66 @@ yyc_dbldocstring: goto yy3107; yy3106: ++YYCURSOR; -#line 204 "../../lnav/src/data_scanner_re.re" +#line 210 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_QUOTED_STRING); - cap_inner.c_end -= 3; return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()}; } -#line 84339 "../../lnav/src/data_scanner_re.cc" +#line 84338 "../../lnav/src/data_scanner_re.cc" yy3107: ++YYCURSOR; yy3108: -#line 210 "../../lnav/src/data_scanner_re.re" +#line 215 "../../lnav/src/data_scanner_re.re" { goto yyc_dbldocstring; } -#line 84347 "../../lnav/src/data_scanner_re.cc" +#line 84346 "../../lnav/src/data_scanner_re.cc" yy3109: yych = *(YYMARKER = ++YYCURSOR); if (yych != '"') goto yy3108; yych = *++YYCURSOR; - if (yych == '"') goto yy3106; + if (yych == '"') goto yy3110; YYCURSOR = YYMARKER; goto yy3108; +yy3110: + ++YYCURSOR; +#line 204 "../../lnav/src/data_scanner_re.re" + { + CAPTURE(DT_QUOTED_STRING); + cap_inner.c_end -= 3; + return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()}; + } +#line 84362 "../../lnav/src/data_scanner_re.cc" /* *********************************** */ yyc_sdocstring: yych = *YYCURSOR; - if (yych <= 0x00) goto yy3111; - if (yych == '\'') goto yy3114; - goto yy3112; -yy3111: + if (yych <= 0x00) goto yy3112; + if (yych == '\'') goto yy3115; + goto yy3113; +yy3112: ++YYCURSOR; -#line 227 "../../lnav/src/data_scanner_re.re" +#line 232 "../../lnav/src/data_scanner_re.re" { CAPTURE(DT_QUOTED_STRING); cap_inner.c_end -= 3; return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()}; } -#line 84369 "../../lnav/src/data_scanner_re.cc" -yy3112: - ++YYCURSOR; +#line 84377 "../../lnav/src/data_scanner_re.cc" yy3113: -#line 233 "../../lnav/src/data_scanner_re.re" + ++YYCURSOR; +yy3114: +#line 238 "../../lnav/src/data_scanner_re.re" { goto yyc_sdocstring; } -#line 84377 "../../lnav/src/data_scanner_re.cc" -yy3114: +#line 84385 "../../lnav/src/data_scanner_re.cc" +yy3115: yych = *(YYMARKER = ++YYCURSOR); - if (yych != '\'') goto yy3113; + if (yych != '\'') goto yy3114; yych = *++YYCURSOR; - if (yych == '\'') goto yy3111; + if (yych == '\'') goto yy3112; YYCURSOR = YYMARKER; - goto yy3113; + goto yy3114; /* *********************************** */ yyc_units: yych = *YYCURSOR; @@ -84388,91 +84396,91 @@ yyc_units: case 'B': case 'S': case 'b': - case 's': goto yy3118; + case 's': goto yy3119; case 'G': case 'K': case 'M': case 'g': - case 'k': goto yy3120; - case 'm': goto yy3121; + case 'k': goto yy3121; + case 'm': goto yy3122; case 'p': - case 'u': goto yy3122; - default: goto yy3116; + case 'u': goto yy3123; + default: goto yy3117; } -yy3116: - ++YYCURSOR; yy3117: + ++YYCURSOR; +yy3118: #line 173 "../../lnav/src/data_scanner_re.re" { return std::nullopt; } -#line 84408 "../../lnav/src/data_scanner_re.cc" -yy3118: - ++YYCURSOR; +#line 84416 "../../lnav/src/data_scanner_re.cc" yy3119: -#line 423 "../../lnav/src/data_scanner_re.re" + ++YYCURSOR; +yy3120: +#line 428 "../../lnav/src/data_scanner_re.re" { RET(DT_UNIT); } -#line 84416 "../../lnav/src/data_scanner_re.cc" -yy3120: +#line 84424 "../../lnav/src/data_scanner_re.cc" +yy3121: yyaccept = 0; yych = *(YYMARKER = ++YYCURSOR); if (yych <= 'a') { - if (yych == 'B') goto yy3118; - goto yy3117; + if (yych == 'B') goto yy3119; + goto yy3118; } else { - if (yych <= 'b') goto yy3118; - if (yych == 'i') goto yy3123; - goto yy3117; + if (yych <= 'b') goto yy3119; + if (yych == 'i') goto yy3124; + goto yy3118; } -yy3121: +yy3122: yyaccept = 1; yych = *(YYMARKER = ++YYCURSOR); if (yych <= 'a') { if (yych <= 'B') { - if (yych <= 'A') goto yy3119; - goto yy3118; - } else { - if (yych == 'S') goto yy3118; + if (yych <= 'A') goto yy3120; goto yy3119; + } else { + if (yych == 'S') goto yy3119; + goto yy3120; } } else { if (yych <= 'i') { - if (yych <= 'b') goto yy3118; - if (yych <= 'h') goto yy3119; - goto yy3125; + if (yych <= 'b') goto yy3119; + if (yych <= 'h') goto yy3120; + goto yy3126; } else { - if (yych == 's') goto yy3118; - goto yy3119; + if (yych == 's') goto yy3119; + goto yy3120; } } -yy3122: - yych = *++YYCURSOR; - if (yych == 'S') goto yy3118; - if (yych == 's') goto yy3118; - goto yy3117; yy3123: yych = *++YYCURSOR; - if (yych == 'B') goto yy3118; - if (yych == 'b') goto yy3118; + if (yych == 'S') goto yy3119; + if (yych == 's') goto yy3119; + goto yy3118; yy3124: + yych = *++YYCURSOR; + if (yych == 'B') goto yy3119; + if (yych == 'b') goto yy3119; +yy3125: YYCURSOR = YYMARKER; if (yyaccept == 0) { - goto yy3117; + goto yy3118; } else { - goto yy3119; + goto yy3120; } -yy3125: +yy3126: yych = *++YYCURSOR; if (yych <= 'a') { - if (yych == 'B') goto yy3118; - goto yy3124; + if (yych == 'B') goto yy3119; + goto yy3125; } else { - if (yych <= 'b') goto yy3118; - if (yych == 'n') goto yy3118; - goto yy3124; + if (yych <= 'b') goto yy3119; + if (yych == 'n') goto yy3119; + goto yy3125; } } -#line 478 "../../lnav/src/data_scanner_re.re" +#line 483 "../../lnav/src/data_scanner_re.re" return std::nullopt; diff --git a/src/data_scanner_re.re b/src/data_scanner_re.re index cda50100..d5c7e37c 100644 --- a/src/data_scanner_re.re +++ b/src/data_scanner_re.re @@ -201,12 +201,17 @@ std::optional data_scanner::tokenize_int(text_for goto yyc_dbldocstring; } - ([\x00]|'"""') { + '"""' { CAPTURE(DT_QUOTED_STRING); cap_inner.c_end -= 3; return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()}; } + [\x00] { + CAPTURE(DT_QUOTED_STRING); + return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()}; + } + * { goto yyc_dbldocstring; } diff --git a/test/Makefile.am b/test/Makefile.am index c81e3779..6df60bf5 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -293,6 +293,7 @@ dist_noinst_DATA = \ datafile_simple.20 \ datafile_simple.21 \ datafile_simple.22 \ + datafile_simple.26 \ datafile_xml.0 \ dhcp.pcapng \ dhcp-trunc.pcapng \ diff --git a/test/datafile_simple.26 b/test/datafile_simple.26 new file mode 100644 index 00000000..c073f348 --- /dev/null +++ b/test/datafile_simple.26 @@ -0,0 +1,20 @@ + a=1 b=2 """ + key 0:1 ^ a + num 2:3 ^ 1 + val 2:3 ^ 1 +pair 0:3 ^-^ a=1 + key 4:5 ^ b + num 6:7 ^ 2 + val 6:7 ^ 2 +pair 4:7 ^-^ b=2 + key 11:11 ^ +quot 11:12 ^ + val 11:12 ^ +pair 11:12 ^ +msg :a=1 b=2 """ +format :a=# b=# ""# +{ + "a": 1, + "b": 2, + "col_0": "\u0000" +} diff --git a/test/lnav_doctests.cc b/test/lnav_doctests.cc index d56c5fe0..8e1252ed 100644 --- a/test/lnav_doctests.cc +++ b/test/lnav_doctests.cc @@ -30,6 +30,8 @@ #include "config.h" #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include + #include "base/from_trait.hh" #include "byte_array.hh" #include "data_scanner.hh" @@ -292,3 +294,29 @@ TEST_CASE("data_scanner CSI") CHECK(tok_res->tr_token == DT_CSI); CHECK(tok_res->to_string() == "\x1b[0m"); } + +TEST_CASE("data_scanner quote") +{ + static const char INPUT[] = "abc \"\"\"\n"; + + { + data_scanner ds(string_fragment::from_const(INPUT)); + + auto tok_res = ds.tokenize2(); + CHECK(tok_res->tr_token == DT_WORD); + CHECK(tok_res->to_string() == "abc"); + tok_res = ds.tokenize2(); + CHECK(tok_res->tr_token == DT_WHITE); + tok_res = ds.tokenize2(); + CHECK(tok_res->tr_token == DT_QUOTED_STRING); + tok_res = ds.tokenize2(); + CHECK_FALSE(tok_res.has_value()); + } + + { + data_scanner ds(string_fragment::from_const(INPUT)); + data_parser dp(&ds); + + dp.parse(); + } +}