[data_scanner] fix for non-terminated quoted string

Fixes #1278

CMakeLists.txt

@@ -37,7 +37,7 @@ set(lnav_LIBS
 )
 
 add_subdirectory(src)
-# add_subdirectory(test)
+add_subdirectory(test)
 
 # ---- Install rules ----
 

src/data_parser.cc

@@ -37,7 +37,8 @@ data_format data_parser::FORMAT_COMMA("comma", DT_INVALID, DT_COMMA);
 data_format data_parser::FORMAT_EMDASH("emdash", DT_INVALID, DT_EMDASH);
 data_format data_parser::FORMAT_PLAIN("plain", DT_INVALID, DT_INVALID);
 
-data_parser::data_parser(data_scanner* ds)
+data_parser::
+data_parser(data_scanner* ds)
     : dp_errors("dp_errors", __FILE__, __LINE__),
       dp_pairs("dp_pairs", __FILE__, __LINE__), dp_msg_format(nullptr),
       dp_msg_format_begin(ds->get_init_offset()), dp_scanner(ds)
@@ -580,6 +581,7 @@ data_parser::discover_format()
 
         require(elem.e_capture.c_begin >= 0);
         require(elem.e_capture.c_end >= 0);
+        require(elem.e_capture.c_begin <= elem.e_capture.c_end);
 
         state_stack.top().update_for_element(elem);
         switch (elem.e_token) {
@@ -1164,14 +1166,16 @@ dfs_comma_next(data_format_state_t state, data_token_t next_token)
     return retval;
 }
 
-data_parser::element::element()
+data_parser::element::
+element()
     : e_capture(-1, -1), e_token(DT_INVALID), e_sub_elements(nullptr)
 {
 }
 
-data_parser::element::element(data_parser::element_list_t& subs,
-                              data_token_t token,
-                              bool assign_subs_elements)
+data_parser::element::
+element(data_parser::element_list_t& subs,
+        data_token_t token,
+        bool assign_subs_elements)
     : e_capture(subs.front().e_capture.c_begin, subs.back().e_capture.c_end),
       e_token(token), e_sub_elements(nullptr)
 {
@@ -1180,7 +1184,8 @@ data_parser::element::element(data_parser::element_list_t& subs,
     }
 }
 
-data_parser::element::element(const data_parser::element& other)
+data_parser::element::
+element(const data_parser::element& other)
 {
     /* require(other.e_sub_elements == nullptr); */
 
@@ -1192,7 +1197,8 @@ data_parser::element::element(const data_parser::element& other)
     }
 }
 
-data_parser::element::~element()
+data_parser::element::~
+element()
 {
     delete this->e_sub_elements;
     this->e_sub_elements = nullptr;
@@ -1346,7 +1352,8 @@ data_parser::element::is_value() const
     }
 }
 
-data_parser::discover_format_state::discover_format_state()
+data_parser::discover_format_state::
+discover_format_state()
     : dfs_prefix_state(DFS_INIT), dfs_semi_state(DFS_INIT),
       dfs_comma_state(DFS_INIT)
 {

src/data_scanner_re.re

@@ -201,12 +201,17 @@ std::optional<data_scanner::tokenize_result> data_scanner::tokenize_int(text_for
            goto yyc_dbldocstring;
        }
 
-       <dbldocstring> ([\x00]|'"""') {
+       <dbldocstring> '"""' {
            CAPTURE(DT_QUOTED_STRING);
            cap_inner.c_end -= 3;
            return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
        }
 
+       <dbldocstring> [\x00] {
+           CAPTURE(DT_QUOTED_STRING);
+           return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
+       }
+
        <dbldocstring> * {
            goto yyc_dbldocstring;
        }

test/Makefile.am

@@ -293,6 +293,7 @@ dist_noinst_DATA = \
 	datafile_simple.20 \
 	datafile_simple.21 \
 	datafile_simple.22 \
+	datafile_simple.26 \
 	datafile_xml.0 \
 	dhcp.pcapng \
 	dhcp-trunc.pcapng \

test/datafile_simple.26

@@ -0,0 +1,20 @@
+             a=1 b=2 """
+ key   0:1   ^            a
+ num   2:3     ^          1
+ val   2:3     ^          1
+pair   0:3   ^-^          a=1
+ key   4:5       ^        b
+ num   6:7         ^      2
+ val   6:7         ^      2
+pair   4:7       ^-^      b=2
+ key  11:11            ^  
+quot  11:12             ^  
+ val  11:12             ^  
+pair  11:12             ^  
+msg         :a=1 b=2 """
+format      :a=# b=# ""#
+{
+    "a": 1,
+    "b": 2,
+    "col_0": "\u0000"
+}

test/lnav_doctests.cc

@@ -30,6 +30,8 @@
 #include "config.h"
 
 #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+#include <data_parser.hh>
+
 #include "base/from_trait.hh"
 #include "byte_array.hh"
 #include "data_scanner.hh"
@@ -292,3 +294,29 @@ TEST_CASE("data_scanner CSI")
     CHECK(tok_res->tr_token == DT_CSI);
     CHECK(tok_res->to_string() == "\x1b[0m");
 }
+
+TEST_CASE("data_scanner quote")
+{
+    static const char INPUT[] = "abc \"\"\"\n";
+
+    {
+        data_scanner ds(string_fragment::from_const(INPUT));
+
+        auto tok_res = ds.tokenize2();
+        CHECK(tok_res->tr_token == DT_WORD);
+        CHECK(tok_res->to_string() == "abc");
+        tok_res = ds.tokenize2();
+        CHECK(tok_res->tr_token == DT_WHITE);
+        tok_res = ds.tokenize2();
+        CHECK(tok_res->tr_token == DT_QUOTED_STRING);
+        tok_res = ds.tokenize2();
+        CHECK_FALSE(tok_res.has_value());
+    }
+
+    {
+        data_scanner ds(string_fragment::from_const(INPUT));
+        data_parser dp(&ds);
+
+        dp.parse();
+    }
+}