[json-log] some more fixes for handling invalid json logs

src/log_format.cc

@@ -555,8 +555,13 @@ log_format::scan_result_t external_log_format::scan(logfile &lf,
                 return log_format::SCAN_NO_MATCH;
             }
             for (int lpc = 0; lpc < line_count; lpc++) {
+                log_level_t level = LEVEL_ERROR;
+
                 ll.set_time(dst.back().get_time());
-                ll.set_level(log_level_t::LEVEL_ERROR);
+                if (lpc > 0) {
+                    level = (log_level_t) (level | LEVEL_CONTINUED);
+                }
+                ll.set_level(level);
                 ll.set_sub_offset(lpc);
                 dst.emplace_back(ll);
             }
@@ -1262,7 +1267,7 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
 
     off_t this_off = 0, next_off = 0;
 
-    if (!this->jlf_line_offsets.empty()) {
+    if (!this->jlf_line_offsets.empty() && ll.get_sub_offset() < this->jlf_line_offsets.size()) {
         require(ll.get_sub_offset() < this->jlf_line_offsets.size());
 
         this_off = this->jlf_line_offsets[ll.get_sub_offset()];
@@ -1272,7 +1277,8 @@ void external_log_format::get_subline(const logline &ll, shared_buffer_ref &sbr,
         else {
             next_off = this->jlf_cached_line.size();
         }
-        if (next_off > 0 && this->jlf_cached_line[next_off - 1] == '\n') {
+        if (next_off > 0 && this->jlf_cached_line[next_off - 1] == '\n' &&
+            this_off != next_off) {
             next_off -= 1;
         }
     }

src/logfile.hh

@@ -313,10 +313,6 @@ public:
             retval = next_line->get_offset() - ll->get_offset() - 1;
         }
 
-        if (retval > line_buffer::MAX_LINE_BUFFER_SIZE / 2) {
-            retval = line_buffer::MAX_LINE_BUFFER_SIZE / 2;
-        }
-
         return retval;
     };
 

src/yajl/yajl.c

@@ -106,6 +106,13 @@ void
 yajl_reset(yajl_handle handle)
 {
     handle->bytesConsumed = 0;
+    if (handle->stateStack.used != 0) {
+        handle->stateStack.used = 0;
+        if (handle->lexer != NULL) {
+            yajl_lex_free(handle->lexer);
+            handle->lexer = NULL;
+        }
+    }
     yajl_bs_push(handle->stateStack, yajl_state_start);
 }
 

test/logfile_invalid_json.json

@@ -1,5 +1,5 @@
 {"ts": "2013-09-06T20:00:48.124817Z", "@fields": { "lvl": "TRACE", "msg": "trace test"}}
 {"ts": "2013-09-06T20:00:49.124817Z", "@fields": { "lvl": "INFO", "msg": "Starting up service"}}
-{"ts": "2013-09-06T22:00:49.124817Z", "@fields": { "lvl": "INFO", ..."msg": "Shutting down service", "user": "steve@example.com"}}
+{"ts": "2013-09-06T22:00:49.124817Z", "@fields": { "lvl": "INFO", "msg": "Shutting down service\nline2\nline3\nline4\nline5\nline6\nline7\nline8\nline9\nline10
 {"ts": "2013-09-06T22:00:59.124817Z", "@fields": { "lvl": "DEBUG5", "msg": "Details..."}}
-{"ts": "2013-09-06T22:00:59.124817Z", "@fields": { "lvl": "DEBUG4", "msg": "Details..."}}
+{"ts": "2013-09-06T22:00:59.222222Z", "@fields": { "lvl": "DEBUG4", "msg": "Details..."}}

test/test_json_format.sh

@@ -528,11 +528,11 @@ check_output "json log format is not working" <<EOF
   @fields: { "lvl": "TRACE", "msg": "trace test"}
 2013-09-06T20:00:49.124 INFO Starting up service
   @fields: { "lvl": "INFO", "msg": "Starting up service"}
-lnav: unable to parse line at offset 186: lexical error: invalid char in json text.
+lnav: unable to parse line at offset 186: parse error: premature EOF
-           "@fields": { "lvl": "INFO", ..."msg": "Shutting down servic
+                                       {"ts": "2013-09-06T22:00:49.124
                      (right here) ------^
 2013-09-06T22:00:59.124 DEBUG5 Details...
   @fields: { "lvl": "DEBUG5", "msg": "Details..."}
-2013-09-06T22:00:59.124 DEBUG4 Details...
+2013-09-06T22:00:59.222 DEBUG4 Details...
   @fields: { "lvl": "DEBUG4", "msg": "Details..."}
 EOF