[generic_log] add time format that includes zone

src/Makefile.am

@@ -28,8 +28,9 @@ init-sql.c: bin2c
 TIME_FORMATS = \
     "%Y-%m-%d %H:%M:%S" \
     "%Y-%m-%d %H:%M" \
-    "%Y-%m-%dT%H:%M:%S" \
+    "%Y-%m-%dT%H:%M:%S.%f%z" \
     "%Y-%m-%dT%H:%M:%SZ" \
+    "%Y-%m-%dT%H:%M:%S" \
     "%Y/%m/%d %H:%M:%S" \
     "%Y/%m/%d %H:%M" \
     "%a %b %d %H:%M:%S %Y" \

src/lnav_util.cc

@@ -471,14 +471,13 @@ const char *date_time_scanner::scan(const char *time_dest,
         /* Try to pull out the milli/micro-second value. */
         if (retval[0] == '.' || retval[0] == ',') {
             off_t off = (retval - time_dest) + 1;
-            int sub_seconds = 0;
 
-            if (ptime_f(sub_seconds, time_dest, off, time_len)) {
-                tv_out.tv_usec = sub_seconds;
+            if (ptime_f(tm_out, time_dest, off, time_len)) {
+                tv_out.tv_usec = tm_out->et_nsec / 1000;
                 this->dts_fmt_len += 7;
             }
-            else if (ptime_F(sub_seconds, time_dest, off, time_len)) {
-                tv_out.tv_usec = sub_seconds * 1000;
+            else if (ptime_F(tm_out, time_dest, off, time_len)) {
+                tv_out.tv_usec = tm_out->et_nsec / 1000;
                 this->dts_fmt_len += 4;
             }
         }

src/log_format.cc

@@ -82,7 +82,7 @@ const char *logline::level_names[LEVEL__MAX + 1] = {
 };
 
 static pcrepp LEVEL_RE(
-        "(?i)(TRACE|DEBUG\\d*|INFO|STATS|WARN(?:ING)?|ERROR|CRITICAL|SEVERE|FATAL)");
+        "(?i)(TRACE|DEBUG\\d*|INFO|STATS|WARN(?:ING)?|ERR(OR)?|CRITICAL|SEVERE|FATAL)");
 
 logline::level_t logline::string2level(const char *levelstr, ssize_t len, bool exact)
 {

src/log_format_impls.cc

@@ -87,7 +87,7 @@ class generic_log_format : public log_format {
 
     static pcre_format *get_pcre_log_formats() {
         static pcre_format log_fmt[] = {
-            { "", pcrepp("^(?<timestamp>[\\dTZ: ,\\.-]+)([^:]+)") },
+            { "", pcrepp("^(?<timestamp>[\\dTZ: +/\\-,\\.-]+)([^:]+)") },
             { "", pcrepp("^(?<timestamp>[\\w:+/\\.-]+) \\[\\w (.*)") },
             { "", pcrepp("^(?<timestamp>[\\w:,/\\.-]+) (.*)") },
             { "", pcrepp("^(?<timestamp>[\\w: \\.,/-]+)\\[[^\\]]+\\](.*)") },

src/ptimec.hh

@@ -426,9 +426,9 @@ inline bool ptime_y(struct exttm *dst, const char *str, off_t &off_inout, ssize_
     return true;
 }
 
+#include "lnav_log.hh"
 inline bool ptime_z(struct exttm *dst, const char *str, off_t &off_inout, ssize_t len)
 {
-#ifdef HAVE_STRUCT_TM_TM_ZONE
     PTIME_CONSUME(5, {
         long sign;
         long hours;
@@ -450,14 +450,15 @@ inline bool ptime_z(struct exttm *dst, const char *str, off_t &off_inout, ssize_
         mins = (
             (str[off_inout + 2] - '0') *   10 +
             (str[off_inout + 3] - '0') *    1) * 60;
+#ifdef HAVE_STRUCT_TM_TM_ZONE
         dst->et_tm.tm_gmtoff = sign * (hours + mins);
-    });
 #endif
+    });
 
     return true;
 }
 
-inline bool ptime_f(int &sub_seconds, const char *str, off_t &off_inout, ssize_t len)
+inline bool ptime_f(struct exttm *dst, const char *str, off_t &off_inout, ssize_t len)
 {
     PTIME_CONSUME(6, {
         for (int lpc = 0; lpc < 6; lpc++) {
@@ -465,25 +466,25 @@ inline bool ptime_f(int &sub_seconds, const char *str, off_t &off_inout, ssize_t
                 return false;
             }
         }
-        sub_seconds = (
+        dst->et_nsec = (
                 (str[off_inout + 0] - '0') * 100000 +
                 (str[off_inout + 1] - '0') *  10000 +
                 (str[off_inout + 2] - '0') *   1000 +
                 (str[off_inout + 3] - '0') *    100 +
                 (str[off_inout + 4] - '0') *     10 +
-                (str[off_inout + 5] - '0') *      1);
+                (str[off_inout + 5] - '0') *      1) * 1000;
     });
 
     return true;
 }
 
-inline bool ptime_F(int &sub_seconds, const char *str, off_t &off_inout, ssize_t len)
+inline bool ptime_F(struct exttm *dst, const char *str, off_t &off_inout, ssize_t len)
 {
     PTIME_CONSUME(3, {
-        sub_seconds = (
+        dst->et_nsec = (
                         (str[off_inout + 0] - '0') * 100 +
                         (str[off_inout + 1] - '0') *  10 +
-                        (str[off_inout + 2] - '0') *   1);
+                        (str[off_inout + 2] - '0') *   1) * 1000 * 1000;
     });
 
     return true;

test/Makefile.am

@@ -221,6 +221,7 @@ dist_noinst_DATA = \
 	logfile_for_join.0 \
 	logfile_generic.0 \
 	logfile_generic.1 \
+	logfile_generic.2 \
 	logfile_glog.0 \
 	logfile_json.json \
 	logfile_multiline.0 \

test/logfile_generic.2

@@ -0,0 +1,2 @@
+2015-04-24T21:08:10.313913+00:00 err rbd  [22968]lotuscreds:ERROR:Could not retrieve lotus account information from db
+2015-04-24T21:08:58.430632+00:00 err rbd  [24206]networkutil:ERROR:The configured address sg01-1-vc1.oc.vmware.com was invalid

test/test_logfile.sh

@@ -179,11 +179,18 @@ EOF
 
 run_test ./drive_logfile -v -f generic_log ${srcdir}/logfile_generic.1
 
-check_output "generic_log level interpreted incorrectly?" <<EOF
+check_output "generic_log (1) level interpreted incorrectly?" <<EOF
 0x07
 0x0a
 EOF
 
+run_test ./drive_logfile -v -f generic_log ${srcdir}/logfile_generic.2
+
+check_output "generic_log (2) level interpreted incorrectly?" <<EOF
+0x0a
+0x0a
+EOF
+
 touch -t 200711030923 ${srcdir}/logfile_glog.0
 run_test ./drive_logfile -t -f glog_log ${srcdir}/logfile_glog.0