[format] add OpenAM log formats

10 years ago · d557002724
parent b205a3ec82
commit d557002724
23 changed files with 395 additions and 110 deletions
--- a/1
+++ b/1
@ -7,6 +7,7 @@ lnav v0.7.1:
       can be a way to use SQL query results in lnav commands.
     * Added a 'jget' SQLite function that can extract fields from a JSON-
       encoded value.
+     * Added log formats for the OpenAM identity provider.

 lnav v0.7.0:
     Features:
--- a/docs/source/formats.rst
+++ b/docs/source/formats.rst
@ -90,7 +90,7 @@ fields:
    regexes.

    :kind: The type of data that was captured **string**, **integer**,
-      **float**.
+      **float**, **json**, **quoted**.
    :collate: The collation function for this value.
    :identifier: A boolean that indicates whether or not this field represents
      an identifier and should be syntax colored.
--- a/src/Makefile.am
+++ b/src/Makefile.am
@ -33,6 +33,7 @@ TIME_FORMATS = \
    "%d/%b/%Y:%H:%M:%S %z" \
    "%b %d %H:%M:%S" \
    "%m/%d/%y %H:%M:%S" \
+    "%m/%d/%Y %I:%M:%S:%L %p %Z" \
    "%N/%e/%Y %l:%M:%S %p" \
    "%m%d %H:%M:%S" \
    "%M:%S"
--- a/src/Makefile.in
+++ b/src/Makefile.in
@ -367,6 +367,7 @@ TIME_FORMATS = \
    "%d/%b/%Y:%H:%M:%S %z" \
    "%b %d %H:%M:%S" \
    "%m/%d/%y %H:%M:%S" \
+    "%m/%d/%Y %I:%M:%S:%L %p %Z" \
    "%N/%e/%Y %l:%M:%S %p" \
    "%m%d %H:%M:%S" \
    "%M:%S"
--- a/src/default-log-formats.json
+++ b/src/default-log-formats.json
@ -218,6 +218,93 @@
            }
        ]
    },
+    "openam_log": {
+        "title" : "OpenAM Log",
+        "description" : "The OpenAM identity provider.",
+        "url" : "http://openam.forgerock.org",
+        "level-field" : "level",
+        "level" : {
+            "error" : "ERROR",
+            "warning" : "WARNING",
+            "info" : "INFO",
+            "critical" : "SEVERE",
+            "trace" : "FINE|FINEST"
+        },
+        "regex" : {
+            "std" : {
+                "pattern" : "^\"(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})\"\\s+(?<data>[^ \"]+|\"(?:[^\"]|\"\")+\")\\s+(?<loginid>[^ \"]+|\"(?:[^\"]|\"\")+\")\\s+(?<contextid>[^ \"]+|\"(?:[^\"]|\"\")+\")\\s+(?<ipaddr>[^ \"]+|\"(?:[^\"]|\"\")+\")\\s+(?<level>[^ \"]+|\"(?:[^\"]|\"\")+\")\\s+(?<domain>[^ \"]+|\"(?:[^\"]|\"\")+\")\\s+(?<loggedby>[^ \"]+|\"(?:[^\"]|\"\")+\")\\s+(?<messageid>[^ \"]+|\"(?:[^\"]|\"\")+\")\\s+(?<modulename>[^ \"]+|\"(?:[^\"]|\"\")+\")\\s+(?<nameid>[^ \"]+|\"(?:[^\"]|\"\")+\")\\s+(?<hostname>[^ \"]+|\"(?:[^\"]|\"\")+\")"
+            }
+        },
+        "value" : {
+            "data" : {
+                "kind" : "quoted"
+            },
+            "loginid" : {
+                "kind" : "quoted",
+                "identifier" : true
+            },
+            "contextid" : {
+                "kind" : "quoted",
+                "identifier" : true
+            },
+            "ipaddr" : {
+                "kind" : "quoted",
+                "identifier" : true,
+                "collate" : "ipaddress"
+            },
+            "domain" : {
+                "kind" : "quoted",
+                "identifier" : true
+            },
+            "loggedby" : {
+                "kind" : "quoted",
+                "identifier" : true
+            },
+            "messageid" : {
+                "kind" : "quoted",
+                "identifier" : true
+            },
+            "modulename" : {
+                "kind" : "quoted",
+                "identifier" : true
+            },
+            "nameid" : {
+                "kind" : "quoted",
+                "identifier" : true
+            },
+            "hostname" : {
+                "kind" : "quoted",
+                "identifier" : true,
+                "collate" : "ipaddress"
+            }
+        },
+        "sample" : [
+            {
+                "line" : "\"2014-06-14 17:08:39\" \"http://localhost:8086|/|<samlp:AuthnRequest ID=\"\"139a40bba4d340108d91022750c2a3a8\"\" Version=\"\"2.0\"\" IssueInstant=\"\"2014-06-14T17:09:04Z\"\" ProtocolBinding=\"\"urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST\"\" AssertionConsumerServiceURL=\"\"http://localhost:8086/api/1/rest/admin/org/530e42ccd6f45fd16d0d0717/saml/consume\"\">\\n<saml:Issuer>http://localhost:8086</saml:Issuer>\\n<samlp:NameIDPolicy  Format=\"\"urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress\"\" AllowCreate=\"\"true\"\"></samlp:NameIDPolicy>\\n<samlp:RequestedAuthnContext Comparison=\"\"exact\"\"><saml:AuthnContextClassRef>urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport</saml:AuthnContextClassRef></samlp:RequestedAuthnContext>\\n</samlp:AuthnRequest>\"  \"cn=dsameuser,ou=DSAME Users,dc=openam\" 8fc43a8f6a8c14101   \"Not Available\" INFO    dc=openam   \"cn=dsameuser,ou=DSAME Users,dc=openam\" SAML2-36    SAML2.access    \"Not Available\" 127.0.1.1"
+            },
+            {
+                "line" : "\"2014-06-09 14:49:56\" /etc/openam/openam/log/ \"cn=dsameuser,ou=DSAME Users,dc=openam\" 3d956febb91fed31    \"Not Available\" INFO    dc=openam   \"cn=dsameuser,ou=DSAME Users,dc=openam\" LOG-1   amPolicy.access \"Not Available\" 127.0.1.1"
+            }
+        ]
+    },
+    "openamdb_log": {
+        "title" : "OpenAM Debug Log",
+        "description" : "Debug logs for the OpenAM identity provider.",
+        "url" : "http://openam.forgerock.org",
+        "regex" : {
+            "std" : {
+                "pattern" : "^(?<module>[\\w]+):(?<timestamp>\\d{2}/\\d{2}/\\d{4} \\d{2}:\\d{2}:\\d{2}:\\d{3} [AP]M \\w+): Thread\\[(?<thread>[^,]+,\\d+,[^,]+)\\]\\n?(?:\\*+|(?<body>.*))$"
+            }
+        },
+        "sample" : [
+            {
+                "line" : "amMonitoring:06/09/2014 02:49:59:447 PM UTC: Thread[http-80-1,5,main]\n**********************************************"
+            },
+            {
+                "line" : "amLog:06/09/2014 04:08:22:515 PM UTC: Thread[http-80-8,5,main]\nERROR: LogMessageProviderBase.createLogRecord: unable to locate message ID object for ATTEMPT_GET_METAALIAS"
+            }
+        ]
+    },
    "page_log" : {
        "title" : "CUPS Page Log",
        "description" : "The CUPS server log of printed pages.",
--- a/src/grep_proc.cc
+++ b/src/grep_proc.cc
@ -204,6 +204,9 @@ void grep_proc::child_loop(void)
                    fprintf(stdout, "[%d:%d]\n", m->c_begin, m->c_end);
                    for (pc_iter = pc.begin(); pc_iter != pc.end();
                         pc_iter++) {
+                        if (!pc_iter->is_valid()) {
+                            continue;
+                        }
                        fprintf(stdout,
                                "(%d:%d)",
                                pc_iter->c_begin,
@ -359,7 +362,7 @@ void grep_proc::check_fd_set(fd_set &ready_fds)

            while ((loop_count < MAX_LOOPS) &&
                   (this->gp_line_buffer.read_line(this->gp_pipe_offset, lv))) {
-                lv.lv_start[lv.lv_len] = '\0';
+                lv.terminate();
                this->dispatch_line(lv.lv_start);
                loop_count += 1;
            }
--- a/src/lnav.cc
+++ b/src/lnav.cc
@ -2666,8 +2666,9 @@ static void rl_search_internal(void *dummy, readline_curses *rc, bool complete =
    textview_curses *tc    = lnav_data.ld_view_stack.top();
    lnav_view_t      index = (lnav_view_t)(tc - lnav_data.ld_views);

-    if (!complete)
+    if (!complete) {
        tc->set_top(lnav_data.ld_search_start_line);
+    }
    execute_search(index, rc->get_value());
 }

--- a/src/lnav_commands.cc
+++ b/src/lnav_commands.cc
@ -95,7 +95,7 @@ static string com_adjust_log_time(string cmdline, vector<string> &args)
        content_line_t top_content;
        date_time_scanner dts;
        vis_line_t top_line;
-        struct tm tm;
+        struct exttm tm;
        logfile *lf;

        top_line = lnav_data.ld_views[LNV_LOG].get_top();
@ -212,7 +212,7 @@ static string com_goto(string cmdline, vector<string> &args)
        int   line_number, consumed;
        date_time_scanner dts;
        struct timeval tv;
-        struct tm tm;
+        struct exttm tm;
        float value;

        if (dts.scan(args[1].c_str(), NULL, &tm, tv) != NULL) {
--- a/src/lnav_util.cc
+++ b/src/lnav_util.cc
@ -40,7 +40,6 @@
 #include <sqlite3.h>

 #include "auto_fd.hh"
-#include "ptimec.hh"
 #include "lnav_util.hh"

 std::string hash_string(const std::string &str)
@ -55,6 +54,24 @@ std::string hash_string(const std::string &str)
    return hash.to_string();
 }

+size_t unquote(char *dst, const char *str, size_t len)
+{
+    char quote_char = str[0];
+    size_t index = 0;
+
+    require(str[0] == '\'' || str[0] == '"');
+
+    for (int lpc = 1; lpc < (len - 1); lpc++, index++) {
+        dst[index] = str[lpc];
+        if (str[lpc] == quote_char) {
+            lpc += 1;
+        }
+    }
+    dst[index] = '\0';
+
+    return index;
+}
+
 std::string time_ago(time_t last_time)
 {
    time_t      delta, current_time = time(NULL);
@ -256,7 +273,7 @@ const char *std_time_fmt[] = {

 const char *date_time_scanner::scan(const char *time_dest,
                                    const char *time_fmt[],
-                                    struct tm *tm_out,
+                                    struct exttm *tm_out,
                                    struct timeval &tv_out)
 {
    int  curr_time_fmt = -1;
@ -279,12 +296,12 @@ const char *date_time_scanner::scan(const char *time_dest,
                time_t gmt = gmt_int;

                if (this->dts_local_time) {
-                    localtime_r(&gmt, tm_out);
+                    localtime_r(&gmt, &tm_out->et_tm);
 #ifdef HAVE_STRUCT_TM_TM_ZONE
-                    tm_out->tm_zone = NULL;
+                    tm_out->et_tm.tm_zone = NULL;
 #endif
-                    tm_out->tm_isdst = 0;
-                    gmt = tm2sec(tm_out);
+                    tm_out->et_tm.tm_isdst = 0;
+                    gmt = tm2sec(&tm_out->et_tm);
                }
                tv_out.tv_sec = gmt;
                tv_out.tv_usec = 0;
@ -301,25 +318,25 @@ const char *date_time_scanner::scan(const char *time_dest,
            off_t off = 0;

 #ifdef HAVE_STRUCT_TM_TM_ZONE
-            tm_out->tm_zone = NULL;
+            tm_out->et_tm.tm_zone = NULL;
 #endif
            if (func(tm_out, time_dest, off, strlen(time_dest))) {
                retval = &time_dest[off];

-                if (tm_out->tm_year < 70) {
-                    tm_out->tm_year = 80;
+                if (tm_out->et_tm.tm_year < 70) {
+                    tm_out->et_tm.tm_year = 80;
                }
                if (this->dts_local_time) {
-                    time_t gmt = tm2sec(tm_out);
+                    time_t gmt = tm2sec(&tm_out->et_tm);

-                    localtime_r(&gmt, tm_out);
+                    localtime_r(&gmt, &tm_out->et_tm);
 #ifdef HAVE_STRUCT_TM_TM_ZONE
-                    tm_out->tm_zone = NULL;
+                    tm_out->et_tm.tm_zone = NULL;
 #endif
-                    tm_out->tm_isdst = 0;
+                    tm_out->et_tm.tm_isdst = 0;
                }
-                tv_out.tv_sec = tm2sec(tm_out);
-                tv_out.tv_usec = 0;
+                tv_out.tv_sec = tm2sec(&tm_out->et_tm);
+                tv_out.tv_usec = tm_out->et_nsec / 1000;

                this->dts_fmt_lock = curr_time_fmt;
                this->dts_fmt_len  = retval - time_dest;
@ -330,7 +347,7 @@ const char *date_time_scanner::scan(const char *time_dest,
        }
        else if ((retval = strptime(time_dest,
                                    time_fmt[curr_time_fmt],
-                                    tm_out)) != NULL) {
+                                    &tm_out->et_tm)) != NULL) {
            if (time_fmt[curr_time_fmt] == time_fmt_with_zone) {
                int lpc;

@ -338,7 +355,7 @@ const char *date_time_scanner::scan(const char *time_dest,

                }
                if (retval[lpc] == ' ' &&
-                    sscanf(&retval[lpc], "%d", &tm_out->tm_year) == 1) {
+                    sscanf(&retval[lpc], "%d", &tm_out->et_tm.tm_year) == 1) {
                    lpc += 1;
                    for (; retval[lpc] && isdigit(retval[lpc]); lpc++) {

@ -347,21 +364,21 @@ const char *date_time_scanner::scan(const char *time_dest,
                }
            }

-            if (tm_out->tm_year < 70) {
-                tm_out->tm_year = 80;
+            if (tm_out->et_tm.tm_year < 70) {
+                tm_out->et_tm.tm_year = 80;
            }
            if (this->dts_local_time) {
-                time_t gmt = tm2sec(tm_out);
+                time_t gmt = tm2sec(&tm_out->et_tm);

-                localtime_r(&gmt, tm_out);
+                localtime_r(&gmt, &tm_out->et_tm);
 #ifdef HAVE_STRUCT_TM_TM_ZONE
-                tm_out->tm_zone = NULL;
+                tm_out->et_tm.tm_zone = NULL;
 #endif
-                tm_out->tm_isdst = 0;
+                tm_out->et_tm.tm_isdst = 0;
            }

-            tv_out.tv_sec = tm2sec(tm_out);
-            tv_out.tv_usec = 0;
+            tv_out.tv_sec = tm2sec(&tm_out->et_tm);
+            tv_out.tv_usec = tm_out->et_nsec / 1000;

            this->dts_fmt_lock = curr_time_fmt;
            this->dts_fmt_len  = retval - time_dest;
--- a/src/lnav_util.hh
+++ b/src/lnav_util.hh
@ -41,6 +41,7 @@

 #include <string>

+#include "ptimec.hh"
 #include "byte_array.hh"

 inline std::string trim(const std::string &str)
@ -53,6 +54,8 @@ inline std::string trim(const std::string &str)
    return str.substr(start, end - start);
 }

+size_t unquote(char *dst, const char *str, size_t len);
+
 #undef rounddown

 /**
@ -191,18 +194,18 @@ struct date_time_scanner {

    void set_base_time(time_t base_time) {
        this->dts_base_time = base_time;
-        localtime_r(&base_time, &this->dts_base_tm);
+        localtime_r(&base_time, &this->dts_base_tm.et_tm);
    };

    bool dts_local_time;
    time_t dts_base_time;
-    struct tm dts_base_tm;
+    struct exttm dts_base_tm;
    int dts_fmt_lock;
    int dts_fmt_len;

    const char *scan(const char *time_src,
                     const char *time_fmt[],
-                     struct tm *tm_out,
+                     struct exttm *tm_out,
                     struct timeval &tv_out);
 };

--- a/src/log_format.cc
+++ b/src/log_format.cc
@ -39,6 +39,7 @@
 #include "sql_util.hh"
 #include "log_format.hh"
 #include "log_vtab_impl.hh"
+#include "ptimec.hh"

 using namespace std;

@ -124,6 +125,9 @@ logline::level_t logline::string2level(const char *levelstr, size_t len, bool ex
    else if (cmpfunc(levelstr, "CRITICAL", len) == 0) {
        retval = logline::LEVEL_CRITICAL;
    }
+    else if (cmpfunc(levelstr, "SEVERE", len) == 0) {
+        retval = logline::LEVEL_CRITICAL;
+    }
    else if (cmpfunc(levelstr, "FATAL", len) == 0) {
        retval = logline::LEVEL_FATAL;
    }
@ -192,6 +196,9 @@ logline_value::kind_t logline_value::string2kind(const char *kindstr)
    else if (strcmp(kindstr, "json") == 0) {
        return VALUE_JSON;
    }
+    else if (strcmp(kindstr, "quoted") == 0) {
+        return VALUE_QUOTED;
+    }

    return VALUE_UNKNOWN;
 }
@ -230,7 +237,7 @@ const char *log_format::log_scanf(const char *line,
                                  int expected_matches,
                                  const char *time_fmt[],
                                  char *time_dest,
-                                  struct tm *tm_out,
+                                  struct exttm *tm_out,
                                  struct timeval &tv_out,
                                  ...)
 {
@ -586,7 +593,7 @@ bool external_log_format::scan(std::vector<logline> &dst,
        pcre_context::capture_t *level_cap = pc[this->elf_level_field_index];
        const char *ts_str = pi.get_substr_start(ts);
        const char *last;
-        struct tm log_time_tm;
+        struct exttm log_time_tm;
        struct timeval log_tv;
        logline::level_t level = logline::LEVEL_INFO;

@ -712,7 +719,7 @@ static int read_json_field(yajlpp_parse_context *ypc, const unsigned char *str,
    vector<external_log_format::json_format_element> &line_format =
        jlu->jlu_format->jlf_line_format;
    string field_name = ypc->get_path_fragment(0);
-    struct tm tm_out;
+    struct exttm tm_out;
    struct timeval tv_out;

    if (field_name == jlu->jlu_format->lf_timestamp_field) {
@ -1067,7 +1074,7 @@ void external_log_format::build(std::vector<std::string> &errors)
                    pc[this->lf_timestamp_field]);
                date_time_scanner dts;
                struct timeval tv;
-                struct tm tm;
+                struct exttm tm;

                found = true;
                if (dts.scan(ts, NULL, &tm, tv) == NULL) {
@ -1077,7 +1084,16 @@ void external_log_format::build(std::vector<std::string> &errors)
                        iter->s_line);
                    errors.push_back("error:" +
                        this->elf_name +
-                        ":unrecognized timestamp format");
+                        ":unrecognized timestamp format -- " + ts);
+
+                    for (int lpc = 0; PTIMEC_FORMATS[lpc].pf_fmt != NULL; lpc++) {
+                        off_t off = 0;
+
+                        PTIMEC_FORMATS[lpc].pf_func(&tm, ts, off,
+                            pc[this->lf_timestamp_field]->length());
+                        errors.push_back("  format: " + string(PTIMEC_FORMATS[lpc].pf_fmt) +
+                            "; matched: " + string(ts, off));
+                    }
                }
            }
        }
@ -1142,6 +1158,7 @@ public:
            case logline_value::VALUE_NULL:
            case logline_value::VALUE_TEXT:
            case logline_value::VALUE_JSON:
+            case logline_value::VALUE_QUOTED:
                type = SQLITE3_TEXT;
                break;
            case logline_value::VALUE_FLOAT:
--- a/src/log_format.hh
+++ b/src/log_format.hh
@ -344,6 +344,7 @@ public:
        VALUE_FLOAT,
        VALUE_BOOLEAN,
        VALUE_JSON,
+        VALUE_QUOTED,

        VALUE__MAX
    };
@ -381,6 +382,7 @@ public:
        switch (kind) {
        case VALUE_JSON:
        case VALUE_TEXT:
+        case VALUE_QUOTED:
            this->lv_sbr = sbr;
            break;

@ -435,6 +437,25 @@ public:
        case VALUE_TEXT:
            return std::string(this->lv_sbr.get_data(), this->lv_sbr.length());

+        case VALUE_QUOTED:
+            if (this->lv_sbr.length() == 0) {
+                return "";
+            } else {
+                switch (this->lv_sbr.get_data()[0]) {
+                case '\'':
+                case '"': {
+                    char unquoted_str[this->lv_sbr.length()];
+                    size_t unquoted_len;
+
+                    unquoted_len = unquote(unquoted_str, this->lv_sbr.get_data(),
+                        this->lv_sbr.length());
+                    return std::string(unquoted_str, unquoted_len);
+                }
+                default:
+                    return std::string(this->lv_sbr.get_data(), this->lv_sbr.length());
+                }
+            }
+
        case VALUE_INTEGER:
            snprintf(buffer, sizeof(buffer), "%" PRId64, this->lv_number.i);
            break;
@ -622,7 +643,7 @@ protected:
                          int expected_matches,
                          const char *time_fmt[],
                          char *time_dest,
-                          struct tm *tm_out,
+                          struct exttm *tm_out,
                          struct timeval &tv_out,
                          ...);
 };
--- a/src/log_format_impls.cc
+++ b/src/log_format_impls.cc
@ -105,7 +105,7 @@ class strace_log_format : public log_format {
        };

        bool      retval = false;
-        struct tm log_time;
+        struct exttm log_time;
        char      timestr[64];
        struct timeval log_tv;
        int       usecs;
@ -285,7 +285,7 @@ class generic_log_format : public log_format {
              int len)
    {
        bool      retval = false;
-        struct tm log_time;
+        struct exttm log_time;
        char      timestr[64 + 32];
        struct timeval log_tv;
        char      level[64];
--- a/src/log_vtab_impl.cc
+++ b/src/log_vtab_impl.cc
@ -391,6 +391,36 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
                                        SQLITE_TRANSIENT);
                    break;
                }
+                case logline_value::VALUE_QUOTED:
+                    if (lv_iter->lv_sbr.length() == 0) {
+                        sqlite3_result_text(ctx, "", 0, SQLITE_STATIC);
+                    }
+                    else {
+                        const char *text_value = lv_iter->lv_sbr.get_data();
+                        size_t text_len = lv_iter->lv_sbr.length();
+
+                        switch (text_value[0]) {
+                        case '\'':
+                        case '"': {
+                            char *val = (char *)sqlite3_malloc(text_len);
+
+                            if (val == NULL) {
+                                sqlite3_result_error_nomem(ctx);
+                            }
+                            else {
+                                size_t unquoted_len = unquote(val, text_value, text_len);
+                                sqlite3_result_text(ctx, val, unquoted_len, sqlite3_free);
+                            }
+                            break;
+                        }
+                        default: {
+                            sqlite3_result_text(ctx, text_value,
+                                lv_iter->lv_sbr.length(), SQLITE_TRANSIENT);
+                            break;
+                        }
+                        }
+                    }
+                    break;

                case logline_value::VALUE_BOOLEAN:
                case logline_value::VALUE_INTEGER:
@ -476,7 +506,7 @@ static int vt_filter(sqlite3_vtab_cursor *p_vtc,
            const unsigned char *datestr = sqlite3_value_text(argv[lpc]);
            date_time_scanner dts;
            struct timeval tv;
-            struct tm mytm;
+            struct exttm mytm;
            vis_line_t vl;

            dts.scan((const char *)datestr, NULL, &mytm, tv);
--- a/src/ptimec.cc
+++ b/src/ptimec.cc
@ -62,15 +62,20 @@ int main(int argc, char *argv[])
        const char *arg = argv[lpc];

        fputs(PRELUDE, stdout);
-        printf("bool ptime_f%d(struct tm *dst, const char *str, off_t &off, size_t len) {\n",
+        printf("bool ptime_f%d(struct exttm *dst, const char *str, off_t &off, size_t len) {\n",
            lpc);
        for (int index = 0; arg[index]; arg++) {
            if (arg[index] == '%') {
                switch (arg[index + 1]) {
                case 'a':
                case 'Z':
-                    printf("    if (!ptime_upto('%s', str, off, len)) return false;\n",
-                        escape_char(arg[index + 2]));
+                    if (arg[index + 2]) {
+                        printf("    if (!ptime_upto('%s', str, off, len)) return false;\n",
+                            escape_char(arg[index + 2]));
+                    }
+                    else {
+                        printf("    if (!ptime_upto_end(str, off, len)) return false;\n");
+                    }
                    arg += 1;
                    break;
                default:
--- a/src/ptimec.hh
+++ b/src/ptimec.hh
@ -39,6 +39,11 @@
 #include <time.h>
 #include <sys/types.h>

+struct exttm {
+    struct tm et_tm;
+    int32_t et_nsec;
+};
+
 #define PTIME_CONSUME(amount, block) \
    if (off_inout + amount > len) { \
        return false; \
@ -63,9 +68,17 @@ bool ptime_upto(char ch, const char *str, off_t &off_inout, size_t len)
    return false;
 }

-bool ptime_b_slow(struct tm *dst, const char *str, off_t &off_inout, size_t len);
+inline
+bool ptime_upto_end(const char *str, off_t &off_inout, size_t len)
+{
+    off_inout = len;
+
+    return true;
+}
+
+bool ptime_b_slow(struct exttm *dst, const char *str, off_t &off_inout, size_t len);

-inline bool ptime_b(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_b(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
    if (off_inout + 3 < len) {
        int *iptr = (int *)(&str[off_inout]);
@ -114,7 +127,7 @@ inline bool ptime_b(struct tm *dst, const char *str, off_t &off_inout, size_t le
        }
        if (val >= 0) {
            off_inout += 3;
-            dst->tm_mon = val;
+            dst->et_tm.tm_mon = val;
            return true;
        }
    }
@ -122,155 +135,186 @@ inline bool ptime_b(struct tm *dst, const char *str, off_t &off_inout, size_t le
    return ptime_b_slow(dst, str, off_inout, len);
 }

-inline bool ptime_S(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_S(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
+{
+    PTIME_CONSUME(2, {
+        if (str[off_inout + 1] > '9') {
+            return false;
+        }
+        dst->et_tm.tm_sec = (str[off_inout] - '0') * 10 + (str[off_inout + 1] - '0');
+    });
+
+    return (dst->et_tm.tm_sec >= 0 && dst->et_tm.tm_sec <= 59);
+}
+
+inline bool ptime_L(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
+{
+    int ms = 0;
+
+    PTIME_CONSUME(3, {
+        if (str[off_inout + 1] > '9') {
+            return false;
+        }
+        ms = ((str[off_inout] - '0') * 100 + (str[off_inout + 1] - '0') * 10 +
+            (str[off_inout + 2] - '0'));
+    });
+
+    if ((ms >= 0 && ms <= 999)) {
+        dst->et_nsec = ms * 1000000;
+        return true;
+    }
+    return false;
+}
+
+inline bool ptime_M(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
    PTIME_CONSUME(2, {
        if (str[off_inout + 1] > '9') {
            return false;
        }
-        dst->tm_sec = (str[off_inout] - '0') * 10 + (str[off_inout + 1] - '0');
+        dst->et_tm.tm_min = (str[off_inout] - '0') * 10 + (str[off_inout + 1] - '0');
    });

-    return (dst->tm_sec >= 0 && dst->tm_sec <= 59);
+    return (dst->et_tm.tm_min >= 0 && dst->et_tm.tm_min <= 59);
 }

-inline bool ptime_M(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_H(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
    PTIME_CONSUME(2, {
        if (str[off_inout + 1] > '9') {
            return false;
        }
-        dst->tm_min = (str[off_inout] - '0') * 10 + (str[off_inout + 1] - '0');
+        dst->et_tm.tm_hour = (str[off_inout] - '0') * 10 + (str[off_inout + 1] - '0');
    });

-    return (dst->tm_min >= 0 && dst->tm_min <= 59);
+    return (dst->et_tm.tm_hour >= 0 && dst->et_tm.tm_hour <= 23);
 }

-inline bool ptime_H(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_I(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
    PTIME_CONSUME(2, {
        if (str[off_inout + 1] > '9') {
            return false;
        }
-        dst->tm_hour = (str[off_inout] - '0') * 10 + (str[off_inout + 1] - '0');
+        dst->et_tm.tm_hour = (str[off_inout] - '0') * 10 + (str[off_inout + 1] - '0');
    });

-    return (dst->tm_hour >= 0 && dst->tm_hour <= 23);
+    return (dst->et_tm.tm_hour >= 1 && dst->et_tm.tm_hour <= 12);
 }

-inline bool ptime_d(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_d(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
    PTIME_CONSUME(2, {
        if (str[off_inout] == ' ') {
-            dst->tm_mday = 0;
+            dst->et_tm.tm_mday = 0;
        }
        else {
-            dst->tm_mday = (str[off_inout] - '0') * 10;
+            dst->et_tm.tm_mday = (str[off_inout] - '0') * 10;
        }
        if (str[off_inout + 1] > '9') {
            return false;
        }
-        dst->tm_mday += (str[off_inout + 1] - '0');
+        dst->et_tm.tm_mday += (str[off_inout + 1] - '0');
    });

-    return (dst->tm_mday >= 1 && dst->tm_mday <= 31);
+    return (dst->et_tm.tm_mday >= 1 && dst->et_tm.tm_mday <= 31);
 }

-inline bool ptime_e(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_e(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
-    dst->tm_mday = 0;
+    dst->et_tm.tm_mday = 0;
    PTIME_CONSUME(1, {
        if (str[off_inout] < '1' || str[off_inout] > '9') {
            return false;
        }
-        dst->tm_mday = str[off_inout] - '0';
+        dst->et_tm.tm_mday = str[off_inout] - '0';
    });
    if (off_inout + 1 < len) {
        if (str[off_inout] >= '0' && str[off_inout] <= '9') {
-            dst->tm_mday *= 10;
-            dst->tm_mday += str[off_inout] - '0';
+            dst->et_tm.tm_mday *= 10;
+            dst->et_tm.tm_mday += str[off_inout] - '0';
            off_inout += 1;
        }
    }

-    return (dst->tm_mday >= 1 && dst->tm_mday <= 31);
+    return (dst->et_tm.tm_mday >= 1 && dst->et_tm.tm_mday <= 31);
 }

-inline bool ptime_N(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_N(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
-    dst->tm_mon = 0;
+    dst->et_tm.tm_mon = 0;
    PTIME_CONSUME(1, {
        if (str[off_inout] < '1' || str[off_inout] > '9') {
            return false;
        }
-        dst->tm_mon = str[off_inout] - '0';
+        dst->et_tm.tm_mon = str[off_inout] - '0';
    });
    if (off_inout + 1 < len) {
        if (str[off_inout] >= '0' && str[off_inout] <= '9') {
-            dst->tm_mon *= 10;
-            dst->tm_mon += str[off_inout] - '0';
+            dst->et_tm.tm_mon *= 10;
+            dst->et_tm.tm_mon += str[off_inout] - '0';
            off_inout += 1;
        }
    }

-    dst->tm_mon -= 1;
+    dst->et_tm.tm_mon -= 1;

-    return (dst->tm_mon >= 0 && dst->tm_mon <= 11);
+    return (dst->et_tm.tm_mon >= 0 && dst->et_tm.tm_mon <= 11);
 }

-inline bool ptime_k(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_k(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
-    dst->tm_hour = 0;
+    dst->et_tm.tm_hour = 0;
    PTIME_CONSUME(1, {
        if (str[off_inout] < '0' || str[off_inout] > '9') {
            return false;
        }
-        dst->tm_hour = str[off_inout] - '0';
+        dst->et_tm.tm_hour = str[off_inout] - '0';
    });
    if (off_inout + 1 < len) {
        if (str[off_inout] >= '0' && str[off_inout] <= '9') {
-            dst->tm_hour *= 10;
-            dst->tm_hour += str[off_inout] - '0';
+            dst->et_tm.tm_hour *= 10;
+            dst->et_tm.tm_hour += str[off_inout] - '0';
            off_inout += 1;
        }
    }

-    return (dst->tm_hour >= 0 && dst->tm_hour <= 23);
+    return (dst->et_tm.tm_hour >= 0 && dst->et_tm.tm_hour <= 23);
 }

-inline bool ptime_l(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_l(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
-    dst->tm_hour = 0;
+    dst->et_tm.tm_hour = 0;
    PTIME_CONSUME(1, {
        if (str[off_inout] < '1' || str[off_inout] > '9') {
            return false;
        }
-        dst->tm_hour = str[off_inout] - '0';
+        dst->et_tm.tm_hour = str[off_inout] - '0';
    });
    if (off_inout + 1 < len) {
        if (str[off_inout] >= '0' && str[off_inout] <= '9') {
-            dst->tm_hour *= 10;
-            dst->tm_hour += str[off_inout] - '0';
+            dst->et_tm.tm_hour *= 10;
+            dst->et_tm.tm_hour += str[off_inout] - '0';
            off_inout += 1;
        }
    }

-    return (dst->tm_hour >= 1 && dst->tm_hour <= 12);
+    return (dst->et_tm.tm_hour >= 1 && dst->et_tm.tm_hour <= 12);
 }

-inline bool ptime_m(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_m(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
    PTIME_CONSUME(2, {
        if (str[off_inout + 1] > '9') {
            return false;
        }
-        dst->tm_mon = ((str[off_inout] - '0') * 10 + (str[off_inout + 1] - '0')) - 1;
+        dst->et_tm.tm_mon = ((str[off_inout] - '0') * 10 + (str[off_inout + 1] - '0')) - 1;
    });

-    return (0 <= dst->tm_mon && dst->tm_mon <= 11);
+    return (0 <= dst->et_tm.tm_mon && dst->et_tm.tm_mon <= 11);
 }

-inline bool ptime_p(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_p(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
    PTIME_CONSUME(2, {
        char lead = str[off_inout];
@ -281,7 +325,7 @@ inline bool ptime_p(struct tm *dst, const char *str, off_t &off_inout, size_t le
        else if ((lead & 0xdf) == 'A') {
        }
        else if ((lead & 0xdf) == 'P') {
-            dst->tm_hour += 12;
+            dst->et_tm.tm_hour += 12;
        }
        else {
            return false;
@ -291,10 +335,10 @@ inline bool ptime_p(struct tm *dst, const char *str, off_t &off_inout, size_t le
    return true;
 }

-inline bool ptime_Y(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_Y(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
    PTIME_CONSUME(4, {
-        dst->tm_year = (
+        dst->et_tm.tm_year = (
            (str[off_inout + 0] - '0') * 1000 +
            (str[off_inout + 1] - '0') *  100 +
            (str[off_inout + 2] - '0') *   10 +
@ -304,16 +348,16 @@ inline bool ptime_Y(struct tm *dst, const char *str, off_t &off_inout, size_t le
    return true;
 }

-inline bool ptime_y(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_y(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
    PTIME_CONSUME(2, {
-        dst->tm_year = (
+        dst->et_tm.tm_year = (
            (str[off_inout + 0] - '0') * 10 +
            (str[off_inout + 1] - '0') *  1);

-        if (dst->tm_year >= 0 && dst->tm_year < 100) {
-            if (dst->tm_year < 69) {
-                dst->tm_year += 100;
+        if (dst->et_tm.tm_year >= 0 && dst->et_tm.tm_year < 100) {
+            if (dst->et_tm.tm_year < 69) {
+                dst->et_tm.tm_year += 100;
            }
            return true;
        }
@ -322,7 +366,7 @@ inline bool ptime_y(struct tm *dst, const char *str, off_t &off_inout, size_t le
    return true;
 }

-inline bool ptime_z(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+inline bool ptime_z(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
 #ifdef HAVE_STRUCT_TM_TM_ZONE
    PTIME_CONSUME(5, {
@ -346,7 +390,7 @@ inline bool ptime_z(struct tm *dst, const char *str, off_t &off_inout, size_t le
        mins = (
            (str[off_inout + 2] - '0') *   10 +
            (str[off_inout + 3] - '0') *    1) * 60;
-        dst->tm_gmtoff = hours + mins;
+        dst->et_tm.tm_gmtoff = hours + mins;
    });
 #endif

@ -364,7 +408,7 @@ inline bool ptime_char(char val, const char *str, off_t &off_inout, size_t len)
    return true;
 }

-typedef bool (*ptime_func)(struct tm *dst, const char *str, off_t &off, size_t len);
+typedef bool (*ptime_func)(struct exttm *dst, const char *str, off_t &off, size_t len);

 struct ptime_fmt {
    const char *pf_fmt;
--- a/src/ptimec_rt.cc
+++ b/src/ptimec_rt.cc
@ -37,11 +37,11 @@

 #include "ptimec.hh"

-bool ptime_b_slow(struct tm *dst, const char *str, off_t &off_inout, size_t len)
+bool ptime_b_slow(struct exttm *dst, const char *str, off_t &off_inout, size_t len)
 {
    const char *end_of_date;

-    if ((end_of_date = strptime(&str[off_inout], "%b", dst)) != NULL) {
+    if ((end_of_date = strptime(&str[off_inout], "%b", &dst->et_tm)) != NULL) {
        off_inout = end_of_date - str;
        return true;
    }
--- a/src/readline_curses.cc
+++ b/src/readline_curses.cc
@ -592,7 +592,12 @@ void readline_curses::line_ready(const char *line)
    char           msg[1024];
    int            rc;

-    rc = history_expand(rl_line_buffer, expanded.out());
+    if (rl_line_buffer[0] == '^') {
+        rc = -1;
+    }
+    else {
+        rc = history_expand(rl_line_buffer, expanded.out());
+    }
    switch (rc) {
 #if 0
        /* TODO: fix clash between history and pcre metacharacters */
--- a/src/session_data.cc
+++ b/src/session_data.cc
@ -445,7 +445,7 @@ static void load_time_bookmarks(void)
                const char *part_name = (const char *)sqlite3_column_text(stmt.in(), 4);
                int64_t mark_time = sqlite3_column_int64(stmt.in(), 3);
                struct timeval log_tv;
-                struct tm log_tm;
+                struct exttm log_tm;

                if (last_mark_time == -1) {
                    last_mark_time = mark_time;
@ -603,7 +603,7 @@ static void load_time_bookmarks(void)
                const char *log_hash = (const char *)sqlite3_column_text(stmt.in(), 2);
                int64_t mark_time = sqlite3_column_int64(stmt.in(), 3);
                struct timeval log_tv;
-                struct tm log_tm;
+                struct exttm log_tm;

                if (last_mark_time == -1) {
                    last_mark_time = mark_time;
--- a/src/view_curses.cc
+++ b/src/view_curses.cc
@ -158,7 +158,7 @@ void view_curses::mvwattrline(WINDOW *window,
        if (attr_range.lr_end != -1) {
            tab_iter = tab_list.lower_bound(attr_range.lr_end);
            if (tab_iter != tab_list.end()) {
-                if (attr_range.lr_end > tab_iter->first) {
+                if (attr_range.lr_end >= tab_iter->first) {
                    attr_range.lr_end += (
                        tab_iter->second - tab_iter->first) - 1;
                }
--- a/test/logfile_openam.0
+++ b/test/logfile_openam.0
@ -0,0 +1,2 @@
+"2014-06-15 01:04:52"	"http://localhost:8086|/|<samlp:Response xmlns:samlp=""urn:oasis:names:tc:SAML:2.0:protocol"" ID=""s2daac0735bf476f4560aab81104b623bedfb0cbc0"" InResponseTo=""84cbf2be33f6410bbe55877545a93f02"" Version=""2.0"" IssueInstant=""2014-06-15T01:04:52Z"" Destination=""http://localhost:8086/api/1/rest/admin/org/530e42ccd6f45fd16d0d0717/saml/consume""><saml:Issuer xmlns:saml=""urn:oasis:names:tc:SAML:2.0:assertion"">http://openam.vagrant.dev/openam</saml:Issuer><samlp:Status xmlns:samlp=""urn:oasis:names:tc:SAML:2.0:protocol"">\\n<samlp:StatusCode  xmlns:samlp=""urn:oasis:names:tc:SAML:2.0:protocol""\\nValue=""urn:oasis:names:tc:SAML:2.0:status:Success"">\\n</samlp:StatusCode>\\n</samlp:Status><saml:Assertion xmlns:saml=""urn:oasis:names:tc:SAML:2.0:assertion"" ID=""s2a0bee0da937e236167e99b209802056033816ac2"" IssueInstant=""2014-06-15T01:04:52Z"" Version=""2.0"">\\n<saml:Issuer>http://openam.vagrant.dev/openam</saml:Issuer><ds:Signature xmlns:ds=""http://www.w3.org/2000/09/xmldsig#"">\\n<ds:SignedInfo>\\n<ds:CanonicalizationMethod Algorithm=""http://www.w3.org/2001/10/xml-exc-c14n#""/>\\n<ds:SignatureMethod Algorithm=""http://www.w3.org/2000/09/xmldsig#rsa-sha1""/>\\n<ds:Reference URI=""#s2a0bee0da937e236167e99b209802056033816ac2"">\\n<ds:Transforms>\\n<ds:Transform Algorithm=""http://www.w3.org/2000/09/xmldsig#enveloped-signature""/>\\n<ds:Transform Algorithm=""http://www.w3.org/2001/10/xml-exc-c14n#""/>\\n</ds:Transforms>\\n<ds:DigestMethod Algorithm=""http://www.w3.org/2000/09/xmldsig#sha1""/>\\n<ds:DigestValue>4uSmVzjovUdQd3px/RcnoxQBsqE=</ds:DigestValue>\\n</ds:Reference>\\n</ds:SignedInfo>\\n<ds:SignatureValue>\\nhm/grge36uA6j1OWif2bTcvVTwESjmuJa27NxepW0AiV5YlcsHDl7RAIk6k/CjsSero3bxGbm56m\\nYncOEi9F1Tu7dS0bfx+vhm/kKTPgwZctf4GWn4qQwP+KeoZywbNj9ShsYJ+zPKzXwN4xBSuPjMxP\\nNf5szzjEWpOndQO/uDs=\\n</ds:SignatureValue>\\n<ds:KeyInfo>\\n<ds:X509Data>\\n<ds:X509Certificate>\\nMIICQDCCAakCBEeNB0swDQYJKoZIhvcNAQEEBQAwZzELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNh\\nbGlmb3JuaWExFDASBgNVBAcTC1NhbnRhIENsYXJhMQwwCgYDVQQKEwNTdW4xEDAOBgNVBAsTB09w\\nZW5TU08xDTALBgNVBAMTBHRlc3QwHhcNMDgwMTE1MTkxOTM5WhcNMTgwMTEyMTkxOTM5WjBnMQsw\\nCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxMLU2FudGEgQ2xhcmExDDAK\\nBgNVBAoTA1N1bjEQMA4GA1UECxMHT3BlblNTTzENMAsGA1UEAxMEdGVzdDCBnzANBgkqhkiG9w0B\\nAQEFAAOBjQAwgYkCgYEArSQc/U75GB2AtKhbGS5piiLkmJzqEsp64rDxbMJ+xDrye0EN/q1U5Of+\\nRkDsaN/igkAvV1cuXEgTL6RlafFPcUX7QxDhZBhsYF9pbwtMzi4A4su9hnxIhURebGEmxKW9qJNY\\nJs0Vo5+IgjxuEWnjnnVgHTs1+mq5QYTA7E6ZyL8CAwEAATANBgkqhkiG9w0BAQQFAAOBgQB3Pw/U\\nQzPKTPTYi9upbFXlrAKMwtFf2OW4yvGWWvlcwcNSZJmTJ8ARvVYOMEVNbsT4OFcfu2/PeYoAdiDA\\ncGy/F2Zuj8XJJpuQRSE6PtQqBuDEHjjmOQJ0rV/r8mO1ZCtHRhpZ5zYRjhRC9eCbjx9VrFax0JDC\\n/FfwWigmrW0Y0Q==\\n</ds:X509Certificate>\\n</ds:X509Data>\\n</ds:KeyInfo>\\n</ds:Signature><saml:Subject>\\n<saml:NameID Format=""urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress"" NameQualifier=""http://openam.vagrant.dev/openam"">user@example.com</saml:NameID><saml:SubjectConfirmation Method=""urn:oasis:names:tc:SAML:2.0:cm:bearer"">\\n<saml:SubjectConfirmationData InResponseTo=""84cbf2be33f6410bbe55877545a93f02"" NotOnOrAfter=""2014-06-15T01:14:52Z"" Recipient=""http://localhost:8086/api/1/rest/admin/org/530e42ccd6f45fd16d0d0717/saml/consume""/></saml:SubjectConfirmation>\\n</saml:Subject><saml:Conditions NotBefore=""2014-06-15T00:54:52Z"" NotOnOrAfter=""2014-06-15T01:14:52Z"">\\n<saml:AudienceRestriction>\\n<saml:Audience>http://localhost:8086</saml:Audience>\\n</saml:AudienceRestriction>\\n</saml:Conditions>\\n<saml:AuthnStatement AuthnInstant=""2014-06-15T01:00:25Z"" SessionIndex=""s2f9b4d4b453d12b40ef3905cc959cdb40579c2301""><saml:AuthnContext><saml:AuthnContextClassRef>urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport</saml:AuthnContextClassRef></saml:AuthnContext></saml:AuthnStatement></saml:Assertion></samlp:Response>"	id=openamuser,ou=user,dc=openam	82e87195d704585501	"Not Available"	INFO	dc=openam	"cn=dsameuser,ou=DSAME Users,dc=openam"	SAML2-37	SAML2.access	user@example.com	192.168.33.1	
+"2014-06-15 01:04:52"	vagrant|/	"cn=dsameuser,ou=DSAME Users,dc=openam"	ec5708a7f199678a01	"Not Available"	FINE	dc=openam	"cn=dsameuser,ou=DSAME Users,dc=openam"	COT-22	COT.access	"Not Available"	127.0.1.1	
--- a/test/test_date_time_scanner.cc
+++ b/test/test_date_time_scanner.cc
@ -54,7 +54,7 @@ int main(int argc, char *argv[])
    for (int lpc = 0; BAD_TIMES[lpc]; lpc++) {
        date_time_scanner dts;
        struct timeval tv;
-        struct tm tm;
+        struct exttm tm;

        assert(dts.scan(BAD_TIMES[lpc], NULL, &tm, tv) == NULL);
    }
@ -63,7 +63,7 @@ int main(int argc, char *argv[])
        const char *en_date = "Jan  1 12:00:00";
        const char *bg_date = "Ene  1 12:00:00";
        struct timeval en_tv, es_tv;
-        struct tm en_tm, es_tm;
+        struct exttm en_tm, es_tm;
        date_time_scanner dts;

        if (setlocale(LC_TIME, "es_ES.utf8") != NULL) {
--- a/test/test_sql.sh
+++ b/test/test_sql.sh
@ -315,3 +315,50 @@ log_line,log_part,log_time,log_idle_msecs,log_level,log_mark,c_ip,cs_method,cs_r
 1,p.0,2009-07-20 22:59:29.000,3000,error,0,192.168.202.254,GET,-,<NULL>,/vmw/vSphere/default/vmkboot.gz,gPXE/0.9.7,-,HTTP/1.0,46210,404
 2,p.0,2009-07-20 22:59:29.000,0,info,0,192.168.202.254,GET,-,<NULL>,/vmw/vSphere/default/vmkernel.gz,gPXE/0.9.7,-,HTTP/1.0,78929,200
 EOF
+
+
+run_test ${lnav_test} -n \
+    -c ";SELECT * FROM openam_log" \
+    -c ":write-json-to -" \
+    ${test_dir}/logfile_openam.0
+
+check_output "" <<EOF
+[
+    {
+        "log_line": 0,
+        "log_part": "p.0",
+        "log_time": "2014-06-15 01:04:52.000",
+        "log_idle_msecs": 0,
+        "log_level": "info",
+        "log_mark": 0,
+        "contextid": "82e87195d704585501",
+        "data": "http://localhost:8086|/|<samlp:Response xmlns:samlp=\"urn:oasis:names:tc:SAML:2.0:protocol\" ID=\"s2daac0735bf476f4560aab81104b623bedfb0cbc0\" InResponseTo=\"84cbf2be33f6410bbe55877545a93f02\" Version=\"2.0\" IssueInstant=\"2014-06-15T01:04:52Z\" Destination=\"http://localhost:8086/api/1/rest/admin/org/530e42ccd6f45fd16d0d0717/saml/consume\"><saml:Issuer xmlns:saml=\"urn:oasis:names:tc:SAML:2.0:assertion\">http://openam.vagrant.dev/openam</saml:Issuer><samlp:Status xmlns:samlp=\"urn:oasis:names:tc:SAML:2.0:protocol\">\\\\\\\\n<samlp:StatusCode  xmlns:samlp=\"urn:oasis:names:tc:SAML:2.0:protocol\"\\\\\\\\nValue=\"urn:oasis:names:tc:SAML:2.0:status:Success\">\\\\\\\\n</samlp:StatusCode>\\\\\\\\n</samlp:Status><saml:Assertion xmlns:saml=\"urn:oasis:names:tc:SAML:2.0:assertion\" ID=\"s2a0bee0da937e236167e99b209802056033816ac2\" IssueInstant=\"2014-06-15T01:04:52Z\" Version=\"2.0\">\\\\\\\\n<saml:Issuer>http://openam.vagrant.dev/openam</saml:Issuer><ds:Signature xmlns:ds=\"http://www.w3.org/2000/09/xmldsig#\">\\\\\\\\n<ds:SignedInfo>\\\\\\\\n<ds:CanonicalizationMethod Algorithm=\"http://www.w3.org/2001/10/xml-exc-c14n#\"/>\\\\\\\\n<ds:SignatureMethod Algorithm=\"http://www.w3.org/2000/09/xmldsig#rsa-sha1\"/>\\\\\\\\n<ds:Reference URI=\"#s2a0bee0da937e236167e99b209802056033816ac2\">\\\\\\\\n<ds:Transforms>\\\\\\\\n<ds:Transform Algorithm=\"http://www.w3.org/2000/09/xmldsig#enveloped-signature\"/>\\\\\\\\n<ds:Transform Algorithm=\"http://www.w3.org/2001/10/xml-exc-c14n#\"/>\\\\\\\\n</ds:Transforms>\\\\\\\\n<ds:DigestMethod Algorithm=\"http://www.w3.org/2000/09/xmldsig#sha1\"/>\\\\\\\\n<ds:DigestValue>4uSmVzjovUdQd3px/RcnoxQBsqE=</ds:DigestValue>\\\\\\\\n</ds:Reference>\\\\\\\\n</ds:SignedInfo>\\\\\\\\n<ds:SignatureValue>\\\\\\\\nhm/grge36uA6j1OWif2bTcvVTwESjmuJa27NxepW0AiV5YlcsHDl7RAIk6k/CjsSero3bxGbm56m\\\\\\\\nYncOEi9F1Tu7dS0bfx+vhm/kKTPgwZctf4GWn4qQwP+KeoZywbNj9ShsYJ+zPKzXwN4xBSuPjMxP\\\\\\\\nNf5szzjEWpOndQO/uDs=\\\\\\\\n</ds:SignatureValue>\\\\\\\\n<ds:KeyInfo>\\\\\\\\n<ds:X509Data>\\\\\\\\n<ds:X509Certificate>\\\\\\\\nMIICQDCCAakCBEeNB0swDQYJKoZIhvcNAQEEBQAwZzELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNh\\\\\\\\nbGlmb3JuaWExFDASBgNVBAcTC1NhbnRhIENsYXJhMQwwCgYDVQQKEwNTdW4xEDAOBgNVBAsTB09w\\\\\\\\nZW5TU08xDTALBgNVBAMTBHRlc3QwHhcNMDgwMTE1MTkxOTM5WhcNMTgwMTEyMTkxOTM5WjBnMQsw\\\\\\\\nCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxMLU2FudGEgQ2xhcmExDDAK\\\\\\\\nBgNVBAoTA1N1bjEQMA4GA1UECxMHT3BlblNTTzENMAsGA1UEAxMEdGVzdDCBnzANBgkqhkiG9w0B\\\\\\\\nAQEFAAOBjQAwgYkCgYEArSQc/U75GB2AtKhbGS5piiLkmJzqEsp64rDxbMJ+xDrye0EN/q1U5Of+\\\\\\\\nRkDsaN/igkAvV1cuXEgTL6RlafFPcUX7QxDhZBhsYF9pbwtMzi4A4su9hnxIhURebGEmxKW9qJNY\\\\\\\\nJs0Vo5+IgjxuEWnjnnVgHTs1+mq5QYTA7E6ZyL8CAwEAATANBgkqhkiG9w0BAQQFAAOBgQB3Pw/U\\\\\\\\nQzPKTPTYi9upbFXlrAKMwtFf2OW4yvGWWvlcwcNSZJmTJ8ARvVYOMEVNbsT4OFcfu2/PeYoAdiDA\\\\\\\\ncGy/F2Zuj8XJJpuQRSE6PtQqBuDEHjjmOQJ0rV/r8mO1ZCtHRhpZ5zYRjhRC9eCbjx9VrFax0JDC\\\\\\\\n/FfwWigmrW0Y0Q==\\\\\\\\n</ds:X509Certificate>\\\\\\\\n</ds:X509Data>\\\\\\\\n</ds:KeyInfo>\\\\\\\\n</ds:Signature><saml:Subject>\\\\\\\\n<saml:NameID Format=\"urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress\" NameQualifier=\"http://openam.vagrant.dev/openam\">user@example.com</saml:NameID><saml:SubjectConfirmation Method=\"urn:oasis:names:tc:SAML:2.0:cm:bearer\">\\\\\\\\n<saml:SubjectConfirmationData InResponseTo=\"84cbf2be33f6410bbe55877545a93f02\" NotOnOrAfter=\"2014-06-15T01:14:52Z\" Recipient=\"http://localhost:8086/api/1/rest/admin/org/530e42ccd6f45fd16d0d0717/saml/consume\"/></saml:SubjectConfirmation>\\\\\\\\n</saml:Subject><saml:Conditions NotBefore=\"2014-06-15T00:54:52Z\" NotOnOrAfter=\"2014-06-15T01:14:52Z\">\\\\\\\\n<saml:AudienceRestriction>\\\\\\\\n<saml:Audience>http://localhost:8086</saml:Audience>\\\\\\\\n</saml:AudienceRestriction>\\\\\\\\n</saml:Conditions>\\\\\\\\n<saml:AuthnStatement AuthnInstant=\"2014-06-15T01:00:25Z\" SessionIndex=\"s2f9b4d4b453d12b40ef3905cc959cdb40579c2301\"><saml:AuthnContext><saml:AuthnContextClassRef>urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport</saml:AuthnContextClassRef></saml:AuthnContext></saml:AuthnStatement></saml:Assertion></samlp:Response>",
+        "domain": "dc=openam",
+        "hostname": "192.168.33.1\t",
+        "ipaddr": "Not Available",
+        "loggedby": "cn=dsameuser,ou=DSAME Users,dc=openam",
+        "loginid": "id=openamuser,ou=user,dc=openam",
+        "messageid": "SAML2-37",
+        "modulename": "SAML2.access",
+        "nameid": "user@example.com"
+    },
+    {
+        "log_line": 1,
+        "log_part": "p.0",
+        "log_time": "2014-06-15 01:04:52.000",
+        "log_idle_msecs": 0,
+        "log_level": "trace",
+        "log_mark": 0,
+        "contextid": "ec5708a7f199678a01",
+        "data": "vagrant|/",
+        "domain": "dc=openam",
+        "hostname": "127.0.1.1\t",
+        "ipaddr": "Not Available",
+        "loggedby": "cn=dsameuser,ou=DSAME Users,dc=openam",
+        "loginid": "cn=dsameuser,ou=DSAME Users,dc=openam",
+        "messageid": "COT-22",
+        "modulename": "COT.access",
+        "nameid": "Not Available"
+    }
+]
+EOF