From d72cab0c1ecd1fd7dd0d0faae96b72eedcbc302d Mon Sep 17 00:00:00 2001 From: Timothy Stack Date: Sat, 22 Jun 2013 07:00:59 -0700 Subject: [PATCH] [log_format] the access_log format wasn't picking up the timestamp range correctly --- src/line_buffer.cc | 4 ++-- src/lnav.cc | 23 ++++++++++++++--------- src/log_format_impls.cc | 22 +++++++++++++--------- src/logfile.cc | 2 +- test/Makefile.am | 1 + test/Makefile.in | 1 + test/logfile_access_log.1 | 1 + 7 files changed, 33 insertions(+), 21 deletions(-) create mode 100644 test/logfile_access_log.1 diff --git a/src/line_buffer.cc b/src/line_buffer.cc index 8d39627f..4f1138c6 100644 --- a/src/line_buffer.cc +++ b/src/line_buffer.cc @@ -48,8 +48,8 @@ using namespace std; static const size_t DEFAULT_LINE_BUFFER_SIZE = 256 * 1024; -static const size_t MAX_LINE_BUFFER_SIZE = 16 * DEFAULT_LINE_BUFFER_SIZE; -static const size_t DEFAULT_INCREMENT = 1024; +static const size_t MAX_LINE_BUFFER_SIZE = 4 * DEFAULT_LINE_BUFFER_SIZE; +static const size_t DEFAULT_INCREMENT = 128 * 1024; static const size_t MAX_COMPRESSED_BUFFER_SIZE = 32 * 1024 * 1024; /* diff --git a/src/lnav.cc b/src/lnav.cc index a25bc571..5b98ca4a 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -540,20 +540,25 @@ void rebuild_indexes(bool force) for (iter = tss->tss_files.begin(); iter != tss->tss_files.end(); ) { - (*iter)->rebuild_index(&obs); - if ((*iter)->get_format() != NULL) { - logfile *lf = *iter; - - if (lnav_data.ld_log_source.insert_file(lf)) { - iter = tss->tss_files.erase(iter); - force = true; + try { + (*iter)->rebuild_index(&obs); + if ((*iter)->get_format() != NULL) { + logfile *lf = *iter; + + if (lnav_data.ld_log_source.insert_file(lf)) { + iter = tss->tss_files.erase(iter); + force = true; + } + else { + ++iter; + } } else { ++iter; } } - else { - ++iter; + catch (const line_buffer::error &e) { + iter = tss->tss_files.erase(iter); } } diff --git a/src/log_format_impls.cc b/src/log_format_impls.cc index fe45ae59..bfb67cc8 100644 --- a/src/log_format_impls.cc +++ b/src/log_format_impls.cc @@ -79,7 +79,7 @@ class access_log_format : public log_format { static pcrepp &value_pattern(void) { static pcrepp VALUE_PATTERN( - "^([\\w\\.-]+) [\\w\\.-]+ ([\\w\\.-]+) " + "^([\\w\\.\\-]+) [\\w\\.\\-]+ ([\\w\\.\\-]+) " "\\[([^\\]]+)\\] \"(?:\\-|(\\w+) ([^ \\?]+)(\\?[^ ]+)? " "([\\w/\\.]+))\" (\\d+) " "(\\d+|-)(?: \"([^\"]+)\" \"([^\"]+)\")?.*"); @@ -180,9 +180,10 @@ class access_log_format : public log_format { pcre_context::iterator iter; struct line_range lr; - iter = pc.begin() + 3; + iter = pc.begin() + 2; lr.lr_start = iter->c_begin; lr.lr_end = iter->c_end; + assert(lr.lr_start != -1); sa[lr].insert(make_string_attr("timestamp", 0)); lr.lr_start = 0; @@ -739,9 +740,9 @@ class strace_log_format : public log_format { static pcrepp &value_pattern(void) { static pcrepp VALUE_PATTERN( - "[0-9:.]* ([a-zA-Z_][a-zA-Z_0-9]*)\\(" + "([0-9:.]*) ([a-zA-Z_][a-zA-Z_0-9]*)\\(" "(.*)\\)" - "\\s+= ([-xa-fA-F\\d\\?]+).*(?:<(\\d+\\.\\d+)>)?"); + "\\s+= ([-xa-fA-F\\d\\?]+)[^<]+(?:<(\\d+\\.\\d+)>)?"); return VALUE_PATTERN; }; @@ -823,10 +824,11 @@ class strace_log_format : public log_format { const char * name; logline_value::kind_t kind; } columns[] = { + { "", logline_value::VALUE_TEXT }, { "funcname", logline_value::VALUE_TEXT }, { "args", logline_value::VALUE_TEXT }, { "result", logline_value::VALUE_TEXT }, - { "duration", logline_value::VALUE_TEXT }, + { "duration", logline_value::VALUE_FLOAT }, { NULL }, }; @@ -834,10 +836,12 @@ class strace_log_format : public log_format { pcre_context::iterator iter; struct line_range lr; - iter = pc.begin() + 3; - lr.lr_start = iter->c_begin; - lr.lr_end = iter->c_end; - sa[lr].insert(make_string_attr("timestamp", 0)); + iter = pc.begin(); + if (iter->c_begin != -1) { + lr.lr_start = iter->c_begin; + lr.lr_end = iter->c_end; + sa[lr].insert(make_string_attr("timestamp", 0)); + } lr.lr_start = 0; lr.lr_end = line.length(); diff --git a/src/logfile.cc b/src/logfile.cc index 1d1f5328..ae497934 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -229,7 +229,7 @@ throw (line_buffer::error) st.st_size); } } - + /* * The file can still grow between the above fstat and when we're * doing the scanning, so use the line buffer's notion of the file diff --git a/test/Makefile.am b/test/Makefile.am index 2d9d4b56..9ce73937 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -173,6 +173,7 @@ dist_noinst_DATA = \ listview_output.5 \ listview_output.6 \ logfile_access_log.0 \ + logfile_access_log.1 \ logfile_empty.0 \ logfile_generic.0 \ logfile_glog.0 \ diff --git a/test/Makefile.in b/test/Makefile.in index ae7db5c5..3ea9508b 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -747,6 +747,7 @@ dist_noinst_DATA = \ listview_output.5 \ listview_output.6 \ logfile_access_log.0 \ + logfile_access_log.1 \ logfile_empty.0 \ logfile_generic.0 \ logfile_glog.0 \ diff --git a/test/logfile_access_log.1 b/test/logfile_access_log.1 new file mode 100644 index 00000000..ad2b37e2 --- /dev/null +++ b/test/logfile_access_log.1 @@ -0,0 +1 @@ +10.112.81.15 - - [15/Feb/2013:06:00:31 +0000] "-" 400 0 "-" "-"