Archives
/
lnav
mirror of https://github.com/tstack/lnav
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[line_buffer] split long lines

Browse Source 
Fixes #81
pull/97/head
Timothy Stack 10 years ago
parent eba916749b
commit 9ff2d3774d
23 changed files with 294 additions and 154 deletions
Show Stats Download Patch File Download Diff File

1
Makefile.in
Unescape Escape View File

@ -201,6 +201,7 @@ AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
BZIP2_CMD = @BZIP2_CMD@
CC = @CC@
CCDEPMODE = @CCDEPMODE@
CFLAGS = @CFLAGS@

6
TESTS_ENVIRONMENT.in
Unescape Escape View File

@ -3,6 +3,9 @@
top_srcdir="@abssrcdir@"
export top_srcdir
srcdir="@abssrcdir@/test"
export srcdir
# The top build directory, derived from the path to this script.
top_builddir=`dirname $0`
export top_builddir
@ -10,6 +13,9 @@ export top_builddir
test_dir="@abssrcdir@/test"
export test_dir
BZIP2_CMD="@BZIP2_CMD@"
export BZIP2_CMD
# The full path of the test case
test_file=$1
# The base name of the test case

42
configure vendored
Unescape Escape View File

@ -646,6 +646,7 @@ CCDEPMODE
ac_ct_CC
CFLAGS
CC
BZIP2_CMD
SQLITE3_CMD
pkgpyexecdir
pyexecdir
@ -4451,6 +4452,47 @@ if test x"$SQLITE3_CMD" = x""; then
as_fn_error $? "The sqlite3 command is required" "$LINENO" 5
fi
# Extract the first word of "bzip2", so it can be a program name with args.
set dummy bzip2; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
$as_echo_n "checking for $ac_word... " >&6; }
if ${ac_cv_path_BZIP2_CMD+:} false; then :
$as_echo_n "(cached) " >&6
else
case $BZIP2_CMD in
[\\/]* | ?:[\\/]*)
ac_cv_path_BZIP2_CMD="$BZIP2_CMD" # Let the user override the test with a path.
;;
*)
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
do
IFS=$as_save_IFS
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
ac_cv_path_BZIP2_CMD="$as_dir/$ac_word$ac_exec_ext"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
fi
done
done
IFS=$as_save_IFS
;;
esac
fi
BZIP2_CMD=$ac_cv_path_BZIP2_CMD
if test -n "$BZIP2_CMD"; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $BZIP2_CMD" >&5
$as_echo "$BZIP2_CMD" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
ac_ext=c
ac_cpp='$CPP $CPPFLAGS'
ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5'

2
configure.ac
Unescape Escape View File

@ -73,6 +73,8 @@ if test x"$SQLITE3_CMD" = x""; then
AC_MSG_ERROR([The sqlite3 command is required])
fi
AC_PATH_PROG(BZIP2_CMD, [bzip2])
AC_CHECK_SIZEOF(off_t)
AC_CHECK_SIZEOF(size_t)

1
src/Makefile.in
Unescape Escape View File

@ -230,6 +230,7 @@ AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
BZIP2_CMD = @BZIP2_CMD@
CC = @CC@
CCDEPMODE = @CCDEPMODE@
CFLAGS = @CFLAGS@

2
src/default-log-formats.json
Unescape Escape View File

@ -286,7 +286,7 @@
"url" : "http://www.snaplogic.com/docs/user-guide/user-guide.htm",
"regex" : {
"std" : {
"pattern" : "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?:(?:(?<level>\\w+) (?<logger>[^ ]+) (?<facility>[^ ]+) (?<msgid>[^ ]+) (?<pipe_rid>[^ \\.]+)(?:\\.(?<comp_rid>[^ ]+))? (?<resource_name>[^ ]+) (?<invoker>[^ ]+))|(?:(?:stdout|stderr): ))(?<body>.*)"
"pattern" : "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?) (?:(?:(?<level>\\w{4,}) (?<logger>[^ ]+) (?<facility>[^ ]+) (?<msgid>[^ ]+) (?<pipe_rid>-|\\d+)(?:\\.(?<comp_rid>[^ ]+))? (?<resource_name>[^ ]+) (?<invoker>[^ ]+))|(?:(?:stdout|stderr): ))(?<body>.*)"
}
},
"level-field" : "level",

10
src/grep_proc.cc
Unescape Escape View File

@ -355,14 +355,12 @@ void grep_proc::check_fd_set(fd_set &ready_fds)
static const int MAX_LOOPS = 100;
int loop_count = 0;
size_t len;
char * line;
line_value lv;
while ((loop_count < MAX_LOOPS) &&
(line = this->gp_line_buffer.read_line(this->gp_pipe_offset,
len)) != NULL) {
line[len] = '\0';
this->dispatch_line(line);
(this->gp_line_buffer.read_line(this->gp_pipe_offset, lv))) {
lv.lv_start[lv.lv_len] = '\0';
this->dispatch_line(lv.lv_start);
loop_count += 1;
}

124
src/line_buffer.cc
Unescape Escape View File

@ -114,6 +114,7 @@ private:
line_buffer::line_buffer()
: lb_gz_file(NULL),
lb_bz_file(false),
lb_gz_offset(0),
lb_file_size((size_t)-1),
lb_file_offset(0),
lb_file_time(0),
@ -181,7 +182,7 @@ throw (error)
throw error(errno);
}
}
this->lb_file_time = *((time_t *)&gz_id[4]);
this->lb_file_time = *((int32_t *)&gz_id[4]);
if (this->lb_file_time < 0)
this->lb_file_time = 0;
this->lb_gz_offset = lseek(this->lb_fd, 0, SEEK_CUR);
@ -214,6 +215,9 @@ throw (error)
{
char *tmp, *old;
require(this->lb_bz_file || this->lb_gz_file ||
new_max <= MAX_LINE_BUFFER_SIZE);
/* Still need more space, try a realloc. */
old = this->lb_buffer.release();
tmp = (char *)realloc(old, new_max);
@ -233,10 +237,7 @@ throw (error)
{
size_t prefill, available;
/* The file is probably bogus if a line has gotten this big. */
if (max_length > MAX_LINE_BUFFER_SIZE) {
throw error(EFBIG);
}
require(max_length <= MAX_LINE_BUFFER_SIZE);
/*
* Check to see if the start is inside the cached range or immediately
@ -273,7 +274,7 @@ throw (error)
require(this->lb_file_offset <= start);
require(prefill <= this->lb_buffer_size);
available = this->lb_buffer_max - this->lb_buffer_size;
available = this->lb_buffer_max - (start - this->lb_file_offset);
require(available <= this->lb_buffer_max);
if (max_length > available) {
@ -287,7 +288,7 @@ throw (error)
&this->lb_buffer[prefill],
this->lb_buffer_size);
available = this->lb_buffer_max - this->lb_buffer_size;
available = this->lb_buffer_max - (start - this->lb_file_offset);
if (max_length > available) {
this->resize_buffer(this->lb_buffer_max +
DEFAULT_LINE_BUFFER_SIZE);
@ -376,6 +377,12 @@ throw (error)
&this->lb_buffer[this->lb_buffer_size],
this->lb_buffer_max - this->lb_buffer_size);
BZ2_bzclose(bz_file);
if (rc != -1 && (
rc < (this->lb_buffer_max - this->lb_buffer_size))) {
this->lb_file_size = (
this->lb_file_offset + this->lb_buffer_size + rc);
}
}
}
#endif
@ -398,7 +405,9 @@ throw (error)
}
switch (rc) {
case 0:
this->lb_file_size = this->lb_file_offset + this->lb_buffer_size;
if (!this->lb_seekable) {
this->lb_file_size = this->lb_file_offset + this->lb_buffer_size;
}
if (start < (off_t) this->lb_file_size) {
retval = true;
}
@ -408,9 +417,7 @@ throw (error)
* For compressed files, increase the buffer size so we don't
* have to spend as much time uncompressing the data.
*/
this->resize_buffer(std::min(this->lb_file_size +
DEFAULT_INCREMENT,
MAX_COMPRESSED_BUFFER_SIZE));
this->resize_buffer(MAX_COMPRESSED_BUFFER_SIZE);
}
break;
@ -442,11 +449,11 @@ throw (error)
return retval;
}
char *line_buffer::read_line(off_t &offset, size_t &len_out, bool include_delim)
bool line_buffer::read_line(off_t &offset, line_value &lv, bool include_delim)
throw (error)
{
size_t request_size = DEFAULT_INCREMENT;
char * retval = NULL;
bool retval = false;
require(this->lb_fd != -1);
@ -456,70 +463,92 @@ throw (error)
* Don't return anything past the last known line. The caller needs
* to try reading at the offset of the last line again.
*/
return NULL;
return false;
}
len_out = 0;
while (retval == NULL) {
lv.lv_len = 0;
lv.lv_partial = false;
while (!retval) {
char *line_start, *lf;
this->fill_range(offset, request_size);
/* Find the data in the cache and */
line_start = this->get_range(offset, len_out);
line_start = this->get_range(offset, lv.lv_len);
/* ... look for the end-of-line or end-of-file. */
if (((lf = (char *)memchr(line_start, '\n', len_out)) != NULL) ||
(request_size >= (MAX_LINE_BUFFER_SIZE - DEFAULT_INCREMENT)) ||
(((offset + len_out) == this->lb_file_size) && len_out > 0)) {
if (((lf = (char *)memchr(line_start, '\n', lv.lv_len)) != NULL) ||
(lv.lv_len >= MAX_LINE_BUFFER_SIZE) ||
(request_size == MAX_LINE_BUFFER_SIZE) ||
((request_size > lv.lv_len) && lv.lv_len > 0)) {
if ((lf != NULL) && ((lf - line_start) >= MAX_LINE_BUFFER_SIZE - 1)) {
lf = NULL;
}
if (lf != NULL) {
len_out = lf - line_start;
lv.lv_partial = false;
lv.lv_len = lf - line_start;
if (include_delim) {
len_out += 1;
lv.lv_len += 1;
}
else {
offset += 1; /* Skip the delimiter. */
}
if (offset >= this->lb_last_line_offset) {
this->lb_last_line_offset = offset + len_out;
this->lb_last_line_offset = offset + lv.lv_len;
}
}
else {
if (lv.lv_len >= MAX_LINE_BUFFER_SIZE) {
lv.lv_len = MAX_LINE_BUFFER_SIZE - 1;
lv.lv_partial = false;
}
else {
lv.lv_partial = true;
}
/*
* Be nice and make sure there is room for the caller to
* add a NULL-terminator.
*/
this->ensure_available(offset, len_out + 1);
line_start = this->get_range(offset, len_out);
this->ensure_available(offset, lv.lv_len + 1);
line_start = this->get_range(offset, lv.lv_len);
/*
* Since no delimiter was seen, we need to remember the offset
* of the last line in the file so we don't mistakenly return
* two partial lines to the caller.
*
* 1. read_line() - returns partial line
* 2. file is written
* 3. read_line() - returns the middle of partial line.
*/
this->lb_last_line_offset = offset;
if (lv.lv_len >= MAX_LINE_BUFFER_SIZE) {
lv.lv_len = MAX_LINE_BUFFER_SIZE - 1;
}
if (lv.lv_partial) {
/*
* Since no delimiter was seen, we need to remember the offset
* of the last line in the file so we don't mistakenly return
* two partial lines to the caller.
*
* 1. read_line() - returns partial line
* 2. file is written
* 3. read_line() - returns the middle of partial line.
*/
this->lb_last_line_offset = offset;
}
else if (offset >= this->lb_last_line_offset) {
this->lb_last_line_offset = offset + lv.lv_len;
}
}
offset += len_out;
lv.lv_start = line_start;
offset += lv.lv_len;
retval = line_start;
retval = true;
}
else {
request_size += DEFAULT_INCREMENT;
}
if ((retval == NULL) && !this->fill_range(offset, request_size)) {
if (!retval && !this->fill_range(offset, request_size)) {
break;
}
}
ensure((retval == NULL) ||
(retval >= this->lb_buffer &&
retval < (this->lb_buffer + this->lb_buffer_size)));
ensure(len_out <= this->lb_buffer_size);
ensure(lv.lv_len <= this->lb_buffer_size);
ensure(!retval ||
(lv.lv_start >= this->lb_buffer &&
(lv.lv_start + lv.lv_len) <= (this->lb_buffer + this->lb_buffer_size)));
ensure(this->invariant());
return retval;
@ -528,18 +557,17 @@ throw (error)
bool line_buffer::read_line(off_t &offset_inout, shared_buffer_ref &sbr)
throw (error)
{
char *line;
size_t len;
line_value lv;
bool retval;
// Clear the incoming ref right away so that an invalidate
// does not cause a wasted malloc/copy.
sbr.disown();
line = this->read_line(offset_inout, len);
if (line != NULL) {
sbr.share(this->lb_share_manager, line, len);
if ((retval = this->read_line(offset_inout, lv))) {
sbr.share(this->lb_share_manager, lv.lv_start, lv.lv_len);
}
return line != NULL;
return retval;
}
bool line_buffer::read_range(off_t offset, size_t len, shared_buffer_ref &sbr)

17
src/line_buffer.hh
Unescape Escape View File

@ -44,6 +44,16 @@
#include "auto_mem.hh"
#include "shared_buffer.hh"
struct line_value {
char *lv_start;
size_t lv_len;
bool lv_partial;
void terminate() {
this->lv_start[this->lv_len] = '\0';
};
};
/**
* Buffer for reading whole lines out of file descriptors. The class presents
* a stateless interface, callers specify the offset where a line starts and
@ -82,6 +92,10 @@ public:
*/
ssize_t get_file_size() const { return this->lb_file_size; };
bool is_compressed() const {
return this->lb_gz_file != NULL || this->lb_bz_file;
};
off_t get_read_offset(off_t off) const
{
if (this->lb_gz_file) {
@ -108,7 +122,7 @@ public:
* NULL termination, the invalidate() must be called before re-reading the
* line to refresh the buffer.
*/
char *read_line(off_t &offset_inout, size_t &len_out,
bool read_line(off_t &offset_inout, line_value &lv,
bool include_delim = false)
throw (error);
@ -128,6 +142,7 @@ public:
this->lb_file_offset += this->lb_buffer_size;
this->lb_buffer_size = 0;
this->lb_file_size = -1;
log_info("invalidate %p", this);
};
/** Release any resources held by this object. */

9
src/lnav.cc
Unescape Escape View File

@ -623,7 +623,7 @@ public:
off = total;
}
if ((size_t)off == total ||
if ((((size_t)off == total) && (this->lo_last_offset != off)) ||
ui_periodic_timer::singleton().time_to_update(index_counter)) {
lnav_data.ld_bottom_source.update_loading(off, total);
this->do_update();
@ -3085,10 +3085,9 @@ static string execute_action(log_data_helper &ldh,
static int exec_count = 0;
string value = lv.to_string();
const char *line;
line_buffer lb;
size_t len;
off_t off = 0;
line_value lv;
lnav_data.ld_children.push_back(child_pid);
@ -3099,7 +3098,7 @@ static string execute_action(log_data_helper &ldh,
lb.set_fd(err_pipe.read_end());
line = lb.read_line(off, len);
lb.read_line(off, lv);
if (out_pipe.read_end() != -1) {
piper_proc *pp = new piper_proc(out_pipe.read_end(), false);
@ -3116,7 +3115,7 @@ static string execute_action(log_data_helper &ldh,
lnav_data.ld_files_to_front.push_back(make_pair(desc, 0));
}
retval = string(line, len);
retval = string(lv.lv_start, lv.lv_len);
}
break;
}

1
src/lnav_util.cc
Unescape Escape View File

@ -286,6 +286,7 @@ const char *date_time_scanner::scan(const char *time_dest,
gmt = tm2sec(tm_out);
}
tv_out.tv_sec = gmt;
tv_out.tv_usec = 0;
this->dts_fmt_lock = curr_time_fmt;
this->dts_fmt_len = off;

35
src/logfile.cc
Unescape Escape View File

@ -232,8 +232,7 @@ throw (line_buffer::error, logfile::error)
/* Check for new data based on the file size. */
if (this->lf_index_size < st.st_size) {
off_t last_off, off;
char * line;
size_t len;
line_value lv;
if (!this->lf_index.empty()) {
off = this->lf_index.back().get_offset();
@ -251,26 +250,20 @@ throw (line_buffer::error, logfile::error)
off = 0;
}
last_off = off;
while ((line = this->lf_line_buffer.read_line(off, len, true)) != NULL) {
if (len > 0 && line[len - 1] == '\n') {
this->lf_partial_line = false;
line[len - 1] = '\0';
len -= 1;
}
else {
this->lf_partial_line = true;
line[len] = '\0';
}
this->process_prefix(last_off, line, len);
if (!this->lf_partial_line) {
line[len - 1] = '\n';
}
while (this->lf_line_buffer.read_line(off, lv)) {
char tmp = lv.lv_start[lv.lv_len];
this->lf_partial_line = lv.lv_partial;
lv.lv_start[lv.lv_len] = '\0';
this->process_prefix(last_off, lv.lv_start, lv.lv_len);
lv.lv_start[lv.lv_len] = tmp;
last_off = off;
if (lo != NULL) {
lo->logfile_indexing(*this,
this->lf_line_buffer.get_read_offset(off),
st.st_size);
lo->logfile_indexing(
*this,
this->lf_line_buffer.get_read_offset(off),
st.st_size);
}
}
@ -279,9 +272,9 @@ throw (line_buffer::error, logfile::error)
* doing the scanning, so use the line buffer's notion of the file
* size.
*/
this->lf_index_size = this->lf_line_buffer.get_file_size();
this->lf_index_size = off;
this->lf_line_buffer.invalidate();
// this->lf_line_buffer.invalidate();
retval = true;
}

4
src/logfile.hh
Unescape Escape View File

@ -113,6 +113,10 @@ public:
/** @return The inode for this log file. */
const struct stat &get_stat() const { return this->lf_stat; };
bool is_compressed() const {
return this->lf_line_buffer.is_compressed();
};
/**
* @return The detected format, rebuild_index() must be called before this
* will return a value other than NULL.

9
src/piper_proc.cc
Unescape Escape View File

@ -108,7 +108,6 @@ piper_proc::piper_proc(int pipefd, bool timestamp, const char *filename)
line_buffer lb;
off_t woff = 0, last_woff = 0;
off_t off = 0, last_off = 0;
size_t len;
int nullfd;
nullfd = open("/dev/null", O_RDWR);
@ -116,14 +115,14 @@ piper_proc::piper_proc(int pipefd, bool timestamp, const char *filename)
fcntl(infd.get(), F_SETFL, O_NONBLOCK);
lb.set_fd(infd);
do {
line_value lv;
fd_set rready;
char * line;
FD_ZERO(&rready);
FD_SET(lb.get_fd(), &rready);
select(lb.get_fd() + 1, &rready, NULL, NULL, NULL);
last_off = off;
while ((line = lb.read_line(off, len, true)) != NULL) {
while (lb.read_line(off, lv, true)) {
int wrc;
last_woff = woff;
@ -139,14 +138,14 @@ piper_proc::piper_proc(int pipefd, bool timestamp, const char *filename)
/* Need to do pwrite here since the fd is used by the main
* lnav process as well.
*/
wrc = pwrite(this->pp_fd, line, len, woff);
wrc = pwrite(this->pp_fd, lv.lv_start, lv.lv_len, woff);
if (wrc == -1) {
perror("Unable to write to output file for stdin");
break;
}
woff += wrc;
if (line[len - 1] != '\n') {
if (lv.lv_start[lv.lv_len - 1] != '\n') {
off = last_off;
woff = last_woff;
}

7
test/Makefile.am
Unescape Escape View File

@ -86,6 +86,8 @@ drive_listview_LDADD = ../src/libdiag.a $(CURSES_LIB) -lz
drive_logfile_SOURCES = drive_logfile.cc
drive_logfile_LDADD = \
../src/libdiag.a \
../src/default-log-formats-json.o \
../src/dump-pid-sh.o \
$(CURSES_LIB) \
$(SQLITE3_LIBS) \
$(PCRE_LIBS) \
@ -213,6 +215,7 @@ TESTS = \
test_grep_proc2 \
test_hist_source \
test_json_format.sh \
test_logfile.sh \
test_pcrepp \
test_sessions.sh \
test_sql.sh \
@ -235,7 +238,9 @@ DISTCLEANFILES = \
*.dpt \
*.diff \
*.index \
*.tmp
*.tmp \
*.gz \
*.bz2
distclean-local:
rm -rf sessions

30
test/Makefile.in
Unescape Escape View File

@ -96,11 +96,11 @@ TESTS = test_ansi_scrubber$(EXEEXT) test_auto_fd$(EXEEXT) \
test_line_buffer2$(EXEEXT) test_line_buffer.sh \
test_listview.sh test_mvwattrline.sh test_grep_proc.sh \
test_grep_proc2$(EXEEXT) test_hist_source$(EXEEXT) \
test_json_format.sh test_pcrepp$(EXEEXT) test_sessions.sh \
test_sql.sh test_sql_coll_func.sh test_sql_fs_func.sh \
test_sql_str_func.sh test_view_colors.sh test_vt52_curses.sh \
test_top_status$(EXEEXT) test_data_parser.sh \
test_yajlpp$(EXEEXT)
test_json_format.sh test_logfile.sh test_pcrepp$(EXEEXT) \
test_sessions.sh test_sql.sh test_sql_coll_func.sh \
test_sql_fs_func.sh test_sql_str_func.sh test_view_colors.sh \
test_vt52_curses.sh test_top_status$(EXEEXT) \
test_data_parser.sh test_yajlpp$(EXEEXT)
subdir = test
DIST_COMMON = $(srcdir)/Makefile.in $(srcdir)/Makefile.am \
$(top_srcdir)/mkinstalldirs $(dist_noinst_SCRIPTS) \
@ -135,8 +135,10 @@ drive_listview_OBJECTS = $(am_drive_listview_OBJECTS)
drive_listview_DEPENDENCIES = ../src/libdiag.a $(am__DEPENDENCIES_1)
am_drive_logfile_OBJECTS = drive_logfile.$(OBJEXT)
drive_logfile_OBJECTS = $(am_drive_logfile_OBJECTS)
drive_logfile_DEPENDENCIES = ../src/libdiag.a $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1)
drive_logfile_DEPENDENCIES = ../src/libdiag.a \
../src/default-log-formats-json.o ../src/dump-pid-sh.o \
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1)
am_drive_mvwattrline_OBJECTS = drive_mvwattrline.$(OBJEXT)
drive_mvwattrline_OBJECTS = $(am_drive_mvwattrline_OBJECTS)
drive_mvwattrline_DEPENDENCIES = ../src/libdiag.a \
@ -492,6 +494,7 @@ AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
BZIP2_CMD = @BZIP2_CMD@
CC = @CC@
CCDEPMODE = @CCDEPMODE@
CFLAGS = @CFLAGS@
@ -648,6 +651,8 @@ drive_listview_LDADD = ../src/libdiag.a $(CURSES_LIB) -lz
drive_logfile_SOURCES = drive_logfile.cc
drive_logfile_LDADD = \
../src/libdiag.a \
../src/default-log-formats-json.o \
../src/dump-pid-sh.o \
$(CURSES_LIB) \
$(SQLITE3_LIBS) \
$(PCRE_LIBS) \
@ -766,7 +771,9 @@ DISTCLEANFILES = \
*.dpt \
*.diff \
*.index \
*.tmp
*.tmp \
*.gz \
*.bz2
all: all-am
@ -1228,6 +1235,13 @@ test_json_format.sh.log: test_json_format.sh
--log-file $$b.log --trs-file $$b.trs \
$(am__common_driver_flags) $(AM_LOG_DRIVER_FLAGS) $(LOG_DRIVER_FLAGS) -- $(LOG_COMPILE) \
"$$tst" $(AM_TESTS_FD_REDIRECT)
test_logfile.sh.log: test_logfile.sh
@p='test_logfile.sh'; \
b='test_logfile.sh'; \
$(am__check_pre) $(LOG_DRIVER) --test-name "$$f" \
--log-file $$b.log --trs-file $$b.trs \
$(am__common_driver_flags) $(AM_LOG_DRIVER_FLAGS) $(LOG_DRIVER_FLAGS) -- $(LOG_COMPILE) \
"$$tst" $(AM_TESTS_FD_REDIRECT)
test_pcrepp.log: test_pcrepp$(EXEEXT)
@p='test_pcrepp$(EXEEXT)'; \
b='test_pcrepp'; \

8
test/drive_grep_proc.cc
Unescape Escape View File

@ -54,12 +54,10 @@ public:
bool retval = false;
try {
size_t len;
char *line;
line_value lv;
if ((line = this->ms_buffer.read_line(this->ms_offset,
len)) != NULL) {
value_out = string(line, len);
if (this->ms_buffer.read_line(this->ms_offset, lv)) {
value_out = string(lv.lv_start, lv.lv_len);
retval = true;
}
}

22
test/drive_line_buffer.cc
Unescape Escape View File

@ -124,16 +124,15 @@ int main(int argc, char *argv[])
try {
off_t last_offset = offset;
line_buffer lb;
line_value lv;
char *maddr;
char *line;
size_t len;
lb.set_fd(fd);
if (index.size() == 0) {
while ((line = lb.read_line(offset, len)) != NULL) {
line[len] = '\0';
printf("%s", line);
if ((last_offset + len) < offset)
while (lb.read_line(offset, lv)) {
lv.terminate();
printf("%s", lv.lv_start);
if ((last_offset + lv.lv_len) < offset)
printf("\n");
last_offset = offset;
}
@ -150,22 +149,23 @@ int main(int argc, char *argv[])
else {
off_t seq_offset = 0;
while (lb.read_line(seq_offset, len) != NULL) { }
while (lb.read_line(seq_offset, lv)) { }
do {
bool ret;
int lpc;
random_shuffle(index.begin(), index.end());
for (lpc = 0; lpc < index.size(); lpc++) {
offset = index[lpc].second;
line = lb.read_line(offset, len);
ret = lb.read_line(offset, lv);
assert(line != NULL);
assert(ret);
assert(offset >= 0);
assert(offset <= st.st_size);
assert(memcmp(line,
assert(memcmp(lv.lv_start,
&maddr[index[lpc].second],
len) == 0);
lv.lv_len) == 0);
}
rnd_iters -= 1;

26
test/drive_logfile.cc
Unescape Escape View File

@ -40,15 +40,17 @@
#include <algorithm>
#include "logfile.hh"
#include "log_format.hh"
#include "log_format_loader.hh"
using namespace std;
typedef enum {
MODE_NONE,
MODE_ECHO,
MODE_LINE_COUNT,
MODE_TIMES,
MODE_LEVELS,
MODE_NONE,
MODE_ECHO,
MODE_LINE_COUNT,
MODE_TIMES,
MODE_LEVELS,
} dl_mode_t;
time_t time(time_t *_unused)
@ -62,10 +64,16 @@ int main(int argc, char *argv[])
dl_mode_t mode = MODE_NONE;
string expected_format;
{
std::vector<std::string> paths, errors;
load_formats(paths, errors);
}
while ((c = getopt(argc, argv, "ef:ltv")) != -1) {
switch (c) {
case 'f':
expected_format = optarg;
expected_format = optarg;
break;
case 'e':
mode = MODE_ECHO;
@ -102,10 +110,12 @@ int main(int argc, char *argv[])
assert(lf.get_format() == NULL);
}
else {
// printf("%s %s\n", lf.get_format()->get_name().c_str(), expected_format.c_str());
//printf("%s %s\n", lf.get_format()->get_name().c_str(), expected_format.c_str());
assert(lf.get_format()->get_name() == expected_format);
}
assert(lf.get_modified_time() == st.st_mtime);
if (!lf.is_compressed()) {
assert(lf.get_modified_time() == st.st_mtime);
}
switch (mode) {
case MODE_NONE:

8
test/drive_sequencer.cc
Unescape Escape View File

@ -61,12 +61,10 @@ public:
bool retval = false;
try {
size_t len;
char *line;
line_value lv;
if ((line = this->ms_buffer.read_line(this->ms_offset,
len)) != NULL) {
value_out = string(line, len);
if (this->ms_buffer.read_line(this->ms_offset, lv)) {
value_out = string(lv.lv_start, lv.lv_len);
retval = true;
}
}

9
test/slicer.cc
Unescape Escape View File

@ -58,12 +58,11 @@ int main(int argc, char *argv[])
int line_number, start, end;
off_t offset = 0;
line_buffer lb;
char *line;
size_t len;
line_value lv;
lb.set_fd(fd);
index.push_back(offset);
while ((line = lb.read_line(offset, len)) != NULL) {
while (lb.read_line(offset, lv)) {
index.push_back(offset);
}
index.pop_back();
@ -73,8 +72,8 @@ int main(int argc, char *argv[])
string str;
offset = index[line_number];
line = lb.read_line(offset, len);
str = string(line, len);
lb.read_line(offset, lv);
str = string(lv.lv_start, lv.lv_len);
str = str.substr(start, end);
printf("%s\n", str.c_str());
}

15
test/test_line_buffer2.cc
Unescape Escape View File

@ -40,24 +40,23 @@
static void single_line(const char *data, int lendiff)
{
line_buffer lb;
line_value lv;
auto_fd pi[2];
off_t off = 0;
size_t len;
char *line;
bool ret;
assert(auto_fd::pipe(pi) == 0);
write(pi[1], data, strlen(data));
pi[1].reset();
lb.set_fd(pi[0]);
line = lb.read_line(off, len);
assert(line != NULL);
ret = lb.read_line(off, lv);
assert(ret);
assert(off == strlen(data));
assert(len == strlen(data) - lendiff);
assert(lv.lv_len == strlen(data) - lendiff);
line = lb.read_line(off, len);
assert(line == NULL);
assert(lb.get_file_size() == strlen(data));
ret = lb.read_line(off, lv);
assert(!ret);
}
int main(int argc, char *argv[])

60
test/test_logfile.sh
Unescape Escape View File

@ -21,7 +21,7 @@ run_test ./drive_logfile ${srcdir}/logfile_empty.0
on_error_fail_with "Didn't handle empty log?"
touch -t 200711030923 ${srcdir}/logfile_syslog.0
run_test ./drive_logfile -t -f syslog_log ${srcdir}/logfile_syslog.0
check_output "Syslog timestamp interpreted incorrectly?" <<EOF
@ -31,6 +31,7 @@ Nov 03 09:23:38 2007 -- 000
Nov 03 09:47:02 2007 -- 000
EOF
touch -t 200711030923 ${srcdir}/logfile_syslog.1
run_test ./drive_logfile -t -f syslog_log ${srcdir}/logfile_syslog.1
check_output "Syslog timestamp interpreted incorrectly for year end?" <<EOF
@ -40,6 +41,31 @@ Dec 03 09:23:38 2006 -- 000
Jan 03 09:47:02 2007 -- 000
EOF
gzip -c ${srcdir}/logfile_syslog.1 > logfile_syslog.1.gz
run_test ./drive_logfile -t -f syslog_log logfile_syslog.1.gz
check_output "Syslog timestamp incorrect for gzipped file?" <<EOF
Dec 03 09:23:38 2006 -- 000
Dec 03 09:23:38 2006 -- 000
Dec 03 09:23:38 2006 -- 000
Jan 03 09:47:02 2007 -- 000
EOF
if test x"$BZIP2_CMD" != x""; then
$BZIP2_CMD -z -c "${srcdir}/logfile_syslog.1" > logfile_syslog.1.bz2
touch -t 200711030923 logfile_syslog.1.bz2
run_test ./drive_logfile -t -f syslog_log logfile_syslog.1.bz2
check_output "bzip2 file not loaded?" <<EOF
Dec 03 09:23:38 2006 -- 000
Dec 03 09:23:38 2006 -- 000
Dec 03 09:23:38 2006 -- 000
Jan 03 09:47:02 2007 -- 000
EOF
fi
run_test ./drive_logfile -t -f tcsh_history ${srcdir}/logfile_tcsh_history.0
check_output "TCSH timestamp interpreted incorrectly?" <<EOF
@ -57,18 +83,19 @@ Jul 20 22:59:29 2009 -- 000
Jul 20 22:59:29 2009 -- 000
EOF
touch -t 200711030923 ${srcdir}/logfile_strace_log.0
run_test ./drive_logfile -t -f strace_log ${srcdir}/logfile_strace_log.0
check_output "strace_log timestamp interpreted incorrectly?" <<EOF
Dec 31 08:09:33 1979 -- 814
Dec 31 08:09:33 1979 -- 815
Dec 31 08:09:33 1979 -- 815
Dec 31 08:09:33 1979 -- 815
Dec 31 08:09:33 1979 -- 816
Dec 31 08:09:33 1979 -- 816
Dec 31 08:09:33 1979 -- 816
Dec 31 08:09:33 1979 -- 816
Dec 31 08:09:33 1979 -- 816
Nov 03 08:09:33 2007 -- 814
Nov 03 08:09:33 2007 -- 815
Nov 03 08:09:33 2007 -- 815
Nov 03 08:09:33 2007 -- 815
Nov 03 08:09:33 2007 -- 816
Nov 03 08:09:33 2007 -- 816
Nov 03 08:09:33 2007 -- 816
Nov 03 08:09:33 2007 -- 816
Nov 03 08:09:33 2007 -- 816
EOF
##
@ -77,18 +104,18 @@ run_test ./drive_logfile -v -f syslog_log ${srcdir}/logfile_syslog.0
check_output "Syslog level interpreted incorrectly?" <<EOF
0x05
0x00
0x03
0x05
0x00
0x03
EOF
run_test ./drive_logfile -v -f tcsh_history ${srcdir}/logfile_tcsh_history.0
check_output "TCSH level interpreted incorrectly?" <<EOF
0x40
0xc0
0x40
0xc0
0x03
0x83
0x03
0x83
EOF
run_test ./drive_logfile -v -f access_log ${srcdir}/logfile_access_log.0
@ -125,6 +152,7 @@ check_output "generic_log level interpreted incorrectly?" <<EOF
0x02
EOF
touch -t 200711030923 ${srcdir}/logfile_glog.0
run_test ./drive_logfile -t -f glog_log ${srcdir}/logfile_glog.0
check_output "glog_log timestamp interpreted incorrectly?" <<EOF

Write Preview
Loading…
Cancel
Save