[sqlite] Major improvements to the sqlite integration.

This is a checkpoint of the improvements to the sqlite integration.  The
data_parser stuff should be much better now and I've tried to improve
other parts of the user experience as well.
pull/69/head
Timothy Stack 11 years ago
parent b04e6bfc78
commit 3128dc772c

4
configure vendored

@ -3751,8 +3751,8 @@ fi
#CFLAGS=`echo $CFLAGS | sed 's/-O2//g'`
#CXXFLAGS=`echo $CXXFLAGS | sed 's/-O2//g'`
# CFLAGS=`echo $CFLAGS | sed 's/-O2//g'`
# CXXFLAGS=`echo $CXXFLAGS | sed 's/-O2//g'`
# Check whether --enable-static was given.
if test "${enable_static+set}" = set; then :

@ -35,8 +35,8 @@ AC_SUBST(abssrcdir)
AC_PROG_CXX
#CFLAGS=`echo $CFLAGS | sed 's/-O2//g'`
#CXXFLAGS=`echo $CXXFLAGS | sed 's/-O2//g'`
# CFLAGS=`echo $CFLAGS | sed 's/-O2//g'`
# CXXFLAGS=`echo $CXXFLAGS | sed 's/-O2//g'`
AC_ARG_ENABLE([static],
AS_HELP_STRING([--disable-static],

@ -23,7 +23,8 @@ LDADD = \
$(READLINE_LIBS) \
$(CURSES_LIB) \
$(SQLITE3_LIBS) \
-lpcrecpp
-lpcrecpp \
-lcrypto
noinst_HEADERS = \
auto_fd.hh \
@ -32,6 +33,7 @@ noinst_HEADERS = \
bookmarks.hh \
bottom_status_source.hh \
byte_array.hh \
column_namer.hh \
data_scanner.hh \
data_parser.hh \
db_sub_source.hh \
@ -55,6 +57,7 @@ noinst_HEADERS = \
sequence_matcher.hh \
sequence_sink.hh \
statusview_curses.hh \
strnatcmp.h \
strong_int.hh \
termios_guard.hh \
textfile_sub_source.hh \
@ -69,6 +72,8 @@ noinst_HEADERS = \
libdiag_a_SOURCES = \
bookmarks.cc \
collation-functions.cc \
extension-functions.c \
grep_proc.cc \
hist_source.cc \
line_buffer.cc \
@ -77,12 +82,14 @@ libdiag_a_SOURCES = \
log_format.cc \
logfile.cc \
logfile_sub_source.cc \
network-extension-functions.cc \
data_scanner.cc \
data_parser.cc \
readline_curses.cc \
sequence_matcher.cc \
statusview_curses.cc \
piper_proc.cc \
strnatcmp.c \
textview_curses.cc \
view_curses.cc \
vt52_curses.cc \

@ -77,16 +77,19 @@ am__v_AR_0 = @echo " AR " $@;
am__v_AR_1 =
libdiag_a_AR = $(AR) $(ARFLAGS)
libdiag_a_LIBADD =
am_libdiag_a_OBJECTS = bookmarks.$(OBJEXT) grep_proc.$(OBJEXT) \
hist_source.$(OBJEXT) line_buffer.$(OBJEXT) \
listview_curses.$(OBJEXT) lnav_commands.$(OBJEXT) \
log_format.$(OBJEXT) logfile.$(OBJEXT) \
logfile_sub_source.$(OBJEXT) data_scanner.$(OBJEXT) \
am_libdiag_a_OBJECTS = bookmarks.$(OBJEXT) \
collation-functions.$(OBJEXT) extension-functions.$(OBJEXT) \
grep_proc.$(OBJEXT) hist_source.$(OBJEXT) \
line_buffer.$(OBJEXT) listview_curses.$(OBJEXT) \
lnav_commands.$(OBJEXT) log_format.$(OBJEXT) logfile.$(OBJEXT) \
logfile_sub_source.$(OBJEXT) \
network-extension-functions.$(OBJEXT) data_scanner.$(OBJEXT) \
data_parser.$(OBJEXT) readline_curses.$(OBJEXT) \
sequence_matcher.$(OBJEXT) statusview_curses.$(OBJEXT) \
piper_proc.$(OBJEXT) textview_curses.$(OBJEXT) \
view_curses.$(OBJEXT) vt52_curses.$(OBJEXT) \
log_vtab_impl.$(OBJEXT) xterm_mouse.$(OBJEXT)
piper_proc.$(OBJEXT) strnatcmp.$(OBJEXT) \
textview_curses.$(OBJEXT) view_curses.$(OBJEXT) \
vt52_curses.$(OBJEXT) log_vtab_impl.$(OBJEXT) \
xterm_mouse.$(OBJEXT)
libdiag_a_OBJECTS = $(am_libdiag_a_OBJECTS)
am__installdirs = "$(DESTDIR)$(bindir)"
PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS)
@ -300,7 +303,8 @@ LDADD = \
$(READLINE_LIBS) \
$(CURSES_LIB) \
$(SQLITE3_LIBS) \
-lpcrecpp
-lpcrecpp \
-lcrypto
noinst_HEADERS = \
auto_fd.hh \
@ -309,6 +313,7 @@ noinst_HEADERS = \
bookmarks.hh \
bottom_status_source.hh \
byte_array.hh \
column_namer.hh \
data_scanner.hh \
data_parser.hh \
db_sub_source.hh \
@ -332,6 +337,7 @@ noinst_HEADERS = \
sequence_matcher.hh \
sequence_sink.hh \
statusview_curses.hh \
strnatcmp.h \
strong_int.hh \
termios_guard.hh \
textfile_sub_source.hh \
@ -346,6 +352,8 @@ noinst_HEADERS = \
libdiag_a_SOURCES = \
bookmarks.cc \
collation-functions.cc \
extension-functions.c \
grep_proc.cc \
hist_source.cc \
line_buffer.cc \
@ -354,12 +362,14 @@ libdiag_a_SOURCES = \
log_format.cc \
logfile.cc \
logfile_sub_source.cc \
network-extension-functions.cc \
data_scanner.cc \
data_parser.cc \
readline_curses.cc \
sequence_matcher.cc \
statusview_curses.cc \
piper_proc.cc \
strnatcmp.c \
textview_curses.cc \
view_curses.cc \
vt52_curses.cc \
@ -487,8 +497,10 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bin2c.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bookmarks.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/collation-functions.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/data_parser.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/data_scanner.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/extension-functions.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/grep_proc.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hist_source.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/line_buffer.Po@am__quote@
@ -499,10 +511,12 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/log_vtab_impl.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/logfile.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/logfile_sub_source.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/network-extension-functions.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/piper_proc.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/readline_curses.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sequence_matcher.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/statusview_curses.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/strnatcmp.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/textview_curses.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/view_curses.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/vt52_curses.Po@am__quote@

@ -38,19 +38,28 @@
#include <exception>
typedef void (*free_func_t)(void *);
/**
* Resource management class for memory allocated by a custom allocator.
*
* @param T The object type.
* @param auto_free The function to call to free the managed object.
*/
template<class T, void (*auto_free)(void *) = free>
template<class T, free_func_t default_free = free>
class auto_mem {
public:
auto_mem(T *ptr = NULL) : am_ptr(ptr) { };
auto_mem(T *ptr = NULL) : am_ptr(ptr), am_free_func(default_free) { };
auto_mem(auto_mem &am)
: am_ptr(am.release()), am_free_func(am.am_free_func)
{
};
auto_mem(auto_mem &am) : am_ptr(am.release()) { };
template<typename F>
auto_mem(F free_func)
: am_ptr(NULL), am_free_func((void (*)(void *))free_func) { };
~auto_mem() { this->reset(); };
@ -82,14 +91,15 @@ public:
void reset(T *ptr = NULL) {
if (this->am_ptr != ptr) {
auto_free(this->am_ptr);
this->am_free_func(this->am_ptr);
this->am_ptr = ptr;
}
};
private:
T *am_ptr;
void (*am_free_func)(void *);
};
#endif

@ -32,6 +32,11 @@
#include <string>
#include "grep_proc.hh"
#include "textview_curses.hh"
#include "logfile_sub_source.hh"
#include "status_controllers.hh"
class bottom_status_source
: public status_data_source,
public grep_proc_control
@ -52,6 +57,7 @@ public:
BSF_ERRORS,
BSF_FILTERED,
BSF_LOADING,
BSF_HELP,
BSF__MAX
} field_t;
@ -64,17 +70,20 @@ public:
bss_error(80, view_colors::VCR_ALERT_STATUS),
bss_hit_spinner(0),
bss_load_percent(0) {
this->bss_fields[BSF_LINE_NUMBER].set_width(8);
this->bss_fields[BSF_LINE_NUMBER].set_width(10);
this->bss_fields[BSF_PERCENT].set_width(4);
this->bss_fields[BSF_HITS].set_width(16);
this->bss_fields[BSF_HITS].set_cylon(true);
this->bss_fields[BSF_WARNINGS].set_width(10);
this->bss_fields[BSF_ERRORS].set_width(10);
this->bss_fields[BSF_ERRORS].set_role(view_colors::VCR_ALERT_STATUS);
this->bss_fields[BSF_FILTERED].set_width(14);
this->bss_fields[BSF_FILTERED].set_width(20);
this->bss_fields[BSF_LOADING].set_width(13);
this->bss_fields[BSF_LOADING].set_cylon(true);
this->bss_fields[BSF_LOADING].right_justify(true);
this->bss_fields[BSF_HELP].set_width(14);
this->bss_fields[BSF_HELP].set_value("?:View Help");
this->bss_fields[BSF_HELP].right_justify(true);
};
virtual ~bottom_status_source() { };
@ -111,10 +120,12 @@ public:
void update_line_number(listview_curses *lc) {
status_field &sf = this->bss_fields[BSF_LINE_NUMBER];
if (lc->get_inner_height() == 0)
if (lc->get_inner_height() == 0) {
sf.set_value("L0");
else
sf.set_value("L%d", (int)lc->get_top());
}
else {
sf.set_value("L%'d", (int)lc->get_top());
}
};
void update_percent(listview_curses *lc) {
@ -153,7 +164,7 @@ public:
bookmark_vector<vis_line_t>::iterator iter;
iter = lower_bound(bv.begin(), bv.end(), tc->get_top() + height);
sfw.set_value("%9dW", distance(iter, bv.end()));
sfw.set_value("%'9dW", distance(iter, bv.end()));
}
else {
sfw.clear();
@ -164,7 +175,7 @@ public:
bookmark_vector<vis_line_t>::iterator iter;
iter = lower_bound(bv.begin(), bv.end(), tc->get_top() + height);
sfe.set_value("%9dE", distance(iter, bv.end()));
sfe.set_value("%'9dE", distance(iter, bv.end()));
}
else {
sfe.clear();
@ -189,7 +200,7 @@ public:
}
sf.set_role(new_role);
this->bss_error.clear();
sf.set_value("%9d hits", tc->get_match_count());
sf.set_value("%'9d hits", tc->get_match_count());
};
void update_loading(off_t off, size_t total) {
@ -218,7 +229,7 @@ public:
if (lss.get_filtered_count() == 0)
sf.clear();
else
sf.set_value("%d Not Shown", lss.get_filtered_count());
sf.set_value("%'9d Not Shown", lss.get_filtered_count());
};
private:

@ -43,7 +43,11 @@ struct byte_array {
bool operator<(const byte_array &other) const {
return memcmp(this->ba_data, other.ba_data, BYTE_COUNT) < 0;
};
bool operator!=(const byte_array &other) const {
return memcmp(this->ba_data, other.ba_data, BYTE_COUNT) != 0;
};
unsigned char ba_data[BYTE_COUNT];
};

@ -0,0 +1,161 @@
/**
* Copyright (c) 2013, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file logfile_sub_source.hh
*/
#include <stdio.h>
#include <string.h>
#include <sqlite3.h>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <algorithm>
extern "C" {
#include "strnatcmp.h"
}
#define MAX_ADDR_LEN 128
static int strncmp2(int a_len, const char *a_str,
int b_len, const char *b_str)
{
int retval = strncmp(a_str, b_str, std::min(a_len, b_len));
if (retval == 0) {
if (a_len < b_len) {
retval = -1;
}
else {
retval = 1;
}
}
return retval;
}
static int try_inet_pton(int p_len, const char *p, char *n)
{
static int family[] = { AF_INET6, AF_INET, AF_MAX };
char buf[MAX_ADDR_LEN];
int retval = AF_MAX;
strncpy(buf, p, p_len);
buf[p_len] = '\0';
for (int lpc = 0; family[lpc] != AF_MAX; lpc++) {
if (inet_pton(family[lpc], buf, n) == 1) {
retval = family[lpc];
break;
}
}
return retval;
}
static int convert_v6_to_v4(int family, char *n)
{
struct in6_addr *ia = (struct in6_addr *)n;
if (family == AF_INET6 &&
(IN6_IS_ADDR_V4COMPAT(ia) ||
IN6_IS_ADDR_V4MAPPED(ia))) {
family = AF_INET;
memmove(n, n + 12, sizeof(struct in_addr));
}
return family;
}
static
int ipaddress(void *ptr,
int a_len, const void *a_in,
int b_len, const void *b_in)
{
char a_addr[sizeof(struct in6_addr)], b_addr[sizeof(struct in6_addr)];
const char *a_str = (const char *)a_in, *b_str = (const char *)b_in;
int a_family, b_family, retval;
if (a_len > MAX_ADDR_LEN || b_len > MAX_ADDR_LEN) {
return strncmp2(a_len, a_str, b_len, b_str);
}
a_family = try_inet_pton(a_len, a_str, a_addr);
b_family = try_inet_pton(b_len, b_str, b_addr);
if (a_family == AF_MAX && b_family != AF_MAX) {
retval = -1;
}
else if (a_family != AF_MAX && b_family == AF_MAX) {
retval = 1;
}
else {
a_family = convert_v6_to_v4(a_family, a_addr);
b_family = convert_v6_to_v4(b_family, b_addr);
if (a_family == b_family) {
retval = memcmp(a_addr, b_addr,
a_family == AF_INET ?
sizeof(struct in_addr) :
sizeof(struct in6_addr));
}
else if (a_family == AF_INET) {
retval = -1;
}
else {
retval = 1;
}
}
return retval;
}
static
int sql_strnatcmp(void *ptr,
int a_len, const void *a_in,
int b_len, const void *b_in)
{
return strnatcmp(a_len, (char *)a_in, b_len, (char *)b_in);
}
static
int sql_strnatcasecmp(void *ptr,
int a_len, const void *a_in,
int b_len, const void *b_in)
{
return strnatcasecmp(a_len, (char *)a_in, b_len, (char *)b_in);
}
int register_collation_functions(sqlite3 *db)
{
sqlite3_create_collation(db, "ipaddress", SQLITE_UTF8, NULL, ipaddress);
sqlite3_create_collation(db, "natural", SQLITE_UTF8, NULL, sql_strnatcmp);
sqlite3_create_collation(db, "naturalnocase", SQLITE_UTF8, NULL, sql_strnatcasecmp);
return 0;
}

@ -0,0 +1,86 @@
/**
* Copyright (c) 2013, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file column_namer.hh
*/
#include <map>
#include <string>
#include <vector>
#include <algorithm>
class column_namer {
public:
column_namer() {
this->cn_builtin_names.push_back("col");
};
bool existing_name(const std::string &in_name) {
if (find(this->cn_builtin_names.begin(),
this->cn_builtin_names.end(),
in_name) != this->cn_builtin_names.end()) {
return true;
}
else if (find(this->cn_names.begin(),
this->cn_names.end(),
in_name) != this->cn_names.end()) {
return true;
}
return false;
};
std::string add_column(const std::string &in_name) {
std::string base_name = in_name, retval;
size_t buf_size;
char *buffer;
int num = 0;
buf_size = in_name.length() + 64;
buffer = (char *)alloca(buf_size);
if (in_name == "") {
base_name = "col";
}
retval = base_name;
while (this->existing_name(retval)) {
snprintf(buffer, buf_size, "%s_%d", base_name.c_str(), num);
retval = buffer;
num += 1;
}
this->cn_names.push_back(retval);
return retval;
};
std::vector<std::string> cn_builtin_names;
std::vector<std::string> cn_names;
};

@ -33,259 +33,88 @@
using namespace std;
static data_token_t PATTERN_KEY[] = {
DT_STRING,
DT_NUMBER,
DT_HEX_NUMBER,
// DT_QUALIFIED_NAME,
};
data_format data_parser::FORMAT_SEMI(DT_COMMA, DT_SEMI);
data_format data_parser::FORMAT_COMMA(DT_INVALID, DT_COMMA);
data_format data_parser::FORMAT_PLAIN(DT_INVALID, DT_INVALID);
static data_token_t UPTO_SEPARATOR[] = {
DT_SEPARATOR,
DT_LINE,
};
static data_token_t UPTO_NT[] = {
DNT_PAIR,
DNT_ROW,
DT_SEPARATOR,
};
static data_token_t PATTERN_PAIR[] = {
DNT_ROW,
DT_SEPARATOR,
// DNT_KEY,
};
static data_token_t PATTERN_ROW[] = {
DT_ANY,
DT_COMMA,
DNT_ROW,
};
static data_token_t PATTERN_DATE_TIME[] = {
DT_TIME,
DT_NUMBER,
DT_STRING,
};
static data_token_t PATTERN_QUAL[] = {
DNT_KEY,
DT_SEPARATOR,
DNT_KEY,
};
bool data_parser::reducePattern(std::list<element> &reduction,
const data_token_t *pattern_start,
const data_token_t *pattern_end,
bool repeating)
{
size_t pattern_size = (pattern_end - pattern_start);
bool found, retval = false;
reduction.clear();
do {
found = false;
if (pattern_size <= this->dp_stack.size() &&
std::equal(pattern_start, pattern_end,
this->dp_stack.begin(),
element_cmp())) {
std::list<element>::iterator match_end = this->dp_stack.begin();
advance(match_end, pattern_size);
reduction.splice(reduction.end(),
this->dp_stack,
this->dp_stack.begin(),
match_end);
retval = found = true;
}
} while (found && repeating);
reduction.reverse();
return retval;
}
void data_parser::reduceQual(const struct element &lookahead)
{
std::list<element> reduction;
if (this->reducePattern(reduction,
PATTERN_QUAL,
PATTERN_QUAL +
sizeof(PATTERN_QUAL) / sizeof(data_token_t))) {
// printf("qual hit\n");
this->dp_qual.splice(this->dp_qual.end(), reduction);
}
}
void data_parser::reduceRow(void)
{
std::list<element> reduction;
if (this->reducePattern(reduction,
PATTERN_ROW,
PATTERN_ROW +
sizeof(PATTERN_ROW) / sizeof(data_token_t))) {
std::list<element>::iterator match_end;
if (reduction.back().e_sub_elements != NULL)
reduction.front().assign_elements(*reduction.back().e_sub_elements);
else
reduction.front().e_sub_elements->push_back(reduction.back());
reduction.front().update_capture();
match_end = reduction.begin();
++match_end;
this->dp_stack.splice(this->dp_stack.begin(),
reduction,
reduction.begin(),
match_end);
}
}
void data_parser::reducePair(void)
data_format_state_t dfs_semi_next(data_format_state_t state,
data_token_t next_token)
{
std::list<element> reduction;
if (this->reducePattern(reduction,
PATTERN_DATE_TIME,
PATTERN_DATE_TIME +
sizeof(PATTERN_DATE_TIME) / sizeof(data_token_t))) {
this->dp_stack.push_front(element(reduction, DNT_DATE_TIME));
this->dp_stack.front().assign_elements(reduction);
}
this->reduceRow();
if (this->reduceUpTo(reduction,
UPTO_SEPARATOR,
UPTO_SEPARATOR +
sizeof(UPTO_SEPARATOR) / sizeof(data_token_t)) &&
!reduction.empty()) {
if (reduction.front().e_token == DNT_ROW) {
reduction.reverse();
this->dp_stack.splice(this->dp_stack.begin(), reduction);
}
else {
this->dp_stack.push_front(element(reduction, DNT_ROW));
this->dp_stack.front().assign_elements(reduction);
data_format_state_t retval = state;
switch (state) {
case DFS_INIT:
switch (next_token) {
case DT_COMMA:
case DT_SEMI:
retval = DFS_ERROR;
break;
default: retval = DFS_KEY; break;
}
break;
case DFS_KEY:
switch (next_token) {
case DT_SEPARATOR: retval = DFS_VALUE; break;
case DT_SEMI: retval = DFS_ERROR; break;
default: break;
}
break;
case DFS_VALUE:
switch (next_token) {
case DT_SEMI: retval = DFS_INIT; break;
default: break;
}
break;
case DFS_ERROR: retval = DFS_ERROR; break;
}
}
if (this->reducePattern(reduction,
PATTERN_PAIR,
PATTERN_PAIR +
sizeof(PATTERN_PAIR) / sizeof(data_token_t))) {
if (this->dp_qual.empty()) {
this->dp_stack.splice(this->dp_stack.begin(), reduction);
}
else {
reduction.push_front(this->dp_qual.back());
this->dp_qual.pop_back();
this->dp_stack.push_front(element(reduction, DNT_PAIR));
this->dp_stack.front().assign_elements(reduction);
}
}
// this->print(stdout);
return retval;
}
#define DEB 0
void data_parser::reduce(const element &lookahead)
data_format_state_t dfs_comma_next(data_format_state_t state,
data_token_t next_token)
{
std::list<element> reduction;
bool push_lookahead = true;
switch (lookahead.e_token) {
case DT_INVALID:
case DT_WHITE:
this->reducePair();
push_lookahead = false;
break;
case DT_GARBAGE:
push_lookahead = false;
break;
case DT_LINE:
this->reduceRow();
if (!this->reduceUpTo(reduction,
UPTO_NT,
UPTO_NT +
sizeof(UPTO_NT) / sizeof(data_token_t))) {
reduction.splice(reduction.begin(), this->dp_stack);
reduction.reverse();
}
if (!reduction.empty()) {
if (this->dp_stack.front().e_token == DNT_ROW) {
this->dp_stack.front().assign_elements(reduction);
}
else if (this->dp_stack.front().e_token == DNT_PAIR) {
this->dp_stack.front().e_sub_elements->back().assign_elements(reduction);
}
else {
this->dp_stack.push_front(element(reduction, DNT_ROW));
this->dp_stack.front().assign_elements(reduction);
}
data_format_state_t retval = state;
switch (state) {
case DFS_INIT:
switch (next_token) {
case DT_COMMA:
case DT_SEMI:
retval = DFS_ERROR;
break;
default:
retval = DFS_KEY;
break;
}
break;
case DFS_KEY:
switch (next_token) {
case DT_SEPARATOR:
retval = DFS_VALUE;
break;
case DT_SEMI:
case DT_COMMA:
retval = DFS_ERROR;
break;
default: break;
}
break;
case DFS_VALUE:
switch (next_token) {
case DT_COMMA:
retval = DFS_INIT;
break;
case DT_SEPARATOR:
retval = DFS_VALUE;
break;
default: break;
}
break;
case DFS_ERROR:
retval = DFS_ERROR;
break;
}
this->reducePair();
push_lookahead = false;
break;
case DT_COMMA:
this->reduceRow();
if (!this->dp_stack.empty() &&
this->dp_stack.front().e_token != DNT_ROW) {
if (this->dp_stack.front().e_token == DT_SEPARATOR) {
push_lookahead = false;
}
else if (this->dp_stack.front().e_token == DNT_PAIR) {
std::list<element>::iterator pair_iter = this->dp_stack.begin();
this->dp_qual.push_front(this->dp_stack.front().e_sub_elements->front());
this->dp_stack.front().e_sub_elements->pop_front();
this->dp_stack.front().e_sub_elements->reverse();
this->dp_stack.splice(this->dp_stack.begin(),
*this->dp_stack.front().e_sub_elements);
this->dp_stack.erase(pair_iter);
}
else {
std::list<element>::iterator next_elem = this->dp_stack.begin();
advance(next_elem, 1);
reduction.splice(reduction.end(),
this->dp_stack,
this->dp_stack.begin(),
next_elem);
this->dp_stack.push_front(element(reduction, DNT_ROW));
this->dp_stack.front().assign_elements(reduction);
}
}
break;
case DT_SEPARATOR:
if (this->reduceAnyOf(reduction,
PATTERN_KEY,
PATTERN_KEY +
sizeof(PATTERN_KEY) / sizeof(data_token_t))) {
this->reducePair();
if (this->dp_stack.front().e_token == DT_SEPARATOR)
this->dp_stack.pop_front();
this->dp_qual.push_back(element(reduction, DNT_KEY));
// this->reduceQual(lookahead);
}
break;
default:
break;
}
if (push_lookahead) {
this->dp_stack.push_front(lookahead);
}
// this->print(stdout);
return retval;
}

@ -32,49 +32,152 @@
#include <stdio.h>
#include <openssl/sha.h>
#include <list>
#include <vector>
#include <iterator>
#include <algorithm>
#include "pcrepp.hh"
#include "byte_array.hh"
#include "data_scanner.hh"
template<class ForwardIterator1, class ForwardIterator2, class BinaryPredicate>
ForwardIterator1 find_first_not_of(ForwardIterator1 first, ForwardIterator1 last,
ForwardIterator2 s_first, ForwardIterator2 s_last,
BinaryPredicate p)
/**
* Switch to the 'parser' view mode when the user hits ';' so they
* can easily see what columns are available.
*
* select * from logline;
* select itemfrom(csv_key, 0) from logline;
* select itemfrom(csv_key, -1) from logline;
* select itemfrom(dict_key, "key") from logline;
* select itemfrom(dict_key, "key[0]") from logline;
* select itemfrom(csv_key, 0:3) from logline; support splices ?
*
* Add a command to create a logline table with a given name so the user can
* do joins across the tables:
*
* create-logline-table sudo_logline
* select * from logline, sudo_logline where sudo_logline.COMMAND=logline.COMMAND;
*
* select timestmap / 60 as minute, sc_status, count(*) from access_log
* group by minute, sc_status
* order by minute, sc_status desc;
* (use group_concat() here?)
*
* The "itemfrom()" function parses the group and lets you specify an
* expression to query the contents.
*
* For 'report-on' command:
* 'report-on PWD'
* select PWD,count(*) as amount from logline group by PWD order by amount desc;
* 'report-on num_col num_col2'
* select avg(num_col),stddev(num_col),... from logline;
* Instead of a command, we should automatically create views with the
* relevant select statements.
*
* Add a tojson() aggregate function to sqlite:
* select foo,tojson(bar) group by foo;
*
* 1 ["a", "b", "c"]
* 2 ["d", "e", "f"]
*
* We should automatically detect sqlite files provided on the command line
* and attach the database.
*
* add a 'metadata' view that has all the metadata crud (sql tables/log)
*
* Add support for sqlite_log that writes to a temp file and is displayed in the
* metadata view.
*
* Add a function that bookmarks all lines in the log view based on line_numbers
* in the sql query result.
*
* select line_number from logline where A="b";
* hit 'y/Y' to move forward and backwards through sql results
* hit 'R' key and all the lines are bookmarked
*
* add path manipulation functions like basename, dirname, splitext
*
* use the vt52_curses emulation to embed a editor for editing queries. For
* example, you could hit 'ctrl+;' and it would split the window in half with
* the bottom being used for nano. When the file was written, lnav should
* notice and do a 'prepare' on the sql to make sure it is correct.
*
* Maybe add other tables for accessing lnav state. For example, you could do
* a query to find log lines of interest and then insert their line numbers
* into the 'bookmarks' table to create new user bookmarks.
*/
template<class Container, class UnaryPredicate>
void strip(Container &container, UnaryPredicate p)
{
for (; first != last; ++first) {
bool found = false;
for (ForwardIterator2 it = s_first; it != s_last; ++it) {
if (p(*first, *it)) {
found = true;
}
}
if (!found)
return first;
}
return last;
while (!container.empty() && p(container.front())) {
container.pop_front();
}
while (!container.empty() && p(container.back())) {
container.pop_back();
}
}
enum data_format_state_t {
DFS_ERROR = -1,
DFS_INIT,
DFS_KEY,
DFS_VALUE,
};
struct data_format {
data_format(data_token_t appender = DT_INVALID,
data_token_t terminator = DT_INVALID)
: df_appender(appender), df_terminator(terminator)
{
};
const data_token_t df_appender;
const data_token_t df_terminator;
};
data_format_state_t dfs_semi_next(data_format_state_t state,
data_token_t next_token);
data_format_state_t dfs_comma_next(data_format_state_t state,
data_token_t next_token);
class data_parser {
public:
static data_format FORMAT_SEMI;
static data_format FORMAT_COMMA;
static data_format FORMAT_PLAIN;
typedef byte_array<20> schema_id_t;
struct element;
typedef std::list<element> element_list_t;
struct element {
element() : e_token(DT_INVALID), e_sub_elements(NULL) { };
element(std::list<element> &subs, data_token_t token)
element(element_list_t &subs,
data_token_t token,
bool assign_subs_elements = true)
: e_capture(subs.front().e_capture.c_begin,
subs.back().e_capture.c_end),
e_token(token),
e_sub_elements(NULL) {
if (assign_subs_elements) {
this->assign_elements(subs);
}
};
element(const element &other) {
assert(other.e_sub_elements == NULL);
// assert(other.e_sub_elements == NULL);
this->e_capture = other.e_capture;
this->e_token = other.e_token;
this->e_sub_elements = NULL;
if (other.e_sub_elements != NULL)
this->assign_elements(*other.e_sub_elements);
};
~element() {
@ -84,10 +187,19 @@ public:
}
};
void assign_elements(std::list<element> &subs) {
element& operator=(const element &other) {
this->e_capture = other.e_capture;
this->e_token = other.e_token;
this->e_sub_elements = NULL;
if (other.e_sub_elements != NULL)
this->assign_elements(*other.e_sub_elements);
return *this;
};
void assign_elements(element_list_t &subs) {
if (this->e_sub_elements == NULL)
this->e_sub_elements = new std::list<element>();
this->e_sub_elements->splice(this->e_sub_elements->end(), subs);
this->e_sub_elements = new element_list_t();
this->e_sub_elements->swap(subs);
this->update_capture();
};
@ -100,9 +212,22 @@ public:
}
};
data_token_t value_token(void) const {
data_token_t retval = DT_INVALID;
if (this->e_token == DNT_VALUE &&
this->e_sub_elements != NULL &&
this->e_sub_elements->size() == 1) {
retval = this->e_sub_elements->front().e_token;
}
return retval;
};
void print(FILE *out, pcre_input &pi, int offset = 0) {
int lpc;
if (this->e_sub_elements != NULL) {
for (std::list<data_parser::element>::iterator iter2 =
for (element_list_t::iterator iter2 =
this->e_sub_elements->begin();
iter2 != this->e_sub_elements->end();
++iter2) {
@ -114,7 +239,7 @@ public:
data_scanner::token2name(this->e_token),
this->e_capture.c_begin,
this->e_capture.c_end);
for (int lpc = 0; lpc < this->e_capture.c_end; lpc++) {
for (lpc = 0; lpc < this->e_capture.c_end; lpc++) {
if (lpc == this->e_capture.c_begin)
fputc('^', out);
else if (lpc == (this->e_capture.c_end - 1))
@ -124,13 +249,18 @@ public:
else
fputc(' ', out);
}
fputc('\n', out);
for (; lpc < (int)pi.pi_length; lpc++) {
fputc(' ', out);
}
std::string sub = pi.get_substr(&this->e_capture);
fprintf(out, " %s\n", sub.c_str());
};
pcre_context::capture_t e_capture;
data_token_t e_token;
std::list<element> *e_sub_elements;
element_list_t *e_sub_elements;
};
struct element_cmp {
@ -154,94 +284,343 @@ public:
data_token_t ei_token;
};
data_parser(data_scanner *ds) : dp_scanner(ds) { };
data_parser(data_scanner *ds) : dp_format(NULL), dp_scanner(ds) { };
void parse(void) {
pcre_context_static<30> pc;
struct element elem;
while (this->dp_scanner->tokenize(pc, elem.e_token)) {
elem.e_capture = *(pc.begin());
void pairup(schema_id_t *schema, element_list_t &pairs_out, element_list_t &in_list) {
element_list_t el_stack, free_row, key_comps, value, prefix;
SHA_CTX context;
for (element_list_t::iterator iter = in_list.begin();
iter != in_list.end();
++iter) {
if (iter->e_token == DNT_GROUP) {
element_list_t group_pairs;
this->reduce(elem);
this->pairup(NULL, group_pairs, *iter->e_sub_elements);
if (!group_pairs.empty()) {
iter->assign_elements(group_pairs);
}
}
if (iter->e_token == DT_SEPARATOR) {
element_list_t::iterator key_iter = key_comps.end();
bool found = false;
--key_iter;
for (;
key_iter != key_comps.begin() && !found;
--key_iter) {
if (key_iter->e_token == this->dp_format->df_appender) {
++key_iter;
value.splice(value.end(),
key_comps,
key_comps.begin(),
key_iter);
key_comps.splice(key_comps.begin(),
key_comps,
key_comps.end());
key_comps.resize(1);
found = true;
}
else if (key_iter->e_token == this->dp_format->df_terminator) {
std::vector<element> key_copy;
value.splice(value.end(),
key_comps,
key_comps.begin(),
key_iter);
++key_iter;
key_comps.pop_front();
strip(key_comps, element_if(DT_WHITE));
found = true;
}
}
if (!found && !el_stack.empty() && !key_comps.empty()) {
element_list_t::iterator value_iter;
value.splice(value.end(),
key_comps,
key_comps.begin(),
key_comps.end());
value_iter = value.end();
std::advance(value_iter, -1);
key_comps.splice(key_comps.begin(),
value,
value_iter);
key_comps.resize(1);
}
strip(value, element_if(DT_WHITE));
value.remove_if(element_if(DT_COMMA));
if (!value.empty()) {
el_stack.push_back(element(value, DNT_VALUE));
}
strip(key_comps, element_if(DT_WHITE));
if (!key_comps.empty()) {
el_stack.push_back(element(key_comps, DNT_KEY, false));
}
key_comps.clear();
value.clear();
}
else {
key_comps.push_back(*iter);
}
}
};
void reduce(const element &elem);
if (el_stack.empty()) {
free_row.splice(free_row.begin(),
key_comps, key_comps.begin(), key_comps.end());
}
else {
value.splice(value.begin(),
key_comps,
key_comps.begin(),
key_comps.end());
strip(value, element_if(DT_WHITE));
value.remove_if(element_if(DT_COMMA));
if (!value.empty()) {
el_stack.push_back(element(value, DNT_VALUE));
}
}
SHA_Init(&context);
while (!el_stack.empty()) {
element_list_t::iterator kv_iter = el_stack.begin();
if (kv_iter->e_token == DNT_VALUE) {
free_row.push_back(el_stack.front());
}
if (kv_iter->e_token != DNT_KEY) {
el_stack.pop_front();
continue;
}
++kv_iter;
if (kv_iter == el_stack.end()) {
el_stack.pop_front();
continue;
}
bool reducePattern(std::list<element> &reduction,
const data_token_t *pattern_start,
const data_token_t *pattern_end,
bool repeating = false);
if (kv_iter->e_token != DNT_VALUE) {
el_stack.pop_front();
continue;
}
bool reduceAnyOf(std::list<element> &reduction,
const data_token_t *possibilities_start,
const data_token_t *possibilities_end) {
std::list<element>::iterator iter;
bool retval = false;
std::string key_val = this->get_element_string(el_stack.front());
element_list_t pair_subs;
reduction.clear();
if (schema != NULL) {
SHA_Update(&context, key_val.c_str(), key_val.length());
}
iter = find_first_not_of(this->dp_stack.begin(),
this->dp_stack.end(),
possibilities_start, possibilities_end,
element_cmp());
if (iter != this->dp_stack.begin()) {
reduction.splice(reduction.end(),
this->dp_stack,
this->dp_stack.begin(),
iter);
++kv_iter;
pair_subs.splice(pair_subs.begin(),
el_stack,
el_stack.begin(),
kv_iter);
pairs_out.push_back(element(pair_subs, DNT_PAIR));
}
retval = true;
if (pairs_out.size() == 1) {
element &pair = pairs_out.front();
element &value = pair.e_sub_elements->back();
if (value.e_token == DNT_VALUE &&
value.e_sub_elements != NULL &&
value.e_sub_elements->size() > 1) {
prefix.splice(prefix.begin(),
*pair.e_sub_elements,
pair.e_sub_elements->begin());
free_row.clear();
free_row.splice(free_row.begin(),
*value.e_sub_elements,
value.e_sub_elements->begin(),
value.e_sub_elements->end());
pairs_out.clear();
SHA_Init(&context);
}
}
reduction.reverse();
return retval;
if (pairs_out.empty() && !free_row.empty()) {
while (!free_row.empty()) {
switch (free_row.front().e_token) {
case DNT_GROUP:
case DT_NUMBER:
case DT_SYMBOL:
case DT_HEX_NUMBER:
case DT_OCTAL_NUMBER:
case DT_VERSION_NUMBER:
case DT_QUOTED_STRING:
case DT_IPV4_ADDRESS:
case DT_IPV6_ADDRESS:
case DT_MAC_ADDRESS:
case DT_UUID:
case DT_URL:
case DT_PATH:
case DT_TIME:
case DT_PERCENTAGE: {
element_list_t pair_subs;
struct element blank;
blank.e_capture.c_begin = blank.e_capture.c_end = free_row.front().e_capture.c_begin;
blank.e_token = DNT_KEY;
pair_subs.push_back(blank);
pair_subs.push_back(free_row.front());
pairs_out.push_back(element(pair_subs, DNT_PAIR));
}
break;
default: {
std::string key_val = this->get_element_string(free_row.front());
SHA_Update(&context, key_val.c_str(), key_val.length());
}
break;
}
free_row.pop_front();
}
}
if (!prefix.empty()) {
element_list_t pair_subs;
struct element blank;
blank.e_capture.c_begin = blank.e_capture.c_end = prefix.front().e_capture.c_begin;
blank.e_token = DNT_KEY;
pair_subs.push_back(blank);
pair_subs.push_back(prefix.front());
pairs_out.push_front(element(pair_subs, DNT_PAIR));
}
if (schema != NULL)
SHA_Final(this->dp_schema_id.ba_data, &context);
};
void discover_format(void) {
pcre_context_static<30> pc;
int hist[DT_TERMINAL_MAX];
struct element elem;
bool reduceUpTo(std::list<element> &reduction,
const data_token_t *possibilities_start,
const data_token_t *possibilities_end) {
std::list<element>::iterator iter;
bool retval = false;
this->dp_group_token.push_back(DT_INVALID);
this->dp_group_stack.resize(1);
reduction.clear();
data_format_state_t semi_state = DFS_INIT;
data_format_state_t comma_state = DFS_INIT;
iter = std::find_first_of(this->dp_stack.begin(), this->dp_stack.end(),
possibilities_start, possibilities_end,
element_cmp());
if (iter != this->dp_stack.end()) {
reduction.splice(reduction.end(),
this->dp_stack,
this->dp_stack.begin(),
iter);
memset(hist, 0, sizeof(hist));
while (this->dp_scanner->tokenize(pc, elem.e_token)) {
pcre_context::iterator pc_iter;
pc_iter = std::find_if(pc.begin(), pc.end(), capture_if_not(-1));
assert(pc_iter != pc.end());
elem.e_capture = *pc_iter;
assert(elem.e_capture.c_begin != -1);
assert(elem.e_capture.c_end != -1);
semi_state = dfs_semi_next(semi_state, elem.e_token);
comma_state = dfs_comma_next(comma_state, elem.e_token);
hist[elem.e_token] += 1;
switch (elem.e_token) {
case DT_LPAREN:
case DT_LANGLE:
case DT_LCURLY:
case DT_LSQUARE:
this->dp_group_token.push_back(elem.e_token);
this->dp_group_stack.push_back(element_list_t());
break;
case DT_RPAREN:
case DT_RANGLE:
case DT_RCURLY:
case DT_RSQUARE:
if (this->dp_group_token.back() == (elem.e_token - 1)) {
this->dp_group_token.pop_back();
std::list<element_list_t>::reverse_iterator riter = this->dp_group_stack.rbegin();
++riter;
if (!this->dp_group_stack.back().empty()) {
(*riter).push_back(element(this->dp_group_stack.back(), DNT_GROUP));
}
this->dp_group_stack.pop_back();
}
else {
this->dp_group_stack.back().push_back(elem);
}
break;
default:
this->dp_group_stack.back().push_back(elem);
break;
}
}
while (this->dp_group_stack.size() > 1) {
this->dp_group_token.pop_back();
retval = true;
std::list<element_list_t>::reverse_iterator riter = this->dp_group_stack.rbegin();
++riter;
if (!this->dp_group_stack.back().empty()) {
(*riter).push_back(element(this->dp_group_stack.back(), DNT_GROUP));
}
this->dp_group_stack.pop_back();
}
reduction.reverse();
if (semi_state != DFS_ERROR && hist[DT_SEMI]) {
this->dp_format = &FORMAT_SEMI;
}
else if (comma_state != DFS_ERROR) {
this->dp_format = &FORMAT_COMMA;
}
else {
this->dp_format = &FORMAT_PLAIN;
}
};
void parse(void) {
return retval;
this->discover_format();
this->pairup(&this->dp_schema_id,
this->dp_pairs,
this->dp_group_stack.front());
for (element_list_t::iterator iter = this->dp_pairs.begin();
iter != this->dp_pairs.end();
++iter) {
if (iter->e_token == DNT_PAIR) {
element_list_t &pair_subs = *iter->e_sub_elements;
std::string key_val = this->get_element_string(pair_subs.front());
}
}
};
void reduceQual(const struct element &lookahead);
void reduceRow(void);
void reducePair(void);
std::string get_element_string(element &elem) {
pcre_input &pi = this->dp_scanner->get_input();
return pi.get_substr(&elem.e_capture);
};
void print(FILE *out) {
void print(FILE *out, element_list_t &el) {
fprintf(out, " %s\n",
this->dp_scanner->get_input().get_string());
for (std::list<data_parser::element>::iterator iter = this->dp_stack.begin();
iter != this->dp_stack.end();
for (element_list_t::iterator iter = el.begin();
iter != el.end();
++iter) {
iter->print(out, this->dp_scanner->get_input());
}
};
std::list<element> dp_qual;
std::list<element> dp_stack;
std::vector<data_token_t> dp_group_token;
std::list<element_list_t> dp_group_stack;
element_list_t dp_errors;
element_list_t dp_pairs;
schema_id_t dp_schema_id;
data_format *dp_format;
private:
data_scanner *dp_scanner;

@ -29,6 +29,8 @@
#include "config.h"
#include <arpa/inet.h>
#include "pcrepp.hh"
#include "data_scanner.hh"
@ -38,33 +40,50 @@ static struct {
const char *name;
pcrepp pcre;
} MATCHERS[DT_TERMINAL_MAX] = {
{ "url", pcrepp("([\\w]+://[^\\s'\"\\[\\](){}]+)"), },
{ "path", pcrepp("(?<![\\w\\d-_])((?:/|\\./|\\.\\./)[\\w\\d\\.-_\\~/]+)"), },
{ "quot", pcrepp("(?:u|r)?\"((?:\\\\.|[^\"])+)\"|"
"(?:u|r)?'((?:\\\\.|[^'])+)'"), },
{ "url", pcrepp("([\\w]+://[^\\s'\"\\[\\](){}]+[a-zA-Z0-9\\-=&])"), },
{ "path", pcrepp("(?<![\\w\\d-_])((?:/|\\./|\\.\\./)[\\w\\.\\-_\\~/]+)"), },
{ "mac", pcrepp("([0-9a-fA-F][0-9a-fA-F](?::[0-9a-fA-F][0-9a-fA-F]){5})"), },
{ "time", pcrepp("\\b(\\d?\\d:\\d\\d(:\\d\\d)?(:\\d\\d)?([,.]\\d{3})?)\\b"), }, // XXX be more specific
{ "mac", pcrepp("([0-9a-fA-F][0-9a-fA-F](?::[0-9a-fA-F][0-9a-fA-F]){5,5})"), },
{ "quot", pcrepp("u?\"([^\"]+)\"|u'([^']+(?:'\\w[^']*)*)'"), },
// { "qual", pcrepp("([^\\s:=]+:[^\\s:=,]+(?!,)(?::[^\\s:=,]+)*)"), },
{ "ipv6", pcrepp("(::|[:\\da-fA-f\\.]+[a-fA-f\\d])"), },
{ "sep", pcrepp("(:|=)"), },
{ "comm", pcrepp("(,|/)"), },
{ "semi", pcrepp("(;)"), },
{ "lcurly", pcrepp("({)"), },
{ "rcurly", pcrepp("(})"), },
{ "lsquare", pcrepp("(\\[)"), },
{ "rsquare", pcrepp("(\\])"), },
{ "lparen", pcrepp("(\\()"), },
{ "rparen", pcrepp("(\\))"), },
{ "langle", pcrepp("(\\<)"), },
{ "rangle", pcrepp("(\\>)"), },
{ "ipv4", pcrepp("(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})"), },
{ "uuid", pcrepp("([0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})"), },
{ "vers", pcrepp("([0-9]+(?:\\.[0-9]+){2,}\\b)"), },
{ "oct", pcrepp("(-?0[0-7]+\\b)"), },
{ "pcnt", pcrepp("(-?[0-9]+(\\.[0-9]+)?[ ]*%\\b)"), },
{ "num", pcrepp("(-?[0-9]+(\\.[0-9]+)?([eE][-+][0-9]+)?\\b)"), },
{ "hex", pcrepp("(-?(?:0x|[0-9])[0-9a-fA-F]+\\b)"), },
{ "word", pcrepp("([^\"';\\s:=,/(){}\\[\\]]+)"), },
{ "word", pcrepp("([a-zA-Z][a-z']+(?=[\\s\\(\\)!\\*:;'\\\"\\?,]|\\.\\s|$))"), },
{ "sym", pcrepp("([^\";\\s:=,/(){}\\[\\]]+)"), },
{ "line", pcrepp("(\r?\n|\r|;)"), },
{ "whit", pcrepp("([ \r\t]+)"), },
{ "wspc", pcrepp("([ \r\t]+)"), },
{ "dot", pcrepp("(\\.)"), },
{ "gbg", pcrepp("(.)"), },
};
const char *DNT_NAMES[] = {
const char *DNT_NAMES[DNT_MAX - DNT_KEY] = {
"key",
"pair",
"val",
@ -74,6 +93,7 @@ const char *DNT_NAMES[] = {
"var",
"rang",
"date",
"grp",
};
const char *data_scanner::token2name(data_token_t token)
@ -88,6 +108,24 @@ const char *data_scanner::token2name(data_token_t token)
return DNT_NAMES[token - DNT_KEY];
}
static
bool find_string_end(const char *str, size_t &start, size_t length, char term)
{
for (; start < length; start++) {
if (str[start] == term) {
start += 1;
return true;
}
if (str[start] == '\\') {
if (start + 1 >= length) {
return false;
}
start += 1;
}
}
return false;
}
bool data_scanner::tokenize(pcre_context &pc, data_token_t &token_out)
{
int lpc;
@ -102,14 +140,74 @@ bool data_scanner::tokenize(pcre_context &pc, data_token_t &token_out)
this->ds_pcre_input.pi_next_offset += 1;
token_out = DT_LINE;
return true;
return false;
}
for (lpc = 0; lpc < DT_TERMINAL_MAX; lpc++) {
if (MATCHERS[lpc].pcre.match(pc, this->ds_pcre_input, PCRE_ANCHORED)) {
token_out = data_token_t(lpc);
break;
}
switch (lpc) {
case DT_QUOTED_STRING: {
pcre_input &pi = this->ds_pcre_input;
const char *str = pi.get_string();
size_t str_start, str_end;
bool found = false;
pi.pi_offset = pi.pi_next_offset;
str_end = str_start = pi.pi_offset + 1;
switch (str[pi.pi_offset]) {
case 'u':
case 'r':
if (pi.pi_offset + 1 < pi.pi_length &&
(str[pi.pi_offset + 1] == '\'' ||
str[pi.pi_offset + 1] == '\"')) {
str_start += 1;
str_end += 1;
found = find_string_end(str,
str_end,
pi.pi_length,
str[pi.pi_offset]);
}
break;
case '\'':
case '\"':
found = find_string_end(str,
str_end,
pi.pi_length,
str[pi.pi_offset]);
break;
}
if (found) {
token_out = data_token_t(DT_QUOTED_STRING);
pi.pi_next_offset = str_end;
pc.all()[0].c_begin = pi.pi_offset;
pc.all()[0].c_end = str_end;
pc.all()[1].c_begin = str_start;
pc.all()[1].c_end = str_end - 1;
pc.set_count(2);
return true;
}
}
break;
default:
if (MATCHERS[lpc].pcre.match(pc, this->ds_pcre_input, PCRE_ANCHORED)) {
switch (lpc) {
case DT_IPV6_ADDRESS: {
std::string addr = this->ds_pcre_input.get_substr(pc.all());
char buf[sizeof(struct in6_addr)];
if (inet_pton(AF_INET6, addr.c_str(), buf) == 1) {
token_out = data_token_t(lpc);
return true;
}
this->ds_pcre_input.pi_next_offset = this->ds_pcre_input.pi_offset;
break;
}
default:
token_out = data_token_t(lpc);
return true;
}
}
break;
}
}
assert((0 <= token_out && token_out < DT_TERMINAL_MAX));

@ -36,18 +36,33 @@
enum data_token_t {
DT_INVALID = -1,
DT_URL = 0,
DT_QUOTED_STRING = 0,
DT_URL,
DT_PATH,
DT_TIME,
DT_MAC_ADDRESS,
DT_QUOTED_STRING,
DT_TIME,
DT_IPV6_ADDRESS,
// DT_QUALIFIED_NAME,
DT_SEPARATOR,
DT_COMMA,
DT_SEMI,
DT_LCURLY,
DT_RCURLY,
DT_IP_ADDRESS,
DT_LSQUARE,
DT_RSQUARE,
DT_LPAREN,
DT_RPAREN,
DT_LANGLE,
DT_RANGLE,
DT_IPV4_ADDRESS,
DT_UUID,
DT_VERSION_NUMBER,
DT_OCTAL_NUMBER,
@ -55,7 +70,8 @@ enum data_token_t {
DT_NUMBER,
DT_HEX_NUMBER,
DT_STRING,
DT_WORD,
DT_SYMBOL,
DT_LINE,
DT_WHITE,
DT_DOT,
@ -73,6 +89,9 @@ enum data_token_t {
DNT_VARIABLE_KEY,
DNT_ROWRANGE,
DNT_DATE_TIME,
DNT_GROUP,
DNT_MAX,
DT_ANY = 100,
};
@ -84,6 +103,9 @@ public:
data_scanner(const std::string &line) :
ds_line(line),
ds_pcre_input(ds_line.c_str()) {
if (!line.empty() && line[line.length() - 1] == '.') {
this->ds_pcre_input.pi_length -= 1;
}
};
bool tokenize(pcre_context &pc, data_token_t &token_out);

@ -33,7 +33,9 @@
#include <string>
#include <vector>
#include "listview_curses.hh"
#include "hist_source.hh"
#include "log_vtab_impl.hh"
class db_label_source : public hist_source::label_source {
public:
@ -43,16 +45,6 @@ public:
void hist_label_for_group(int group, std::string &label_out) {
label_out.clear();
for (int lpc = 0; lpc < (int)this->dls_headers.size(); lpc++) {
int before, total_fill =
this->dls_column_sizes[lpc] - this->dls_headers[lpc].length();
before = total_fill / 2;
total_fill -= before;
label_out.append(before, ' ');
label_out.append(this->dls_headers[lpc]);
label_out.append(total_fill, ' ');
}
};
void hist_label_for_bucket(int bucket_start_value,
@ -67,11 +59,42 @@ public:
if (bucket_start_value >= (int)this->dls_rows.size())
return;
for (int lpc = 0; lpc < (int)this->dls_rows[bucket_start_value].size(); lpc++) {
label_out.append(this->dls_column_sizes[lpc] - this->dls_rows[bucket_start_value][lpc].length(), ' ');
int padding = (this->dls_column_sizes[lpc] -
this->dls_rows[bucket_start_value][lpc].length() -
1);
if (this->dls_column_types[lpc] != SQLITE3_TEXT) {
label_out.append(padding, ' ');
}
label_out.append(this->dls_rows[bucket_start_value][lpc]);
if (this->dls_column_types[lpc] == SQLITE3_TEXT) {
label_out.append(padding, ' ');
}
label_out.append(1, ' ');
}
};
void hist_attrs_for_bucket(int bucket_start_value,
const hist_source::bucket_t &bucket,
string_attrs_t &sa) {
struct line_range lr = { 0, 0 };
struct line_range lr2 = { 0, -1 };
if (bucket_start_value >= (int)this->dls_rows.size())
return;
for (size_t lpc = 0; lpc < this->dls_column_sizes.size() - 1; lpc++) {
if (bucket_start_value % 2 == 0) {
sa[lr2].insert(make_string_attr("style", A_BOLD));
}
lr.lr_start += this->dls_column_sizes[lpc] - 1;
lr.lr_end = lr.lr_start + 1;
sa[lr].insert(make_string_attr("graphic", ACS_VLINE));
lr.lr_start += 1;
}
}
// TODO: add support for left and right justification... numbers should
// be right justified and strings should be left.
void push_column(const char *colstr) {
int index = this->dls_rows.back().size();
@ -83,7 +106,7 @@ public:
std::max(this->dls_column_sizes[index], strlen(colstr) + 1);
};
void push_header(const std::string &colstr) {
void push_header(const std::string &colstr, int type, bool graphable) {
int index = this->dls_headers.size();
this->dls_headers.push_back(colstr);
@ -92,17 +115,74 @@ public:
}
this->dls_column_sizes[index] =
std::max(this->dls_column_sizes[index], colstr.length() + 1);
this->dls_column_types.push_back(type);
this->dls_headers_to_graph.push_back(graphable);
}
void clear(void) {
this->dls_headers.clear();
this->dls_headers_to_graph.clear();
this->dls_column_types.clear();
this->dls_rows.clear();
this->dls_column_sizes.clear();
}
std::vector< std::string > dls_headers;
std::vector< bool > dls_headers_to_graph;
std::vector<int> dls_column_types;
std::vector< std::vector< std::string > > dls_rows;
std::vector< size_t > dls_column_sizes;
};
class db_overlay_source : public list_overlay_source {
public:
db_overlay_source() : dos_labels(NULL) { };
size_t list_overlay_count(const listview_curses &lv) {
return 1;
};
bool list_value_for_overlay(const listview_curses &lv,
vis_line_t y,
attr_line_t &value_out) {
view_colors &vc = view_colors::singleton();
if (y != 0) {
return false;
}
std::string &line = value_out.get_string();
db_label_source *dls = this->dos_labels;
for (size_t lpc = 0;
lpc < this->dos_labels->dls_column_sizes.size();
lpc++) {
int before, total_fill =
dls->dls_column_sizes[lpc] - dls->dls_headers[lpc].length();
struct line_range header_range = { line.length(), line.length() + dls->dls_column_sizes[lpc] };
int attrs = vc.attrs_for_role(this->dos_hist_source->get_role_for_type(bucket_type_t(lpc))) | A_UNDERLINE;
if (!this->dos_labels->dls_headers_to_graph[lpc]) {
attrs = A_UNDERLINE;
}
value_out.get_attrs()[header_range].insert(make_string_attr("style", attrs));
before = total_fill / 2;
total_fill -= before;
line.append(before, ' ');
line.append(dls->dls_headers[lpc]);
line.append(total_fill, ' ');
}
struct line_range lr = { 0, -1 };
value_out.get_attrs()[lr].insert(make_string_attr("style", A_BOLD | A_UNDERLINE));
return true;
};
db_label_source *dos_labels;
hist_source *dos_hist_source;
};
#endif

File diff suppressed because it is too large Load Diff

@ -57,9 +57,8 @@ On color displays, the lines will be highlighted as follows:
* Errors will be colored in red;
* warnings will be yellow;
* boundaries between days will be underlined; and
* various color highlights will be applied to: SQL keywords, XML
tags, file and line numbers in Java backtraces, and
quoted strings.
* various color highlights will be applied to: IP addresses, SQL keywords,
XML tags, file and line numbers in Java backtraces, and quoted strings.
To give you an idea of where you are in the file spatially, the right
side of the display has a proportionally sized 'scrollbar' that
@ -221,6 +220,9 @@ through the file.
that can be used in queries. See the SQL section
below for more information.
y/Y Move forward/backward through the log view based on the
"line_number" column in the SQL result view.
v Switch to/from the SQL result view.
V Switch between the log and SQL result views while
@ -232,6 +234,11 @@ through the file.
to the log view and move to the line number that was
selected in the "line_number" column.
p Enable or disable the display of the fields that the
log message parser knows about or has discovered.
This overlay is temporarily enabled when the semicolon
key (;) is pressed so that it is easier to write queries.
MOUSE SUPPORT (experimental)
----------------------------
@ -244,6 +251,8 @@ environment variable to "mouse".
COMMANDS
--------
help Switch to this help text view.
unix-time <secs-or-date>
Convert a unix-timestamp in seconds to a
human-readable form or vice-versa.
@ -297,7 +306,7 @@ COMMANDS
filter-related commands can be added to the session file.
SQL QUERIES
SQL QUERIES (experimental)
-----------
Lnav has support for performing SQL queries on log files using the
@ -312,24 +321,46 @@ being accessed in any loaded Apache log files, you can execute:
The query result view shows the results as text and graphs any number
values found in the result, much like the histogram view.
The basic set of log file tables are:
The builtin set of log tables are listed below. Note that only the
log messages that match a particular format can be queried by a
particular table. You can find the file format and table name for
the top log message by looking in the upper right hand corner of the
log file view.
syslog_log, generic_log
Contains any syslog lines and any lines picked up
by the generic log file parser.
The log table names are as follows:
access_log Apache common access log format
syslog_log Syslog format
glog_log Google glog format
strace_log Strace log format
generic_log 'Generic' log format. This table contains messages
from files that have a very simple format with a
leading timestamp followed by the message.
The columns available for the top log line in the view will
automatically be displayed after pressing the semicolon (;) key.
All log tables contain at least the following columns:
line_number The line number in the file, starting at zero.
path The full path to the file.
log_time The time of the log entry.
idle_msecs The amount of time, in milliseconds, between the
current log message and the previous one.
level The log level (e.g. info, error, etc...).
path The full path to the file.
raw_line The raw line of text.
The following tables include the basic columns as listed above and
include a few more columns since the log file format is more
structured.
access_log Contains Apache log file lines. The column names
are the same as those in the Microsoft LogParser.
syslog_log
log_hostname The hostname the message was received from.
log_procname The name of the process that sent the message.
log_pid The process ID of the process that sent the message.
access_log (The column names are the same as those in the
Microsoft LogParser tool.)
c_ip The client IP address.
cs_username The client user name.
@ -342,8 +373,9 @@ structured.
cs_referrer The URL of the referring page.
cs_user_agent The user agent string.
strace_log Contains strace log file lines. Currently, you
need to run strace with the "-tt -T" options.
strace_log (Currently, you need to run strace with the
"-tt -T" options so there are timestamps for
each function call.)
funcname The name of the syscall.
result The result code.
@ -360,3 +392,51 @@ example of a top ten query into the "/tmp/topten.db" file, you can do:
;create table topten as select cs_uri_stem, count(*) as total
from access_log group by cs_uri_stem order by total desc
limit 10;
DYNAMIC LOG LINE TABLE (experimental)
----------------------
(NOTE: This feature is still very new and not completely reliable yet,
use with care.)
For log formats that lack message structure, lnav can parse the log
message and attempt to extract any data fields that it finds. This
feature is available through the "logline" log table. This table is
dynamically created and defined based on the message at the top of
the log view. For example, given the following log message from "sudo",
lnav will create the "logline" table with columns for "TTY", "PWD",
"USER", and "COMMAND":
May 24 06:48:38 Tim-Stacks-iMac.local sudo[76387]: stack : TTY=ttys003 ;
PWD=/Users/stack/github/lbuild ; USER=root ;
COMMAND=/bin/echo Hello, World!
Queries executed against this table will then only return results for
other log messages that have the same format. So, if you were to
execute the following query while viewing the above line, you might
get the following results:
;select USER,COMMAND from logline;
USER | COMMAND
---- | -------------------------
root | /bin/echo Hello, World!
mal | /bin/echo Goodbye, World!
The log parser works by examining each message for key/value pairs
separated by an equal sign (=) or a colon (:). For example, in the
previous example of a "sudo" message, the parser sees the "USER=root"
string as a pair where the key is "USER" and the value is "root".
If no pairs can be found, then anything that looks like a value is
extracted and assigned a numbered column. For example, the following
line is from "dhcpd":
Sep 16 22:35:57 drill dhcpd: DHCPDISCOVER from 00:16:ce:54:4e:f3 via hme3
In this case, the lnav parser recognizes that "DHCPDISCOVER", the MAC
address and the "hme3" device name are values and not normal words. So,
it builds a table with three columns for each of these values. The
regular words in the message, like "from" and "via", are then used to
find other messages with a similar format.

@ -88,6 +88,11 @@ void hist_source::text_attrs_for_line(textview_curses &tc,
int row,
string_attrs_t &value_out)
{
int grow = row / (this->buckets_per_group() + 1);
int brow = row % (this->buckets_per_group() + 1);
bucket_group_t bg = this->hs_group_keys[grow];
int bucket_index = brow - 1;
assert(this->hs_analyzed);
if (this->hs_token_bucket != NULL) {
@ -120,9 +125,14 @@ void hist_source::text_attrs_for_line(textview_curses &tc,
lr.lr_end = lr.lr_start + amount;
value_out[lr].insert(make_string_attr("style", attrs));
lr.lr_start = lr.lr_end;
}
this->hs_label_source->hist_attrs_for_bucket((bg * this->hs_group_size) +
(bucket_index * this->hs_bucket_size),
*this->hs_token_bucket,
value_out);
}
}

@ -79,6 +79,10 @@ public:
virtual void hist_label_for_bucket(int bucket_start_value,
const bucket_t &bucket,
std::string &label_out) { };
virtual void hist_attrs_for_bucket(int bucket_start_value,
const bucket_t &bucket,
string_attrs_t &sa) { };
};
hist_source();
@ -186,6 +190,21 @@ public:
void add_value(unsigned int value,
bucket_type_t bt,
bucket_count_t amount = 1.0);
void add_empty_value(unsigned int value) {
bucket_group_t bg;
this->hs_analyzed = false;
bg = bucket_group_t(value / this->hs_group_size);
bucket_array_t &ba = this->hs_groups[bg];
if (ba.empty()) {
ba.resize(this->buckets_per_group());
}
};
void analyze(void);
protected:

@ -125,11 +125,18 @@ bool listview_curses::handle_key(int ch)
void listview_curses::do_update(void)
{
if (this->lv_window != NULL && this->lv_needs_update) {
vis_line_t y(this->lv_y), height, bottom, lines;
vis_line_t y(this->lv_y), height, bottom, lines, row;
attr_line_t overlay_line;
vis_line_t overlay_height(0);
struct line_range lr;
unsigned long width;
size_t row_count;
if (this->lv_overlay_source != NULL) {
overlay_height = vis_line_t(
this->lv_overlay_source->list_overlay_count(*this));
}
this->get_dimensions(height, width);
lr.lr_start = this->lv_left;
lr.lr_end = this->lv_left + width;
@ -139,20 +146,31 @@ void listview_curses::do_update(void)
this->lv_top = max(vis_line_t(0), vis_line_t(row_count) - height);
}
lines = y + min(height, vis_line_t(row_count) - this->lv_top);
row = this->lv_top;
lines = min(height - overlay_height,
vis_line_t(row_count) - this->lv_top);
bottom = y + height;
for (; y < lines; ++y) {
vis_line_t row = this->lv_top + y - vis_line_t(this->lv_y);
attr_line_t al;
this->lv_source->listview_value_for_row(*this, row, al);
this->mvwattrline(this->lv_window, y, 0, al, lr);
}
/* Clear out any remaining lines on the display. */
for (; y < bottom; ++y) {
wmove(this->lv_window, y, 0);
wclrtoeol(this->lv_window);
if (this->lv_overlay_source != NULL &&
this->lv_overlay_source->list_value_for_overlay(
*this,
y - vis_line_t(this->lv_y),
overlay_line)) {
this->mvwattrline(this->lv_window, y, 0, overlay_line, lr);
overlay_line.clear();
}
else if (lines > 0) {
attr_line_t al;
this->lv_source->listview_value_for_row(*this, row, al);
this->mvwattrline(this->lv_window, y, 0, al, lr);
--lines;
++row;
}
else {
wmove(this->lv_window, y, 0);
wclrtoeol(this->lv_window);
}
}
if (this->lv_show_scrollbar) {

@ -35,6 +35,7 @@
#include <sys/types.h>
#include <string>
#include <vector>
#include <algorithm>
#include "strong_int.hh"
@ -66,6 +67,31 @@ public:
attr_line_t &value_out) = 0;
};
struct listview_overlay {
listview_overlay(int y, const attr_line_t &al) : lo_y(y), lo_line(al) { };
int get_absolute_y(int height) {
if (this->lo_y >= 0)
return this->lo_y;
return height + this->lo_y;
};
int lo_y;
attr_line_t lo_line;
};
class list_overlay_source {
public:
virtual ~list_overlay_source() { };
virtual size_t list_overlay_count(const listview_curses &lv) = 0;
virtual bool list_value_for_overlay(const listview_curses &lv,
vis_line_t y,
attr_line_t &value_out) = 0;
};
/**
* View that displays a list of lines that can optionally contain highlighting.
*/
@ -87,7 +113,19 @@ public:
};
/** @return The data source delegate. */
list_data_source *get_data_source() { return this->lv_source; };
list_data_source *get_data_source() const { return this->lv_source; };
/** @param src The data source delegate. */
void set_overlay_source(list_overlay_source *src)
{
this->lv_overlay_source = src;
this->reload_data();
};
/** @return The overlay source delegate. */
list_overlay_source *get_overlay_source() {
return this->lv_overlay_source;
};
/**
* @param va The action to invoke when the view is scrolled.
@ -265,6 +303,7 @@ public:
else {
height_out = this->lv_height;
}
};
/** This method should be called when the data source has changed. */
@ -283,6 +322,7 @@ public:
protected:
list_data_source *lv_source; /*< The data source delegate. */
list_overlay_source *lv_overlay_source;
action lv_scroll; /*< The scroll action. */
WINDOW *lv_window; /*< The window that contains this view. */
unsigned int lv_y; /*< The y offset of this view. */

File diff suppressed because it is too large Load Diff

@ -120,6 +120,7 @@ struct _lnav_data {
std::set< std::pair<std::string, int> > ld_file_names;
std::list<logfile *> ld_files;
std::list<std::string> ld_other_files;
sig_atomic_t ld_looping;
sig_atomic_t ld_winched;
unsigned long ld_flags;
@ -152,6 +153,8 @@ struct _lnav_data {
hist_source ld_db_source;
db_label_source ld_db_rows;
db_overlay_source ld_db_overlay;
std::vector<std::string> ld_db_key_names;
int ld_max_fd;
fd_set ld_read_fds;
@ -168,6 +171,7 @@ void rebuild_indexes(bool force);
std::string dotlnav_path(const char *sub);
void ensure_view(textview_curses *expected_tc);
bool toggle_view(textview_curses *toggle_tc);
#endif

@ -159,13 +159,15 @@ static string com_save_to(string cmdline, vector<string> &args)
return "error: expecting file name";
}
snprintf(command, sizeof(command), "echo -n %s", args[1].c_str());
snprintf(command, sizeof(command), "/bin/echo -n %s", args[1].c_str());
if ((pfile = popen(command, "r")) == NULL) {
return "error: unable to compute file name";
}
if (fgets(command, sizeof(command), pfile) == 0)
if (fgets(command, sizeof(command), pfile) == 0) {
perror("fgets");
return "error: unable to compute file name";
}
fclose(pfile);
pfile = NULL;
@ -280,6 +282,20 @@ static string com_graph(string cmdline, vector<string> &args)
return retval;
}
static string com_help(string cmdline, vector<string> &args)
{
string retval = "";
if (args.size() == 0) {
}
else {
ensure_view(&lnav_data.ld_views[LNV_HELP]);
}
return retval;
}
class pcre_filter
: public logfile_filter {
public:
@ -507,6 +523,7 @@ void init_lnav_commands(readline_context::command_map_t &cmd_map)
cmd_map["current-time"] = com_current_time;
cmd_map["goto"] = com_goto;
cmd_map["graph"] = com_graph;
cmd_map["help"] = com_help;
cmd_map["highlight"] = com_highlight;
cmd_map["filter-in"] = com_filter;
cmd_map["filter-out"] = com_filter;

@ -41,6 +41,8 @@
#include <vector>
#include <memory>
#include "view_curses.hh"
/**
* Metadata for a single line in a log file.
*/
@ -139,6 +141,65 @@ private:
uint8_t ll_module;
};
class logline_value {
public:
enum kind_t {
VALUE_TEXT,
VALUE_INTEGER,
VALUE_FLOAT,
};
logline_value(std::string name, int64_t i)
: lv_name(name), lv_kind(VALUE_INTEGER), lv_number(i) { };
logline_value(std::string name, double i)
: lv_name(name), lv_kind(VALUE_FLOAT), lv_number(i) { };
logline_value(std::string name, std::string s)
: lv_name(name), lv_kind(VALUE_TEXT), lv_string(s) { };
logline_value(std::string name, kind_t kind, std::string s)
: lv_name(name), lv_kind(kind) {
switch (kind) {
case VALUE_TEXT:
this->lv_string = s;
break;
case VALUE_INTEGER:
sscanf(s.c_str(), "%qd", &this->lv_number.i);
break;
case VALUE_FLOAT:
sscanf(s.c_str(), "%lf", &this->lv_number.d);
break;
}
};
const std::string to_string() {
char buffer[128];
switch (this->lv_kind) {
case VALUE_TEXT:
return this->lv_string;
case VALUE_INTEGER:
snprintf(buffer, sizeof(buffer), "%qd", this->lv_number.i);
break;
case VALUE_FLOAT:
snprintf(buffer, sizeof(buffer), "%lf", this->lv_number.d);
break;
}
return std::string(buffer);
};
std::string lv_name;
kind_t lv_kind;
union value_u {
int64_t i;
double d;
value_u() : i(0) { };
value_u(int64_t i) : i(i) { };
value_u(double d) : d(d) { };
} lv_number;
std::string lv_string;
};
/**
* Base class for implementations of log format parsers.
*/
@ -199,6 +260,11 @@ public:
* @param line The log line to edit.
*/
virtual void scrub(std::string &line) { };
virtual void annotate(const std::string &line,
string_attrs_t &sa,
std::vector<logline_value> &values) const
{ };
virtual std::auto_ptr<log_format> specialized(void) = 0;

@ -76,6 +76,17 @@ static string scrub_rdns(const string &str)
}
class access_log_format : public log_format {
static pcrepp &value_pattern(void) {
static pcrepp VALUE_PATTERN(
"^([\\w\\.-]+) [\\w\\.-]+ ([\\w\\.-]+) "
"\\[([^\\]]+)\\] \"(?:\\-|(\\w+) ([^ \\?]+)(\\?[^ ]+)? "
"([\\w/\\.]+))\" (\\d+) "
"(\\d+|-)(?: \"([^\"]+)\" \"([^\"]+)\")?.*");
return VALUE_PATTERN;
};
string get_name() { return "access_log"; };
bool scan(vector < logline > &dst,
@ -126,12 +137,75 @@ class access_log_format : public log_format {
return retval;
};
void annotate(const std::string &line,
string_attrs_t &sa,
std::vector<logline_value> &values) const {
pcre_context_static<30> pc;
pcre_input pi(line);
if (value_pattern().match(pc, pi)) {
static struct {
const char *name;
logline_value::kind_t kind;
} columns[] = {
{ "c_ip", logline_value::VALUE_TEXT },
{ "cs_username", logline_value::VALUE_TEXT },
{ "", },
{ "cs_method", logline_value::VALUE_TEXT },
{ "cs_uri_stem", logline_value::VALUE_TEXT },
{ "cs_uri_query", logline_value::VALUE_TEXT },
{ "cs_version", logline_value::VALUE_TEXT },
{ "sc_status", logline_value::VALUE_INTEGER },
{ "sc_bytes", logline_value::VALUE_INTEGER },
{ "cs_referer", logline_value::VALUE_TEXT },
{ "cs_user_agent", logline_value::VALUE_TEXT },
{ NULL },
};
pcre_context::iterator iter;
struct line_range lr;
iter = pc.begin() + 3;
lr.lr_start = iter->c_begin;
lr.lr_end = iter->c_end;
sa[lr].insert(make_string_attr("timestamp", 0));
lr.lr_start = 0;
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("prefix", 0));
lr.lr_start = line.length();
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("body", 0));
for (int lpc = 0; columns[lpc].name; lpc++) {
if (columns[lpc].name[0] == '\0')
continue;
values.push_back(logline_value(columns[lpc].name,
columns[lpc].kind,
pi.get_substr(pc.begin() + lpc)));
}
}
else {
fprintf(stderr, "bad match! %s\n", line.c_str());
}
};
};
log_format::register_root_format<access_log_format> access_log_instance;
class syslog_log_format : public log_format {
static const int TIMESTAMP_LENGTH = 16;
static pcrepp &repeated_pattern(void) {
static pcrepp REPEATED_PATTERN("last message repeated \\d+ times");
return REPEATED_PATTERN;
}
static pcrepp &scrub_pattern(void) {
static pcrepp SCRUB_PATTERN("(\\w+\\s[\\s\\d]\\d \\d+:\\d+:\\d+) [\\.\\-\\w]+( .*)");
@ -169,6 +243,90 @@ class syslog_log_format : public log_format {
}
};
void annotate(const string &line,
string_attrs_t &sa,
std::vector<logline_value> &values) const {
bool found_hostname = false, found_procname = false, found_prefix = false;
struct line_range lr, hostname_range = { 0, 0 };
struct line_range procname_range = { 0, 0 };
int log_pid = -1;
lr.lr_start = 0;
lr.lr_end = TIMESTAMP_LENGTH;
sa[lr].insert(make_string_attr("timestamp", 0));
for (size_t lpc = TIMESTAMP_LENGTH; lpc < line.size(); lpc++) {
if (line[lpc] == ' ' && !found_hostname) {
hostname_range.lr_start = TIMESTAMP_LENGTH;
hostname_range.lr_end = lpc;
sa[hostname_range].insert(make_string_attr("log_hostname", 0));
found_hostname = true;
}
if (line[lpc] == ' ' && found_hostname && !found_procname) {
bool done = false;
procname_range.lr_start = procname_range.lr_end = lpc + 1;
while (!done) {
switch (line[lpc]) {
case '\0':
done = true;
break;
case ':':
case '[':
procname_range.lr_end = lpc;
done = true;
break;
default:
lpc += 1;
break;
}
}
sa[procname_range].insert(make_string_attr("log_procname", 0));
found_procname = true;
}
if (line[lpc] == '[' && log_pid == -1) {
const char *line_c_str = line.c_str();
sscanf(&line_c_str[lpc + 1], "%d", &log_pid);
}
if (line[lpc] == ':') {
lr.lr_start = 0;
lr.lr_end = lpc + 1;
sa[lr].insert(make_string_attr("prefix", 0));
lr.lr_start = lpc + 1;
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("body", 0));
found_prefix = true;
break;
}
}
if (!found_prefix) {
lr.lr_start = 0;
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("prefix", 0));
lr.lr_start = line.length();
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("body", 0));
hostname_range.lr_start = 0;
hostname_range.lr_end = 0;
}
values.push_back(logline_value("log_hostname", line.substr(hostname_range.lr_start, hostname_range.length())));
values.push_back(logline_value("log_procname", line.substr(procname_range.lr_start, procname_range.length())));
values.push_back(logline_value("log_pid", (int64_t)log_pid));
};
bool scan(vector < logline > &dst,
off_t offset,
char *prefix,
@ -192,7 +350,10 @@ class syslog_log_format : public log_format {
logline::level_t ll = logline::LEVEL_UNKNOWN;
time_t log_gmt;
if (error_pattern().match(context, pi)) {
if (repeated_pattern().match(context, pi)) {
ll = logline::LEVEL_CONTINUED;
}
else if (error_pattern().match(context, pi)) {
ll = logline::LEVEL_ERROR;
}
else if (warning_pattern().match(context, pi)) {
@ -281,6 +442,24 @@ class generic_log_format : public log_format {
return SCRUB_PATTERN;
}
static const char**get_log_formats() {
static const char *log_fmt[] = {
"%63[0-9: ,.-]%63[^:]%n",
"%63[a-zA-Z0-9:-+/.] [%*x %63[^\n]%n",
"%63[a-zA-Z0-9:.,-] %63[^\n]%n",
"%63[a-zA-Z0-9: .,-] [%*[^]]]%63[^:]%n",
"%63[a-zA-Z0-9: .,-] %63[^\n]%n",
"[%63[0-9: .-] %*s %63[^\n]%n",
"[%63[a-zA-Z0-9: -+/]] %63[^\n]%n",
"[%63[a-zA-Z0-9: -+/]] [%63[a-zA-Z]]%n",
"[%63[a-zA-Z0-9: .-+/] %*s %63[^\n]%n",
"[%63[a-zA-Z0-9: -+/]] (%*d) %63[^\n]%n",
NULL
};
return log_fmt;
};
string get_name() { return "generic_log"; };
void scrub(string &line) {
@ -303,19 +482,6 @@ class generic_log_format : public log_format {
off_t offset,
char *prefix,
int len) {
static const char *log_fmt[] = {
"%63[0-9: ,.-]%63[^:]",
"%63[a-zA-Z0-9:-+/.] [%*x %63[^\n]",
"%63[a-zA-Z0-9:.,-] %63[^\n]",
"%63[a-zA-Z0-9: .,-] [%*[^]]]%63[^:]",
"%63[a-zA-Z0-9: .,-] %63[^\n]",
"[%63[0-9: .-] %*s %63[^\n]",
"[%63[a-zA-Z0-9: -+/]] %63[^\n]",
"[%63[a-zA-Z0-9: -+/]] [%63[a-zA-Z]]",
"[%63[a-zA-Z0-9: .-+/] %*s %63[^\n]",
"[%63[a-zA-Z0-9: -+/]] (%*d) %63[^\n]",
NULL
};
bool retval = false;
struct tm log_time;
@ -323,9 +489,10 @@ class generic_log_format : public log_format {
time_t line_time;
char level[64];
char *last_pos;
int prefix_len;
if ((last_pos = this->log_scanf(prefix,
log_fmt,
get_log_formats(),
2,
NULL,
timestr,
@ -333,7 +500,8 @@ class generic_log_format : public log_format {
line_time,
timestr,
level)) != NULL) {
level,
&prefix_len)) != NULL) {
uint16_t millis = 0;
/* Try to pull out the milliseconds value. */
@ -352,6 +520,30 @@ class generic_log_format : public log_format {
return retval;
};
void annotate(const string &line,
string_attrs_t &sa,
std::vector<logline_value> &values) const {
const char *fmt = get_log_formats()[this->lf_fmt_lock];
char timestr[64 + 32] = "";
char level[64] = "";
struct line_range lr;
int prefix_len;
sscanf(line.c_str(), fmt, timestr, level, &prefix_len);
lr.lr_start = 0 ? fmt[0] == '%' : 1;
lr.lr_end = lr.lr_start + strlen(timestr);
sa[lr].insert(make_string_attr("timestamp", 0));
lr.lr_start = 0;
lr.lr_end = prefix_len;
sa[lr].insert(make_string_attr("prefix", 0));
lr.lr_start = prefix_len;
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("body", 0));
};
auto_ptr<log_format> specialized() {
auto_ptr<log_format> retval((log_format *)
new generic_log_format(*this));
@ -363,6 +555,16 @@ class generic_log_format : public log_format {
log_format::register_root_format<generic_log_format> generic_log_instance;
class glog_log_format : public log_format {
static pcrepp &value_pattern(void) {
static pcrepp VALUE_PATTERN(
"\\s*(?:[IWECF])\\d+ \\d+:\\d+:\\d+\\.(\\d+)" // level, date
"\\s*([0-9]*)" // thread
"\\s*(.*):(\\d*)\\]" // filename:number
"\\s*(.*)");
return VALUE_PATTERN;
};
string get_name() { return "glog_log"; };
bool scan(vector < logline > &dst,
@ -419,11 +621,68 @@ class glog_log_format : public log_format {
return retval;
};
void annotate(const std::string &line,
string_attrs_t &sa,
std::vector<logline_value> &values) const {
pcre_context_static<30> pc;
pcre_input pi(line);
if (value_pattern().match(pc, pi)) {
static struct {
const char *name;
logline_value::kind_t kind;
} columns[] = {
{ "micros", logline_value::VALUE_INTEGER },
{ "thread", logline_value::VALUE_TEXT },
{ "src_file", logline_value::VALUE_TEXT },
{ "src_line", logline_value::VALUE_INTEGER },
{ "message", logline_value::VALUE_TEXT },
{ NULL }
};
pcre_context::iterator iter;
struct line_range lr;
lr.lr_start = 1;
lr.lr_end = 21;
sa[lr].insert(make_string_attr("timestamp", 0));
iter = pc.begin() + 4;
lr.lr_start = 0;
lr.lr_end = iter->c_begin;
sa[lr].insert(make_string_attr("prefix", 0));
lr.lr_start = iter->c_begin;
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("body", 0));
for (int lpc = 0; columns[lpc].name; lpc++) {
if (columns[lpc].name[0] == '\0')
continue;
values.push_back(logline_value(columns[lpc].name,
columns[lpc].kind,
pi.get_substr(pc.begin() + lpc)));
}
}
else {
fprintf(stderr, "bad match! %s\n", line.c_str());
}
};
};
log_format::register_root_format<glog_log_format> glog_instance;
class strace_log_format : public log_format {
static pcrepp &value_pattern(void) {
static pcrepp VALUE_PATTERN(
"[0-9:.]* ([a-zA-Z_][a-zA-Z_0-9]*)\\("
"(.*)\\)"
"\\s+= ([-xa-fA-F\\d\\?]+).*(?:<(\\d+\\.\\d+)>)?");
return VALUE_PATTERN;
};
string get_name() { return "strace_log"; };
bool scan(vector < logline > &dst,
@ -486,6 +745,54 @@ class strace_log_format : public log_format {
return retval;
};
void annotate(const std::string &line,
string_attrs_t &sa,
std::vector<logline_value> &values) const {
pcre_context_static<30> pc;
pcre_input pi(line);
if (value_pattern().match(pc, pi)) {
static struct {
const char *name;
logline_value::kind_t kind;
} columns[] = {
{ "funcname", logline_value::VALUE_TEXT },
{ "args", logline_value::VALUE_TEXT },
{ "result", logline_value::VALUE_TEXT },
{ "duration", logline_value::VALUE_TEXT },
{ NULL },
};
pcre_context::iterator iter;
struct line_range lr;
iter = pc.begin() + 3;
lr.lr_start = iter->c_begin;
lr.lr_end = iter->c_end;
sa[lr].insert(make_string_attr("timestamp", 0));
lr.lr_start = 0;
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("prefix", 0));
lr.lr_start = line.length();
lr.lr_end = line.length();
sa[lr].insert(make_string_attr("body", 0));
for (int lpc = 0; columns[lpc].name; lpc++) {
if (columns[lpc].name[0] == '\0')
continue;
values.push_back(logline_value(columns[lpc].name,
columns[lpc].kind,
pi.get_substr(pc.begin() + lpc)));
}
}
else {
fprintf(stderr, "bad match! %s\n", line.c_str());
}
};
};
log_format::register_root_format<strace_log_format> strace_log_instance;

@ -35,25 +35,52 @@
using namespace std;
static struct log_cursor log_cursor_latest;
static sql_progress_callback_t vtab_progress_callback;
static const char *type_to_string(int type)
{
switch (type) {
case SQLITE_FLOAT:
return "float";
case SQLITE_INTEGER:
return "integer";
case SQLITE_TEXT:
return "text";
}
assert("Invalid sqlite type");
return NULL;
}
static string declare_table_statement(log_vtab_impl *vi)
{
std::vector<log_vtab_impl::vtab_column> cols;
std::vector<log_vtab_impl::vtab_column>::const_iterator iter;
std::ostringstream oss;
oss << "CREATE TABLE unused (\n"
<< " line_number int,\n"
<< " path text,\n"
<< " line_number text,\n"
<< " log_time datetime,\n"
<< " idle_msecs int,\n"
<< " level text,\n"
<< " raw_line text";
<< " level text,\n";
vi->get_columns(cols);
vi->vi_column_count = cols.size();
for (iter = cols.begin(); iter != cols.end(); iter++) {
oss << ",\n";
oss << " " << iter->vc_name << " " << iter->vc_type;
auto_mem<char, sqlite3_free> coldecl;
coldecl = sqlite3_mprintf(" %Q %s collate %Q,\n",
iter->vc_name,
type_to_string(iter->vc_type),
iter->vc_collator == NULL ?
"BINARY" : iter->vc_collator);
oss << coldecl;
}
oss << "\n);";
oss << " path text collate naturalnocase,\n"
<< " raw_line text\n"
<< ");";
return oss.str();
}
@ -68,6 +95,7 @@ struct vtab {
struct vtab_cursor {
sqlite3_vtab_cursor base;
struct log_cursor log_cursor;
std::vector<logline_value> line_values;
};
static int vt_destructor(sqlite3_vtab *p_svt);
@ -95,6 +123,9 @@ static int vt_create( sqlite3 *db,
/* Declare the vtable's structure */
p_vt->vi = vm->lookup_impl(argv[3]);
if (p_vt->vi == NULL) {
return SQLITE_ERROR;
}
p_vt->lss = vm->get_source();
rc = sqlite3_declare_vtab(db, declare_table_statement(p_vt->vi).c_str());
@ -138,8 +169,7 @@ static int vt_open(sqlite3_vtab *p_svt, sqlite3_vtab_cursor **pp_cursor)
vtab* p_vt = (vtab*)p_svt;
p_vt->base.zErrMsg = NULL;
vtab_cursor *p_cur =
(vtab_cursor*)sqlite3_malloc(sizeof(vtab_cursor));
vtab_cursor *p_cur = (vtab_cursor*)new vtab_cursor();
*pp_cursor = (sqlite3_vtab_cursor*)p_cur;
@ -156,7 +186,7 @@ static int vt_close(sqlite3_vtab_cursor *cur)
vtab_cursor *p_cur = (vtab_cursor*)cur;
/* Free cursor struct. */
sqlite3_free(p_cur);
delete p_cur;
return SQLITE_OK;
}
@ -175,7 +205,13 @@ static int vt_next(sqlite3_vtab_cursor *cur)
vtab *vt = (vtab *)cur->pVtab;
bool done = false;
vc->line_values.clear();
do {
log_cursor_latest = vc->log_cursor;
if (vtab_progress_callback(log_cursor_latest)) {
done = true;
break;
}
done = vt->vi->next(vc->log_cursor, *vt->lss);
}
while (!done);
@ -202,16 +238,6 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
sqlite3_result_int64( ctx, vc->log_cursor.lc_curr_line );
}
break;
case VT_COL_PATH:
{
const string &fn = lf->get_filename();
sqlite3_result_text( ctx,
fn.c_str(),
fn.length(),
SQLITE_STATIC );
}
break;
case VT_COL_LOG_TIME:
{
time_t line_time;
@ -252,25 +278,57 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
SQLITE_STATIC);
}
break;
case VT_COL_RAW_LINE:
{
string line;
lf->read_line(ll, line);
sqlite3_result_text(ctx,
line.c_str(),
line.length(),
SQLITE_TRANSIENT);
}
break;
default:
{
logfile::iterator line_iter;
string line, value;
line_iter = lf->begin() + cl;
lf->read_line(line_iter, line);
vt->vi->extract(line, col - VT_COL_MAX, ctx);
if (col > (VT_COL_MAX + vt->vi->vi_column_count - 1)) {
int post_col_number = col - (VT_COL_MAX + vt->vi->vi_column_count - 1) - 1;
if (post_col_number == 0)
{
const string &fn = lf->get_filename();
sqlite3_result_text( ctx,
fn.c_str(),
fn.length(),
SQLITE_STATIC );
}
else {
string line;
lf->read_line(ll, line);
sqlite3_result_text(ctx,
line.c_str(),
line.length(),
SQLITE_TRANSIENT);
}
}
else {
if (vc->line_values.empty()) {
logfile::iterator line_iter;
string line, value;
line_iter = lf->begin() + cl;
lf->read_line(line_iter, line);
vt->vi->extract(lf, line, vc->line_values);
}
{
logline_value &lv = vc->line_values[col - VT_COL_MAX];
switch (lv.lv_kind) {
case logline_value::VALUE_TEXT:
sqlite3_result_text(ctx,
lv.lv_string.c_str(),
lv.lv_string.length(),
SQLITE_TRANSIENT);
break;
case logline_value::VALUE_INTEGER:
sqlite3_result_int64(ctx, lv.lv_number.i);
break;
case logline_value::VALUE_FLOAT:
sqlite3_result_double(ctx, lv.lv_number.d);
break;
}
}
}
break;
}
@ -322,30 +380,64 @@ static sqlite3_module vtab_module = {
NULL, /* xFindFunction - function overloading */
};
log_vtab_manager::log_vtab_manager(sqlite3 *memdb, logfile_sub_source &lss)
static int progress_callback(void *ptr)
{
int retval = 0;
if (vtab_progress_callback != NULL) {
retval = vtab_progress_callback(log_cursor_latest);
}
return retval;
}
log_vtab_manager::log_vtab_manager(sqlite3 *memdb,
logfile_sub_source &lss,
sql_progress_callback_t pc)
: vm_db(memdb), vm_source(lss)
{
sqlite3_create_module(this->vm_db, "log_vtab_impl", &vtab_module, this);
vtab_progress_callback = pc;
sqlite3_progress_handler(memdb, 10, progress_callback, NULL);
}
void log_vtab_manager::register_vtab(log_vtab_impl *vi) {
if (this->vm_impls.find(vi->get_name()) == this->vm_impls.end()) {
char *sql;
int rc;
this->vm_impls[vi->get_name()] = vi;
sql = sqlite3_mprintf("CREATE VIRTUAL TABLE %s "
"USING log_vtab_impl(%s)",
vi->get_name().c_str(),
vi->get_name().c_str());
rc = sqlite3_exec(this->vm_db,
sql,
NULL,
NULL,
NULL);
assert(rc == SQLITE_OK);
sqlite3_free(sql);
char *sql;
int rc;
this->vm_impls[vi->get_name()] = vi;
sql = sqlite3_mprintf("CREATE VIRTUAL TABLE %s "
"USING log_vtab_impl(%s)",
vi->get_name().c_str(),
vi->get_name().c_str());
rc = sqlite3_exec(this->vm_db,
sql,
NULL,
NULL,
NULL);
assert(rc == SQLITE_OK);
sqlite3_free(sql);
}
}
void log_vtab_manager::unregister_vtab(std::string name) {
if (this->vm_impls.find(name) != this->vm_impls.end()) {
char *sql;
int rc;
sql = sqlite3_mprintf("DROP TABLE %s ", name.c_str());
rc = sqlite3_exec(this->vm_db,
sql,
NULL,
NULL,
NULL);
assert(rc == SQLITE_OK);
sqlite3_free(sql);
}
this->vm_impls.erase(name);
}

@ -41,11 +41,9 @@
enum {
VT_COL_LINE_NUMBER,
VT_COL_PATH,
VT_COL_LOG_TIME,
VT_COL_IDLE_MSECS,
VT_COL_LEVEL,
VT_COL_RAW_LINE,
VT_COL_MAX
};
@ -59,11 +57,12 @@ struct log_cursor {
class log_vtab_impl {
public:
struct vtab_column {
vtab_column(const char *name, const char *type)
: vc_name(name), vc_type(type) { };
vtab_column(const char *name, int type, const char *collator=NULL)
: vc_name(name), vc_type(type), vc_collator(collator) { };
const char *vc_name;
const char *vc_type;
int vc_type;
const char *vc_collator;
};
log_vtab_impl(const std::string name)
@ -84,6 +83,11 @@ public:
content_line_t cl(lss.at(lc.lc_curr_line));
logfile *lf = lss.find(cl);
logfile::iterator lf_iter = lf->begin() + cl;
if (lf_iter->get_level() & logline::LEVEL_CONTINUED) {
return false;
}
log_format *format = lf->get_format();
if (format != NULL && format->get_name() == this->vi_name)
@ -94,22 +98,32 @@ public:
virtual void get_columns(std::vector<vtab_column> &cols) { };
virtual void extract(const std::string &line,
int column,
sqlite3_context *ctx) {
virtual void extract(logfile *lf,
const std::string &line,
std::vector<logline_value> &values) {
log_format *format = lf->get_format();
string_attrs_t sa;
format->annotate(line, sa, values);
};
int vi_column_count;
private:
const std::string vi_name;
};
typedef int (*sql_progress_callback_t)(const log_cursor &lc);
class log_vtab_manager {
public:
log_vtab_manager(sqlite3 *db, logfile_sub_source &lss);
log_vtab_manager(sqlite3 *db,
logfile_sub_source &lss,
sql_progress_callback_t);
logfile_sub_source *get_source() { return &this->vm_source; };
void register_vtab(log_vtab_impl *vi);
void unregister_vtab(std::string name);
log_vtab_impl *lookup_impl(std::string name) {
return this->vm_impls[name];
};

@ -168,7 +168,7 @@ void logfile_sub_source::text_value_for_line(textview_curses &tc,
size_t tab;
assert(row >= 0);
assert(row < this->lss_index.size());
assert((size_t)row < this->lss_index.size());
line = this->lss_index[row];
this->lss_token_file = this->find(line);
@ -309,13 +309,21 @@ void logfile_sub_source::text_attrs_for_line(textview_curses &lv,
lr.lr_end = -1;
value_out[lr].insert(make_string_attr("file", this->lss_token_file));
if (this->lss_token_date_end > 0 &&
((this->lss_token_line->get_time() / (60 * 60)) % 2) == 0) {
attrs = vc.attrs_for_role(view_colors::VCR_ALT_ROW);
lr.lr_start = time_offset_end;
lr.lr_end = this->lss_token_date_end;
value_out[lr].insert(make_string_attr("style", attrs));
if ((((this->lss_token_line->get_time() / (5 * 60)) % 2) == 0) &&
!(this->lss_token_line->get_level() & logline::LEVEL_CONTINUED)) {
log_format *format = this->lss_token_file->get_format();
std::vector<logline_value> line_values;
format->annotate(this->lss_token_value, value_out, line_values);
struct line_range time_range = find_string_attr_range(value_out, "timestamp");
if (time_range.lr_end != -1) {
time_range.lr_start += time_offset_end;
time_range.lr_end += time_offset_end;
attrs = vc.attrs_for_role(view_colors::VCR_ALT_ROW);
value_out[time_range].insert(make_string_attr("style", attrs));
}
}
}
@ -394,6 +402,9 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force)
++start_line;
}
}
else {
this->lss_filtered_count += 1;
}
start_line = con_line;
action = logfile_filter::MAYBE;
action_priority = -1;
@ -450,6 +461,9 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force)
++start_line;
}
}
else {
this->lss_filtered_count += 1;
}
iter->ld_lines_indexed = iter->ld_file->size();
@ -473,8 +487,12 @@ void logfile_sub_source::text_update_marks(vis_bookmarks &bm)
bm[&BM_WARNINGS].clear();
bm[&BM_ERRORS].clear();
bm[&BM_FILES].clear();
bm[&textview_curses::BM_USER].clear();
bm[&textview_curses::BM_SEARCH].clear();
for (bookmarks<content_line_t>::type::iterator iter = this->lss_user_marks.begin();
iter != this->lss_user_marks.end();
++iter) {
bm[iter->first].clear();
}
for (; vl < (int)this->lss_index.size(); ++vl) {
content_line_t cl = this->lss_index[vl];

@ -0,0 +1,169 @@
/**
* Copyright (c) 2013, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file nextwork-extension-functions.cc
*/
#include <stdio.h>
#include <assert.h>
#include <errno.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <arpa/inet.h>
#include "sqlite3.h"
static void sql_gethostbyname(sqlite3_context *context,
int argc, sqlite3_value **argv)
{
char buffer[INET6_ADDRSTRLEN];
const char *name_in;
struct addrinfo *ai;
void *addr_ptr;
int rc;
assert(argc >= 1 && argc <= 2);
if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
sqlite3_result_null(context);
return;
}
name_in = (const char *)sqlite3_value_text(argv[0]);
while ((rc = getaddrinfo(name_in, NULL, NULL, &ai)) == EAI_AGAIN) {
sqlite3_sleep(10);
}
if (rc != 0) {
sqlite3_result_text(context, name_in, -1, SQLITE_TRANSIENT);
return;
}
switch (ai->ai_family) {
case AF_INET:
addr_ptr = &((struct sockaddr_in *)ai->ai_addr)->sin_addr;
break;
case AF_INET6:
addr_ptr = &((struct sockaddr_in6 *)ai->ai_addr)->sin6_addr;
break;
}
inet_ntop(ai->ai_family, addr_ptr, buffer, sizeof(buffer));
sqlite3_result_text(context, buffer, -1, SQLITE_TRANSIENT);
freeaddrinfo(ai);
}
static void sql_gethostbyaddr(sqlite3_context *context,
int argc, sqlite3_value **argv)
{
union {
struct sockaddr_in sin;
struct sockaddr_in6 sin6;
} sa;
const char *addr_str;
char buffer[NI_MAXHOST];
int family, socklen;
char *addr_raw;
int rc;
assert(argc == 1);
if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
sqlite3_result_null(context);
return;
}
addr_str = (const char *)sqlite3_value_text(argv[0]);
memset(&sa, 0, sizeof(sa));
if (strchr(addr_str, ':')) {
family = AF_INET6;
socklen = sizeof(struct sockaddr_in6);
sa.sin6.sin6_family = family;
addr_raw = (char *)&sa.sin6.sin6_addr;
}
else {
family = AF_INET;
socklen = sizeof(struct sockaddr_in);
sa.sin.sin_family = family;
addr_raw = (char *)&sa.sin.sin_addr;
}
if (inet_pton(family, addr_str, addr_raw) != 1) {
sqlite3_result_text(context, addr_str, -1, SQLITE_TRANSIENT);
return;
}
while ((rc = getnameinfo((struct sockaddr *)&sa, socklen,
buffer, sizeof(buffer), NULL, 0, 0)) == EAI_AGAIN) {
sqlite3_sleep(10);
}
if (rc != 0) {
sqlite3_result_text(context, addr_str, -1, SQLITE_TRANSIENT);
return;
}
sqlite3_result_text(context, buffer, -1, SQLITE_TRANSIENT);
}
int register_network_extension_functions(sqlite3 *db)
{
static const struct {
const char *name;
char narg;
uint8_t text_rep;
void (*func)(sqlite3_context*,int,sqlite3_value**);
} plain_funcs[] = {
{ "gethostbyname", 1, SQLITE_UTF8, sql_gethostbyname },
{ "gethostbyaddr", 1, SQLITE_UTF8, sql_gethostbyaddr },
{ NULL }
};
int retval;
for (int lpc = 0; plain_funcs[lpc].name; lpc++) {
retval = sqlite3_create_function(db,
plain_funcs[lpc].name,
plain_funcs[lpc].narg,
plain_funcs[lpc].text_rep,
NULL,
plain_funcs[lpc].func,
NULL,
NULL);
if (retval != SQLITE_OK)
return retval;
}
return SQLITE_OK;
}

@ -100,7 +100,7 @@ public:
iterator begin() { return pc_captures + 1; };
/** @return An iterator that refers to the end of the capture array. */
iterator end() { return pc_captures + pc_count; };
protected:
pcre_context(capture_t *captures, int max_count)
: pc_captures(captures), pc_max_count(max_count), pc_count(0) { };
@ -110,6 +110,28 @@ protected:
int pc_count;
};
struct capture_if_not {
capture_if_not(int begin) : cin_begin(begin) { };
bool operator()(const pcre_context::capture_t &cap) {
return cap.c_begin != this->cin_begin;
}
int cin_begin;
};
inline
pcre_context::iterator skip_invalid_captures(pcre_context::iterator iter,
pcre_context::iterator pc_end)
{
for (; iter != pc_end; ++iter) {
if (iter->c_begin == -1)
continue;
}
return iter;
}
/**
* A pcre_context that allocates storage for the capture array within the object
* itself.
@ -152,6 +174,9 @@ public:
};
std::string get_substr(pcre_context::const_iterator iter) const {
if (iter->c_begin == -1) {
return "";
}
return std::string(this->pi_string,
iter->c_begin,
iter->length());

@ -132,8 +132,13 @@ char *readline_context::completion_generator(const char *text, int state)
for (iter = arg_possibilities->begin();
iter != arg_possibilities->end();
++iter) {
int (*cmpfunc)(const char *, const char *, size_t);
fprintf(stderr, " cmp %s %s\n", text, iter->c_str());
if (strncmp(text, iter->c_str(), len) == 0) {
cmpfunc = (loaded_context->is_case_sensitive() ?
strncmp : strncasecmp);
if (cmpfunc(text, iter->c_str(), len) == 0) {
fprintf(stderr, "match!!! %d %s %s\n", len, text, iter->c_str());
matches.push_back(*iter);
}
}
@ -143,6 +148,8 @@ char *readline_context::completion_generator(const char *text, int state)
if (!matches.empty()) {
retval = strdup(matches.back().c_str());
matches.pop_back();
fprintf(stderr, "comp gen %s\n", retval);
}
return retval;
@ -154,6 +161,8 @@ char **readline_context::attempted_completion(const char *text,
{
char **retval = NULL;
fprintf(stderr, "attempted %s\n", text);
if (loaded_context->rc_possibilities.find("*") != loaded_context->rc_possibilities.end()) {
fprintf(stderr, "all poss\n");
arg_possibilities = &loaded_context->rc_possibilities["*"];
@ -378,6 +387,9 @@ void readline_curses::start(void)
rem_possibility(string(type),
string(&msg[prompt_start]));
}
else if (sscanf(msg, "cp:%d:%s", &context, type)) {
this->rc_contexts[context]->clear_possibilities(type);
}
else {
fprintf(stderr, "unhandled message: %s\n", msg);
}
@ -594,6 +606,20 @@ void readline_curses::rem_possibility(int context, string type, string value)
}
}
void readline_curses::clear_possibilities(int context, string type)
{
char buffer[1024];
snprintf(buffer, sizeof(buffer),
"cp:%d:%s",
context, type.c_str());
if (reliable_send(this->rc_command_pipe[RCF_MASTER],
buffer,
strlen(buffer) + 1) == -1) {
perror("clear_possiblity: write failed");
}
}
void readline_curses::do_update(void)
{
if (this->rc_active_context == -1) {

@ -63,8 +63,11 @@ public:
std::vector<std::string> &args);
typedef std::map<std::string, command_t> command_map_t;
readline_context(const std::string &name, command_map_t *commands = NULL)
: rc_name(name)
readline_context(const std::string &name,
command_map_t *commands = NULL,
bool case_sensitive = true)
: rc_name(name),
rc_case_sensitive(case_sensitive)
{
char *home;
@ -97,6 +100,16 @@ public:
void load(void)
{
char buffer[128];
/*
* XXX Need to keep the input on a single line since the display screws
* up if it wraps around.
*/
snprintf(buffer, sizeof(buffer),
"set completion-ignore-case %s",
this->rc_case_sensitive ? "off" : "on");
rl_parse_and_bind(buffer); // NOTE: buffer is modified
loaded_context = this;
rl_attempted_completion_function = attempted_completion;
history_set_history_state(&this->rc_history);
@ -126,6 +139,15 @@ public:
this->rc_possibilities[type].erase(value);
};
void clear_possibilities(std::string type)
{
this->rc_possibilities[type].clear();
};
bool is_case_sensitive(void) const {
return this->rc_case_sensitive;
};
private:
static char **attempted_completion(const char *text, int start, int end);
static char *completion_generator(const char *text, int state);
@ -137,6 +159,7 @@ private:
HISTORY_STATE rc_history;
std::map<std::string, std::set<std::string> > rc_possibilities;
std::map<std::string, std::vector<std::string> > rc_prototypes;
bool rc_case_sensitive;
};
/**
@ -211,6 +234,7 @@ public:
void add_possibility(int context, std::string type, std::string value);
void rem_possibility(int context, std::string type, std::string value);
void clear_possibilities(int context, std::string type);
private:
enum {

@ -37,7 +37,7 @@ using namespace std;
void statusview_curses::do_update(void)
{
int top, attrs, field, field_count, left = 1, right;
int top, attrs, field, field_count, left = 1, right;
view_colors &vc = view_colors::singleton();
unsigned long width, height;
@ -60,6 +60,7 @@ void statusview_curses::do_update(void)
int x;
val = sf.get_value();
if (sf.is_right_justified()) {
right -= 1 + sf.get_width();
x = right;

@ -53,7 +53,8 @@ public:
sf_right_justify(false),
sf_cylon(false),
sf_cylon_pos(0),
sf_role(role) { };
sf_role(role),
sf_share(0) { };
virtual ~status_field() { };
@ -77,6 +78,14 @@ public:
value = abbrev;
}
}
if (this->sf_right_justify) {
int padding = this->sf_width - value.size();
if (padding > 2) {
value.insert(0, padding, ' ');
}
}
this->sf_value = value;
@ -136,13 +145,23 @@ public:
/** @param width The maximum display width, in characters. */
size_t get_width() const { return this->sf_width; };
/** @param width The maximum display width, in characters. */
void set_min_width(int width) { this->sf_min_width = width; };
/** @param width The maximum display width, in characters. */
size_t get_min_width() const { return this->sf_min_width; };
void set_share(int share) { this->sf_share = share; };
int get_share() const { return this->sf_share; };
protected:
size_t sf_width; /*< The maximum display width, in chars. */
size_t sf_min_width; /*< The maximum display width, in chars. */
bool sf_right_justify;
bool sf_cylon;
size_t sf_cylon_pos;
attr_line_t sf_value; /*< The value to display for this field. */
view_colors::role_t sf_role; /*< The color role for this field. */
int sf_share;
};
/**
@ -187,6 +206,39 @@ public:
void set_window(WINDOW *win) { this->sc_window = win; };
WINDOW *get_window() { return this->sc_window; };
void window_change(void) {
int field_count = this->sc_source->statusview_fields();
int remaining, total_shares = 0;
unsigned long width, height;
getmaxyx(this->sc_window, height, width);
remaining = width - 4;
for (int field = 0; field < field_count; field++) {
status_field &sf = this->sc_source->statusview_value_for_field(field);
remaining -= sf.get_share() ? sf.get_min_width() : sf.get_width();
remaining -= 1;
total_shares += sf.get_share();
}
if (remaining < 2) {
remaining = 0;
}
for (int field = 0; field < field_count; field++) {
status_field &sf = this->sc_source->statusview_value_for_field(field);
if (sf.get_share()) {
int actual_width;
actual_width = sf.get_min_width();
actual_width += remaining / (sf.get_share() / total_shares);
sf.set_width(actual_width);
}
}
};
void do_update(void);
private:

@ -0,0 +1,209 @@
/* -*- mode: c; c-file-style: "k&r" -*-
strnatcmp.c -- Perform 'natural order' comparisons of strings in C.
Copyright (C) 2000, 2004 by Martin Pool <mbp sourcefrog net>
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
*/
/* partial change history:
*
* 2004-10-10 mbp: Lift out character type dependencies into macros.
*
* Eric Sosman pointed out that ctype functions take a parameter whose
* value must be that of an unsigned int, even on platforms that have
* negative chars in their default char type.
*/
#include <ctype.h>
#include <string.h>
#include <assert.h>
#include <stdio.h>
#include "strnatcmp.h"
/* These are defined as macros to make it easier to adapt this code to
* different characters types or comparison functions. */
static inline int
nat_isdigit(nat_char a)
{
return isdigit((unsigned char) a);
}
static inline int
nat_isspace(nat_char a)
{
return isspace((unsigned char) a);
}
static inline nat_char
nat_toupper(nat_char a)
{
return toupper((unsigned char) a);
}
static int
compare_right(int a_len, nat_char const *a, int b_len, nat_char const *b)
{
int bias = 0;
/* The longest run of digits wins. That aside, the greatest
value wins, but we can't know that it will until we've scanned
both numbers to know that they have the same magnitude, so we
remember it in BIAS. */
for (;; a++, b++, a_len--, b_len--) {
if (a_len == 0 && b_len == 0)
return 0;
if (a_len == 0)
return -1;
if (b_len == 0)
return 1;
if (!nat_isdigit(*a) && !nat_isdigit(*b))
return bias;
else if (!nat_isdigit(*a))
return -1;
else if (!nat_isdigit(*b))
return +1;
else if (*a < *b) {
if (!bias)
bias = -1;
} else if (*a > *b) {
if (!bias)
bias = +1;
} else if (!*a && !*b)
return bias;
}
return 0;
}
static int
compare_left(int a_len, nat_char const *a, int b_len, nat_char const *b)
{
/* Compare two left-aligned numbers: the first to have a
different value wins. */
for (;; a++, b++, a_len--, b_len--) {
if (a_len == 0 && b_len == 0)
return 0;
if (a_len == 0)
return -1;
if (b_len == 0)
return 1;
if (!nat_isdigit(*a) && !nat_isdigit(*b))
return 0;
else if (!nat_isdigit(*a))
return -1;
else if (!nat_isdigit(*b))
return +1;
else if (*a < *b)
return -1;
else if (*a > *b)
return +1;
}
return 0;
}
static int strnatcmp0(int a_len, nat_char const *a,
int b_len, nat_char const *b,
int fold_case)
{
int ai, bi;
nat_char ca, cb;
int fractional, result;
assert(a && b);
ai = bi = 0;
while (1) {
if (ai >= a_len)
ca = 0;
else
ca = a[ai];
if (bi >= b_len)
cb = 0;
else
cb = b[bi];
/* skip over leading spaces or zeros */
while (nat_isspace(ca)) {
ai += 1;
if (ai >= a_len)
ca = 0;
else
ca = a[ai];
}
while (nat_isspace(cb)) {
bi += 1;
if (bi >= b_len)
cb = 0;
else
cb = b[bi];
}
/* process run of digits */
if (nat_isdigit(ca) && nat_isdigit(cb)) {
fractional = (ca == '0' || cb == '0');
if (fractional) {
if ((result = compare_left(a_len - ai, a+ai, b_len - bi, b+bi)) != 0)
return result;
} else {
if ((result = compare_right(a_len - ai, a+ai, b_len - bi, b+bi)) != 0)
return result;
}
}
if (!ca && !cb) {
/* The strings compare the same. Perhaps the caller
will want to call strcmp to break the tie. */
return 0;
}
if (fold_case) {
ca = nat_toupper(ca);
cb = nat_toupper(cb);
}
if (ca < cb)
return -1;
else if (ca > cb)
return +1;
++ai; ++bi;
}
}
int strnatcmp(int a_len, nat_char const *a, int b_len, nat_char const *b) {
return strnatcmp0(a_len, a, b_len, b, 0);
}
/* Compare, recognizing numeric string and ignoring case. */
int strnatcasecmp(int a_len, nat_char const *a, int b_len, nat_char const *b) {
return strnatcmp0(a_len, a, b_len, b, 1);
}

@ -0,0 +1,31 @@
/* -*- mode: c; c-file-style: "k&r" -*-
strnatcmp.c -- Perform 'natural order' comparisons of strings in C.
Copyright (C) 2000, 2004 by Martin Pool <mbp sourcefrog net>
This software is provided 'as-is', without any express or implied
warranty. In no event will the authors be held liable for any damages
arising from the use of this software.
Permission is granted to anyone to use this software for any purpose,
including commercial applications, and to alter it and redistribute it
freely, subject to the following restrictions:
1. The origin of this software must not be misrepresented; you must not
claim that you wrote the original software. If you use this software
in a product, an acknowledgment in the product documentation would be
appreciated but is not required.
2. Altered source versions must be plainly marked as such, and must not be
misrepresented as being the original software.
3. This notice may not be removed or altered from any source distribution.
*/
/* CUSTOMIZATION SECTION
*
* You can change this typedef, but must then also change the inline
* functions in strnatcmp.c */
typedef char nat_char;
int strnatcmp(int a_len, nat_char const *a, int b_len, nat_char const *b);
int strnatcasecmp(int a_len, nat_char const *a, int b_len, nat_char const *b);

@ -149,7 +149,7 @@ public:
}
} while (rc > 0);
for (int lpc = 0; lpc < range_queue.size(); lpc++) {
for (size_t lpc = 0; lpc < range_queue.size(); lpc++) {
sa[range_queue[lpc]].insert(attr_queue[lpc]);
}
};

@ -65,7 +65,8 @@ public:
this->tss_fields[TSF_ERRORS].set_role(view_colors::VCR_ALERT_STATUS);
this->tss_fields[TSF_FORMAT].set_width(15);
this->tss_fields[TSF_FORMAT].right_justify(true);
this->tss_fields[TSF_FILENAME].set_width(35); // XXX
this->tss_fields[TSF_FILENAME].set_min_width(35); // XXX
this->tss_fields[TSF_FILENAME].set_share(1);
this->tss_fields[TSF_FILENAME].right_justify(true);
};

@ -2,10 +2,10 @@
* Copyright (c) 2007-2012, Timothy Stack
*
* All rights reserved.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
@ -14,7 +14,7 @@
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@ -55,7 +55,7 @@ void view_curses::mvwattrline(WINDOW *window,
size_t exp_index = 0;
string full_line;
assert(lr.lr_end != -1);
assert(lr.lr_end >= 0);
line_width = lr.length();
buffer = (char *)alloca(line_width + 1);
@ -95,13 +95,18 @@ void view_curses::mvwattrline(WINDOW *window,
whline(window, ' ', lr.lr_end - full_line.size());
wattroff(window, attrs);
std::vector<line_range> graphic_range;
std::vector<int> graphic_in;
for (iter = sa.begin(); iter != sa.end(); ++iter) {
struct line_range attr_range = iter->first;
std::map<size_t, size_t>::iterator tab_iter;
assert(attr_range.lr_start >= 0);
assert(attr_range.lr_end >= -1);
fprintf(stderr, "attr %d %d\n", attr_range.lr_start, attr_range.lr_end);
tab_iter = tab_list.lower_bound(attr_range.lr_start);
if (tab_iter != tab_list.end())
attr_range.lr_start += (tab_iter->second - tab_iter->first) - 1;
@ -111,7 +116,7 @@ void view_curses::mvwattrline(WINDOW *window,
if (tab_iter != tab_list.end())
attr_range.lr_end += (tab_iter->second - tab_iter->first) - 1;
}
attr_range.lr_start = max(0, attr_range.lr_start - lr.lr_start);
if (attr_range.lr_end == -1) {
attr_range.lr_end = line_width;
@ -128,19 +133,36 @@ void view_curses::mvwattrline(WINDOW *window,
attrs = 0;
for (am_iter = am.begin(); am_iter != am.end(); ++am_iter) {
if (am_iter->first == "style") {
attrs |= am_iter->second.sa_int;
}
}
/* This silliness is brought to you by a buggy old curses lib. */
mvwinnstr(window, y, x + attr_range.lr_start, buffer, awidth);
wattron(window, attrs);
mvwaddnstr(window, y, x + attr_range.lr_start, buffer, awidth);
wattroff(window, attrs);
}
attrs = text_attrs; /* Reset attrs to regular text. */
if (am_iter->first == "style") {
attrs |= am_iter->second.sa_int;
}
}
if (attrs != 0) {
fprintf(stderr, "text %d %d %x\n", y, x + attr_range.lr_start, attrs);
/* This silliness is brought to you by a buggy old curses lib. */
mvwinnstr(window, y, x + attr_range.lr_start, buffer, awidth);
wattron(window, attrs);
mvwaddnstr(window, y, x + attr_range.lr_start, buffer, awidth);
wattroff(window, attrs);
}
for (am_iter = am.begin(); am_iter != am.end(); ++am_iter) {
if (am_iter->first == "graphic") {
graphic_range.push_back(attr_range);
graphic_in.push_back(am_iter->second.sa_int | attrs);
}
}
}
attrs = text_attrs; /* Reset attrs to regular text. */
}
for (size_t lpc = 0; lpc < graphic_range.size(); lpc++) {
for (int lpc2 = graphic_range[lpc].lr_start;
lpc2 < graphic_range[lpc].lr_end;
lpc2++) {
mvwaddch(window, y, lpc2, graphic_in[lpc]);
}
}
}
@ -180,6 +202,8 @@ view_colors::view_colors()
this->vc_role_colors[VCR_DIFF_ADD] = COLOR_PAIR(VC_GREEN);
this->vc_role_colors[VCR_DIFF_SECTION] = COLOR_PAIR(VC_MAGENTA);
this->vc_role_colors[VCR_SHADOW] = COLOR_PAIR(VC_GRAY);
for (lpc = 0; lpc < VCR__MAX; lpc++) {
this->vc_role_reverse_colors[lpc] =
this->vc_role_colors[lpc] | A_REVERSE;
@ -211,7 +235,7 @@ void view_colors::init(void)
init_pair(VC_CYAN, COLOR_CYAN, COLOR_BLACK);
init_pair(VC_GREEN, COLOR_GREEN, COLOR_BLACK);
init_pair(VC_MAGENTA, COLOR_MAGENTA, COLOR_BLACK);
init_pair(VC_BLUE_ON_WHITE, COLOR_BLUE, COLOR_WHITE);
init_pair(VC_CYAN_ON_BLACK, COLOR_CYAN, COLOR_BLACK);
init_pair(VC_GREEN_ON_WHITE, COLOR_GREEN, COLOR_WHITE);
@ -224,12 +248,14 @@ void view_colors::init(void)
init_pair(VC_BLACK_ON_WHITE, COLOR_BLACK, COLOR_WHITE);
init_pair(VC_RED_ON_WHITE, COLOR_RED, COLOR_WHITE);
init_pair(VC_YELLOW_ON_WHITE, COLOR_YELLOW, COLOR_WHITE);
init_pair(VC_WHITE_ON_GREEN, COLOR_WHITE, COLOR_GREEN);
init_pair(VC_GRAY, COLOR_BLACK, COLOR_BLACK);
}
}
view_colors::role_t view_colors::next_highlight(void)
view_colors::role_t view_colors::next_highlight()
{
role_t retval = (role_t)(VCR__MAX + this->vc_next_highlight);
@ -238,3 +264,13 @@ view_colors::role_t view_colors::next_highlight(void)
return retval;
}
view_colors::role_t view_colors::next_plain_highlight()
{
role_t retval = (role_t)(VCR__MAX + this->vc_next_plain_highlight);
this->vc_next_plain_highlight = (this->vc_next_plain_highlight + 2) %
(HL_COLOR_COUNT * 2);
return retval;
}

@ -92,6 +92,8 @@ struct line_range {
bool operator<(const struct line_range &rhs) const {
if (this->lr_start < rhs.lr_start) return true;
else if (this->lr_start > rhs.lr_start) return false;
if (this->lr_end == rhs.lr_end) return false;
if (this->lr_end < rhs.lr_end) return true;
return false;
@ -143,12 +145,31 @@ typedef std::multimap<std::string, string_attr_t> attrs_map_t;
/** A map of line ranges to attributes for that range. */
typedef std::map<struct line_range, attrs_map_t> string_attrs_t;
inline struct line_range
find_string_attr_range(const string_attrs_t &sa, const std::string &name) {
struct line_range retval = { -1, -1 };
for (string_attrs_t::const_iterator iter = sa.begin();
iter != sa.end();
++iter) {
attrs_map_t::const_iterator prefix_iter;
if ((prefix_iter = iter->second.find(name)) != iter->second.end()) {
retval = iter->first;
break;
}
}
return retval;
}
/**
* A line that has attributes.
*/
class attr_line_t {
public:
attr_line_t() { };
attr_line_t(const std::string &str) : al_string(str) { };
/** @return The string itself. */
std::string &get_string() { return this->al_string; };
@ -319,6 +340,8 @@ public:
VCR_DIFF_ADD, /*< Added line in a diff. */
VCR_DIFF_SECTION, /*< Section marker in a diff. */
VCR_SHADOW,
VCR__MAX
} role_t;
@ -357,7 +380,9 @@ public:
* method will iterate through eight-or-so attributes combinations so there
* is some variety in how text is highlighted.
*/
role_t next_highlight(void);
role_t next_highlight();
role_t next_plain_highlight();
enum {
VC_EMPTY = 0, /* XXX Dead color pair, doesn't work. */
@ -381,6 +406,8 @@ public:
VC_RED_ON_WHITE,
VC_WHITE_ON_GREEN,
VC_GRAY,
};
private:
@ -397,6 +424,7 @@ private:
int vc_role_reverse_colors[VCR__MAX + (HL_COLOR_COUNT * 2)];
/** The index of the next highlight color to use. */
int vc_next_highlight;
int vc_next_plain_highlight;
};
/**

@ -110,6 +110,7 @@ drive_data_scanner_SOURCES = \
../src/data_parser.cc \
../src/data_scanner.cc \
drive_data_scanner.cc
drive_data_scanner_LDADD = -lcrypto
drive_view_colors_SOURCES = \
../src/view_curses.cc \
@ -151,6 +152,8 @@ dist_noinst_DATA = \
datafile_simple.3 \
datafile_simple.4 \
datafile_simple.5 \
datafile_simple.6 \
datafile_simple.7 \
listview_output.0 \
listview_output.1 \
listview_output.2 \

@ -86,7 +86,6 @@ CONFIG_CLEAN_VPATH_FILES =
am_drive_data_scanner_OBJECTS = data_parser.$(OBJEXT) \
data_scanner.$(OBJEXT) drive_data_scanner.$(OBJEXT)
drive_data_scanner_OBJECTS = $(am_drive_data_scanner_OBJECTS)
drive_data_scanner_LDADD = $(LDADD)
drive_data_scanner_DEPENDENCIES =
am_drive_grep_proc_OBJECTS = line_buffer.$(OBJEXT) grep_proc.$(OBJEXT) \
drive_grep_proc.$(OBJEXT)
@ -651,6 +650,7 @@ drive_data_scanner_SOURCES = \
../src/data_scanner.cc \
drive_data_scanner.cc
drive_data_scanner_LDADD = -lcrypto
drive_view_colors_SOURCES = \
../src/view_curses.cc \
drive_view_colors.cc
@ -690,6 +690,8 @@ dist_noinst_DATA = \
datafile_simple.3 \
datafile_simple.4 \
datafile_simple.5 \
datafile_simple.6 \
datafile_simple.7 \
listview_output.0 \
listview_output.1 \
listview_output.2 \

@ -1,17 +1,14 @@
a=1 b=2 c=3,4
key 8:9 ^
sep 9:10 ^
num 10:11 ^
num 12:13 ^
row 10:13 ^-^
pair 8:13 ^---^
key 4:5 ^
sep 5:6 ^
num 6:7 ^
row 6:7 ^
pair 4:7 ^-^
key 0:1 ^
sep 1:2 ^
num 2:3 ^
row 2:3 ^
pair 0:3 ^-^
key 0:1 ^ a
num 2:3 ^ 1
val 2:3 ^ 1
pair 0:3 ^-^ a=1
key 4:5 ^ b
num 6:7 ^ 2
val 6:7 ^ 2
pair 4:7 ^-^ b=2
key 8:9 ^ c
num 10:11 ^ 3
num 12:13 ^ 4
val 10:13 ^-^ 3,4
pair 8:13 ^---^ c=3,4

@ -1,7 +1,7 @@
current speed: 38 mph
key 0:13 ^-----------^
sep 13:14 ^
num 15:17 ^^
word 18:21 ^-^
row 15:21 ^----^
pair 0:17 ^---------------^
key 0:0
key 0:13 ^-----------^ current speed
pair 0:13 ^-----------^ current speed
key 15:15 ^
num 15:17 ^^ 38
pair 15:17 ^^ 38

@ -1,9 +1,16 @@
1,2,3,4,five,six,7
num 0:1 ^
num 2:3 ^
num 4:5 ^
num 6:7 ^
word 8:12 ^--^
word 13:16 ^-^
num 17:18 ^
row 0:18 ^----------------^
key 0:0
num 0:1 ^ 1
pair 0:1 ^ 1
key 2:2 ^
num 2:3 ^ 2
pair 2:3 ^ 2
key 4:4 ^
num 4:5 ^ 3
pair 4:5 ^ 3
key 6:6 ^
num 6:7 ^ 4
pair 6:7 ^ 4
key 17:17 ^
num 17:18 ^ 7
pair 17:18 ^ 7

@ -1,9 +1,16 @@
1 2 3 4 five six 7
num 0:1 ^
num 2:3 ^
num 4:5 ^
num 6:7 ^
word 8:12 ^--^
word 13:16 ^-^
num 17:18 ^
row 0:18 ^----------------^
key 0:0
num 0:1 ^ 1
pair 0:1 ^ 1
key 2:2 ^
num 2:3 ^ 2
pair 2:3 ^ 2
key 4:4 ^
num 4:5 ^ 3
pair 4:5 ^ 3
key 6:6 ^
num 6:7 ^ 4
pair 6:7 ^ 4
key 17:17 ^
num 17:18 ^ 7
pair 17:18 ^ 7

@ -1,6 +1,5 @@
the-value: "Hello, World!"
key 0:9 ^-------^
sep 9:10 ^
quot 12:25 ^-----------^
row 12:25 ^-----------^
pair 0:25 ^-----------------------^
key 0:9 ^-------^ the-value
quot 12:25 ^-----------^ Hello, World!
val 12:25 ^-----------^ Hello, World!
pair 0:25 ^-----------------------^ the-value: "Hello, World!

@ -1,6 +1,5 @@
this is a url: http://www.example.com/foo-bar
key 0:13 ^-----------^
sep 13:14 ^
url 15:45 ^----------------------------^
row 15:45 ^----------------------------^
pair 0:45 ^-------------------------------------------^
key 0:13 ^-----------^ this is a url
url 15:45 ^----------------------------^ http://www.example.com/foo-bar
val 15:45 ^----------------------------^ http://www.example.com/foo-bar
pair 0:45 ^-------------------------------------------^ this is a url: http://www.example.com/foo-bar

@ -1,11 +1,9 @@
qualified:name: foo=1 bar=2
key 22:25 ^-^
sep 25:26 ^
num 26:27 ^
row 26:27 ^
pair 22:27 ^---^
key 16:19 ^-^
sep 19:20 ^
num 20:21 ^
row 20:21 ^
pair 16:21 ^---^
key 16:19 ^-^ foo
num 20:21 ^ 1
val 20:21 ^ 1
pair 16:21 ^---^ foo=1
key 22:25 ^-^ bar
num 26:27 ^ 2
val 26:27 ^ 2
pair 22:27 ^---^ bar=2

@ -0,0 +1,12 @@
func(arg1="a", arg2="b")
key 5:5 ^
key 5:9 ^--^ arg1
quot 11:12 ^ a
val 11:12 ^ a
pair 5:12 ^-----^ arg1="a
key 15:19 ^--^ arg2
quot 21:22 ^ b
val 21:22 ^ b
pair 15:22 ^-----^ arg2="b
grp 5:22 ^---------------^ arg1="a", arg2="b
pair 5:22 ^---------------^ arg1="a", arg2="b

@ -0,0 +1,31 @@
Succeeded authorizing right 'system.privilege.taskport.debug' by client '/usr/libexec/taskgated' [76339] for authorization created by '/usr/libexec/taskgated' [77395] (100003,1)
key 29:29 ^
quot 29:60 ^-----------------------------^ system.privilege.taskport.debug
pair 29:60 ^-----------------------------^ system.privilege.taskport.debug
key 73:73 ^
quot 73:95 ^--------------------^ /usr/libexec/taskgated
pair 73:95 ^--------------------^ /usr/libexec/taskgated
key 98:98 ^
key 98:98 ^
num 98:103 ^---^ 76339
pair 98:103 ^---^ 76339
grp 98:103 ^---^ 76339
pair 98:103 ^---^ 76339
key 135:135 ^
quot 135:157 ^--------------------^ /usr/libexec/taskgated
pair 135:157 ^--------------------^ /usr/libexec/taskgated
key 160:160 ^
key 160:160 ^
num 160:165 ^---^ 77395
pair 160:165 ^---^ 77395
grp 160:165 ^---^ 77395
pair 160:165 ^---^ 77395
key 168:168 ^
key 168:168 ^
num 168:174 ^----^ 100003
pair 168:174 ^----^ 100003
key 175:175 ^
num 175:176 ^ 1
pair 175:176 ^ 1
grp 168:176 ^------^ 100003,1
pair 168:176 ^------^ 100003,1

@ -1,32 +1,20 @@
Nov 3 09:47:02 veridian sudo: timstack : TTY=pts/6 ; PWD=/auto/wstimstack/rpms/lbuild/test ; USER=root ; COMMAND=/usr/bin/tail /var/log/messages
key 106:113 ^-----^
sep 113:114 ^
path 114:127 ^-----------^
path 128:145 ^---------------^
row 114:145 ^-----------------------------^
pair 106:127 ^-------------------^
key 94:98 ^--^
sep 98:99 ^
word 99:103 ^--^
row 99:103 ^--^
pair 94:103 ^-------^
key 54:57 ^-^
sep 57:58 ^
path 58:91 ^-------------------------------^
row 58:91 ^-------------------------------^
pair 54:91 ^-----------------------------------^
key 42:45 ^-^
sep 45:46 ^
word 46:49 ^-^
path 49:51 ^^
row 46:51 ^---^
pair 42:51 ^-------^
key 16:29 ^-----------^
sep 29:30 ^
word 31:39 ^------^
row 31:39 ^------^
pair 16:39 ^---------------------^
word 0:3 ^-^
num 5:6 ^
time 7:15 ^------^
date 0:15 ^-------------^
timstack : TTY=pts/6 ; PWD=/auto/wstimstack/rpms/lbuild/test ; USER=root ; COMMAND=/usr/bin/tail /var/log/messages
key 11:14 ^-^ TTY
sym 15:18 ^-^ pts
num 19:20 ^ 6
val 15:20 ^---^ pts/6
pair 11:20 ^-------^ TTY=pts/6
key 23:26 ^-^ PWD
path 27:60 ^-------------------------------^ /auto/wstimstack/rpms/lbuild/test
val 27:60 ^-------------------------------^ /auto/wstimstack/rpms/lbuild/test
pair 23:60 ^-----------------------------------^ PWD=/auto/wstimstack/rpms/lbuild/test
key 63:67 ^--^ USER
word 68:72 ^--^ root
val 68:72 ^--^ root
pair 63:72 ^-------^ USER=root
key 75:82 ^-----^ COMMAND
path 83:96 ^-----------^ /usr/bin/tail
wspc 96:97 ^
path 97:114 ^---------------^ /var/log/messages
val 83:114 ^-----------------------------^ /usr/bin/tail /var/log/messages
pair 75:114 ^-------------------------------------^ COMMAND=/usr/bin/tail /var/log/messages

@ -1,33 +1,21 @@
Jun 18 16:13:52 Tim-Stacks-iMac Safari[81045]: INSERT-HANG-DETECTED: Tx time:3.093364, # of Inserts: 89, # of bytes written: 465365, Did shrink: NO
key 137:143 ^----^
sep 143:144 ^
word 145:147 ^^
row 145:147 ^^
pair 137:147 ^--------^
key 107:123 ^--------------^
sep 123:124 ^
num 125:131 ^----^
word 133:136 ^-^
row 125:136 ^---------^
pair 107:136 ^---------------------------^
key 89:99 ^--------^
sep 99:100 ^
num 101:103 ^^
word 105:106 ^
row 101:106 ^---^
pair 89:106 ^---------------^
key 72:76 ^--^
sep 76:77 ^
num 77:85 ^------^
word 87:88 ^
row 77:88 ^---------^
pair 72:88 ^--------------^
key 47:67 ^------------------^
sep 67:68 ^
word 69:71 ^^
row 69:71 ^^
pair 47:71 ^----------------------^
word 0:3 ^-^
num 4:6 ^^
time 7:15 ^------^
date 0:15 ^-------------^
INSERT-HANG-DETECTED: Tx time:3.093364, # of Inserts: 89, # of bytes written: 465365, Did shrink: NO
key 0:20 ^------------------^ INSERT-HANG-DETECTED
word 22:24 ^^ Tx
val 22:24 ^^ Tx
pair 0:24 ^----------------------^ INSERT-HANG-DETECTED: Tx
key 25:29 ^--^ time
num 30:38 ^------^ 3.093364
val 30:38 ^------^ 3.093364
pair 25:38 ^-----------^ time:3.093364
key 40:52 ^----------^ # of Inserts
num 54:56 ^^ 89
val 54:56 ^^ 89
pair 40:56 ^--------------^ # of Inserts: 89
key 58:76 ^----------------^ # of bytes written
num 78:84 ^----^ 465365
val 78:84 ^----^ 465365
pair 58:84 ^------------------------^ # of bytes written: 465365
key 86:96 ^--------^ Did shrink
sym 98:100 ^^ NO
val 98:100 ^^ NO
pair 86:100 ^------------^ Did shrink: NO

@ -105,12 +105,10 @@ int main(int argc, char *argv[])
}
data_scanner ds(line.substr(13));
data_token_t token;
data_parser dp(&ds);
dp.parse();
dp.print(out);
dp.print(out, dp.dp_pairs);
fclose(out);
sprintf(cmd, "diff -u %s %s", argv[0], TMP_NAME);

Loading…
Cancel
Save