Archives/lnav
2
0
Fork 0
mirror of https://github.com/tstack/lnav synced 2024-11-01 21:40:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

[format] initial support for json formats and some bug fixes

Browse Source
This commit is contained in:
Timothy Stack 2013-09-10 06:20:37 -07:00
parent f61f87e22d
commit c59acba758
22 changed files with 932 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
src/lnav.cc
View File

@ -183,7 +183,19 @@ public:
content_line_t cl = lss.at(lv.get_top());
logfile * lf = lss.find(cl);
std::string line = lf->read_line(lf->begin() + cl);
logfile::iterator ll = lf->begin() + cl;
if (ll->is_continued()) {
if (this->fos_parser) {
delete this->fos_parser;
delete this->fos_scanner;
delete this->fos_namer;
}
this->fos_parser = NULL;
return 0;
}
std::string line = lf->read_line(ll);
struct line_range body;
string_attrs_t sa;
@ -214,7 +226,7 @@ public:
delete this->fos_scanner;
delete this->fos_namer;
}
this->fos_scanner = new data_scanner(line, body.lr_start, body.lr_end);
this->fos_parser = new data_parser(this->fos_scanner);
this->fos_parser->parse();
@ -506,10 +518,10 @@ public:
};
void logfile_sub_source_filtering(logfile_sub_source &lss,
content_line_t cl,
vis_line_t cl,
size_t total)
{
if (std::abs(cl - this->lo_last_line) > 1024 || (size_t)cl ==
if (std::abs(cl - this->lo_last_line) > (4 * 1024) || (size_t)cl ==
(total - 1)) {
lnav_data.ld_bottom_source.update_loading(cl, (total - 1));
this->do_update();
@ -531,7 +543,7 @@ private:
};
off_t lo_last_offset;
content_line_t lo_last_line;
vis_line_t lo_last_line;
};
static void rebuild_hist(size_t old_count, bool force)
@ -2007,7 +2019,7 @@ void execute_search(lnav_view_t view, const std::string &regex)
lnav_data.ld_last_search[view] = regex;
}
static void rl_search(void *dummy, readline_curses *rc)
static void rl_search_internal(void *dummy, readline_curses *rc, bool complete = false)
{
string term_val;
string name;
@ -2061,10 +2073,16 @@ static void rl_search(void *dummy, readline_curses *rc)
textview_curses *tc = lnav_data.ld_view_stack.top();
lnav_view_t index = (lnav_view_t)(tc - lnav_data.ld_views);
tc->set_top(lnav_data.ld_search_start_line);
if (!complete)
tc->set_top(lnav_data.ld_search_start_line);
execute_search(index, rc->get_value());
}
static void rl_search(void *dummy, readline_curses *rc)
{
rl_search_internal(dummy, rc);
}
static void rl_abort(void *dummy, readline_curses *rc)
{
textview_curses *tc = lnav_data.ld_view_stack.top();
@ -2112,7 +2130,7 @@ static void rl_callback(void *dummy, readline_curses *rc)
case LNM_SEARCH:
case LNM_CAPTURE:
rl_search(dummy, rc);
rl_search_internal(dummy, rc, true);
if (rc->get_value().size() > 0) {
lnav_data.ld_view_stack.top()->set_follow_search(false);
lnav_data.ld_rl_view->

33
src/lnav_commands.cc
View File

@ -349,24 +349,26 @@ static string com_highlight(string cmdline, vector<string> &args)
if (args.size() == 0) { }
else if (args.size() > 1) {
textview_curses *tc = lnav_data.ld_view_stack.top();
textview_curses::highlight_map_t &hm = tc->get_highlights();
const char *errptr;
pcre * code;
int eoff;
args[1] = cmdline.substr(cmdline.find(args[1]));
if ((code = pcre_compile(args[1].c_str(),
PCRE_CASELESS,
&errptr,
&eoff,
NULL)) == NULL) {
if (hm.find(args[1]) != hm.end()) {
retval = "error: highlight already exists";
}
else if ((code = pcre_compile(args[1].c_str(),
PCRE_CASELESS,
&errptr,
&eoff,
NULL)) == NULL) {
retval = "error: " + string(errptr);
}
else {
textview_curses * tc = lnav_data.ld_view_stack.top();
textview_curses::highlighter hl(code, false);
textview_curses::highlight_map_t &hm = tc->get_highlights();
hm[args[1]] = hl;
retval = "info: highlight pattern now active";
@ -491,20 +493,23 @@ static string com_filter(string cmdline, vector<string> &args)
args.push_back("filter");
}
else if (args.size() > 1) {
logfile_sub_source &lss = lnav_data.ld_log_source;
const char *errptr;
pcre * code;
int eoff;
args[1] = cmdline.substr(cmdline.find(args[1]));
if ((code = pcre_compile(args[1].c_str(),
0,
&errptr,
&eoff,
NULL)) == NULL) {
if (lss.get_filter(args[1]) != NULL) {
retval = "error: filter already exists";
}
else if ((code = pcre_compile(args[1].c_str(),
0,
&errptr,
&eoff,
NULL)) == NULL) {
retval = "error: " + string(errptr);
}
else {
logfile_sub_source & lss = lnav_data.ld_log_source;
logfile_filter::type_t lt = (args[0] == "filter-out") ?
logfile_filter::EXCLUDE :
logfile_filter::INCLUDE;

10
src/log_data_table.hh
View File

@ -50,7 +50,8 @@ public:
log_data_table(content_line_t template_line,
std::string table_name = "logline")
: log_vtab_impl(table_name),
ldt_template_line(template_line) {
ldt_template_line(template_line),
ldt_parent_column_count(0) {
logfile *lf = lnav_data.ld_log_source.find(template_line);
log_format *format = lf->get_format();
@ -71,6 +72,7 @@ public:
if (this->ldt_format_impl != NULL) {
this->ldt_format_impl->get_columns(cols);
}
this->ldt_parent_column_count = cols.size();
lf->read_full_message(lf->begin() + cl_copy, val);
format->annotate(val, sa, line_values);
body = find_string_attr_range(sa, "body");
@ -132,7 +134,7 @@ public:
logfile * lf = lss.find(cl);
logfile::iterator lf_iter = lf->begin() + cl;
if (lf_iter->get_level() & logline::LEVEL_CONTINUED) {
if (lf_iter->is_continued()) {
return false;
}
@ -174,6 +176,8 @@ public:
const std::string &line,
std::vector<logline_value> &values)
{
int next_column = this->ldt_parent_column_count;
this->ldt_format_impl->extract(lf, line, values);
for (data_parser::element_list_t::iterator pair_iter =
this->ldt_pairs.begin();
@ -196,6 +200,7 @@ public:
values.push_back(logline_value("", tmp));
break;
}
values.back().lv_column = next_column++;
}
};
@ -206,5 +211,6 @@ private:
std::string ldt_current_line;
data_parser::element_list_t ldt_pairs;
log_vtab_impl *ldt_format_impl;
int ldt_parent_column_count;
};
#endif

539
src/log_format.cc
View File

@ -32,7 +32,10 @@
#include <stdio.h>
#include <stdarg.h>
#include <string.h>
#include <strings.h>
#include "yajlpp.hh"
#include "sql_util.hh"
#include "log_format.hh"
#include "log_vtab_impl.hh"
@ -64,46 +67,51 @@ const char *logline::level_names[LEVEL__MAX] = {
"fatal",
};
static int strcasestr_i(const char *s1, const char *s2)
static int strncasestr_i(const char *s1, const char *s2, size_t len)
{
return strcasestr(s1, s2) == NULL;
}
logline::level_t logline::string2level(const char *levelstr, bool exact)
logline::level_t logline::string2level(const char *levelstr, size_t len, bool exact)
{
logline::level_t retval = logline::LEVEL_UNKNOWN;
int (*cmpfunc)(const char *, const char *);
int (*cmpfunc)(const char *, const char *, size_t);
assert(len == (size_t)-1 || (len != (size_t)-1 && exact));
if (len == (size_t)-1)
len = strlen(levelstr);
if (exact) {
cmpfunc = strcasecmp;
cmpfunc = strncasecmp;
}
else{
cmpfunc = strcasestr_i;
cmpfunc = strncasestr_i;
}
if (cmpfunc(levelstr, "TRACE") == 0) {
if (cmpfunc(levelstr, "TRACE", len) == 0) {
retval = logline::LEVEL_TRACE;
}
else if (cmpfunc(levelstr, "VERBOSE") == 0) {
else if (cmpfunc(levelstr, "VERBOSE", len) == 0) {
retval = logline::LEVEL_DEBUG;
}
else if (cmpfunc(levelstr, "DEBUG") == 0) {
else if (cmpfunc(levelstr, "DEBUG", len) == 0) {
retval = logline::LEVEL_DEBUG;
}
else if (cmpfunc(levelstr, "INFO") == 0) {
else if (cmpfunc(levelstr, "INFO", len) == 0) {
retval = logline::LEVEL_INFO;
}
else if (cmpfunc(levelstr, "WARNING") == 0) {
else if (cmpfunc(levelstr, "WARNING", len) == 0) {
retval = logline::LEVEL_WARNING;
}
else if (cmpfunc(levelstr, "ERROR") == 0) {
else if (cmpfunc(levelstr, "ERROR", len) == 0) {
retval = logline::LEVEL_ERROR;
}
else if (cmpfunc(levelstr, "CRITICAL") == 0) {
else if (cmpfunc(levelstr, "CRITICAL", len) == 0) {
retval = logline::LEVEL_CRITICAL;
}
else if (cmpfunc(levelstr, "FATAL") == 0) {
else if (cmpfunc(levelstr, "FATAL", len) == 0) {
retval = logline::LEVEL_FATAL;
}
@ -121,6 +129,9 @@ logline_value::kind_t logline_value::string2kind(const char *kindstr)
else if (strcmp(kindstr, "float") == 0) {
return VALUE_FLOAT;
}
else if (strcmp(kindstr, "boolean") == 0) {
return VALUE_BOOLEAN;
}
return VALUE_UNKNOWN;
}
@ -197,11 +208,242 @@ const char *log_format::log_scanf(const char *line,
return retval;
}
/*
* XXX This needs some cleanup.
*/
struct json_log_userdata {
json_log_userdata() : jlu_sub_line_count(1) { };
external_log_format *jlu_format;
const logline *jlu_line;
logline *jlu_base_line;
int jlu_sub_line_count;
yajl_handle jlu_handle;
const char *jlu_line_value;
size_t jlu_sub_start;
};
struct json_field_cmp {
json_field_cmp(external_log_format::json_log_field type,
const std::string &name)
: jfc_type(type), jfc_field_name(name) {
};
bool operator()(const external_log_format::json_format_element &jfe) const {
return (this->jfc_type == jfe.jfe_type &&
this->jfc_field_name == jfe.jfe_value);
};
external_log_format::json_log_field jfc_type;
const std::string &jfc_field_name;
};
static int read_json_field(yajlpp_parse_context *ypc, const unsigned char *str, size_t len);
static int read_json_null(yajlpp_parse_context *ypc)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
vector<external_log_format::json_format_element> &line_format =
jlu->jlu_format->jlf_line_format;
string field_name = ypc->get_path_fragment(0);
if (find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
}
return 1;
}
static int read_json_bool(yajlpp_parse_context *ypc, int val)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
vector<external_log_format::json_format_element> &line_format =
jlu->jlu_format->jlf_line_format;
string field_name = ypc->get_path_fragment(0);
if (find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
}
return 1;
}
static int read_json_int(yajlpp_parse_context *ypc, long long val)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
vector<external_log_format::json_format_element> &line_format =
jlu->jlu_format->jlf_line_format;
string field_name = ypc->get_path_fragment(0);
if (find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
}
return 1;
}
static int read_json_double(yajlpp_parse_context *ypc, double val)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
vector<external_log_format::json_format_element> &line_format =
jlu->jlu_format->jlf_line_format;
string field_name = ypc->get_path_fragment(0);
if (find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
}
return 1;
}
static int json_array_start(void *ctx)
{
yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx;
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
vector<external_log_format::json_format_element> &line_format =
jlu->jlu_format->jlf_line_format;
if (ypc->ypc_path_index_stack.size() == 2) {
string field_name = ypc->get_path_fragment(0);
if (find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
}
jlu->jlu_sub_start = yajl_get_bytes_consumed(jlu->jlu_handle) - 1;
}
return 1;
}
static int json_array_end(void *ctx)
{
yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx;
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
if (ypc->ypc_path_index_stack.size() == 1) {
string field_name = ypc->get_path_fragment(0);
size_t sub_end = yajl_get_bytes_consumed(jlu->jlu_handle);
string str = string(&jlu->jlu_line_value[jlu->jlu_sub_start],
sub_end - jlu->jlu_sub_start);
jlu->jlu_format->jlf_line_values.push_back(
logline_value(field_name, str));
}
return 1;
}
static struct json_path_handler json_log_handlers[] = {
json_path_handler("^/\\w+$").
add_cb(read_json_null).
add_cb(read_json_bool).
add_cb(read_json_int).
add_cb(read_json_double).
add_cb(read_json_field),
json_path_handler()
};
static int rewrite_json_field(yajlpp_parse_context *ypc, const unsigned char *str, size_t len);
static int rewrite_json_null(yajlpp_parse_context *ypc)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
string field_name = ypc->get_path_fragment(0);
jlu->jlu_format->jlf_line_values.push_back(logline_value(field_name));
return 1;
}
static int rewrite_json_bool(yajlpp_parse_context *ypc, int val)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
string field_name = ypc->get_path_fragment(0);
jlu->jlu_format->jlf_line_values.push_back(logline_value(field_name, (bool)val));
return 1;
}
static int rewrite_json_int(yajlpp_parse_context *ypc, long long val)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
string field_name = ypc->get_path_fragment(0);
jlu->jlu_format->jlf_line_values.push_back(logline_value(field_name, val));
return 1;
}
static int rewrite_json_double(yajlpp_parse_context *ypc, double val)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
string field_name = ypc->get_path_fragment(0);
jlu->jlu_format->jlf_line_values.push_back(logline_value(field_name, val));
return 1;
}
static struct json_path_handler json_log_rewrite_handlers[] = {
json_path_handler("^/\\w+$").
add_cb(rewrite_json_null).
add_cb(rewrite_json_bool).
add_cb(rewrite_json_int).
add_cb(rewrite_json_double).
add_cb(rewrite_json_field),
json_path_handler()
};
bool external_log_format::scan(std::vector<logline> &dst,
off_t offset,
char *prefix,
int len)
{
if (this->jlf_json) {
auto_mem<yajl_handle_t> handle(yajl_free);
yajlpp_parse_context ypc("", json_log_handlers);
logline ll(offset, 0, 0, logline::LEVEL_INFO);
json_log_userdata jlu;
bool retval = false;
ypc.ypc_userdata = &jlu;
ypc.ypc_ignore_unused = true;
ypc.ypc_alt_callbacks.yajl_start_array = json_array_start;
ypc.ypc_alt_callbacks.yajl_start_map = json_array_start;
handle = yajl_alloc(&ypc.ypc_callbacks, NULL, &ypc); // XXX
jlu.jlu_format = this;
jlu.jlu_base_line = &ll;
jlu.jlu_line_value = prefix;
jlu.jlu_handle = handle;
if (yajl_parse(handle.in(), (const unsigned char *)prefix, len) == yajl_status_ok &&
yajl_complete_parse(handle.in()) == yajl_status_ok) {
for (int lpc = 0; lpc < jlu.jlu_sub_line_count; lpc++) {
ll.set_sub_offset(lpc);
if (lpc > 0) {
ll.set_level((logline::level_t) (ll.get_level() |
logline::LEVEL_CONTINUED));
}
dst.push_back(ll);
}
retval = true;
}
return retval;
}
pcre_input pi(prefix, 0, len);
pcre_context_static<128> pc;
bool retval = false;
@ -212,7 +454,7 @@ bool external_log_format::scan(std::vector<logline> &dst,
continue;
}
pcre_context::capture_t *ts = pc["timestamp"];
pcre_context::capture_t *ts = pc[this->lf_timestamp_field];
pcre_context::capture_t *level_cap = pc[this->elf_level_field];
const char *ts_str = pi.get_substr_start(ts);
const char *last;
@ -276,13 +518,19 @@ void external_log_format::annotate(const std::string &line,
struct line_range lr;
pcre_context::capture_t *cap;
if (this->jlf_json) {
values = this->jlf_line_values;
sa = this->jlf_line_attrs;
return;
}
pattern &pat = *this->elf_pattern_order[this->lf_fmt_lock];
if (!pat.p_pcre->match(pc, pi)) {
return;
}
cap = pc["timestamp"];
cap = pc[this->lf_timestamp_field];
lr.lr_start = cap->c_begin;
lr.lr_end = cap->c_end;
sa[lr].insert(make_string_attr("timestamp", 0));
@ -324,7 +572,8 @@ void external_log_format::annotate(const std::string &line,
vd.vd_kind,
pi.get_substr(pc[vd.vd_index]),
vd.vd_identifier,
scaling));
scaling,
vd.vd_column));
if (pc[vd.vd_index]->c_begin != -1 && vd.vd_identifier) {
lr.lr_start = pc[vd.vd_index]->c_begin;
@ -334,8 +583,210 @@ void external_log_format::annotate(const std::string &line,
}
}
static int read_json_field(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
vector<external_log_format::json_format_element> &line_format =
jlu->jlu_format->jlf_line_format;
string field_name = ypc->get_path_fragment(0);
struct tm tm_out;
struct timeval tv_out;
if (field_name == jlu->jlu_format->lf_timestamp_field) {
jlu->jlu_format->lf_date_time.scan((const char *)str, NULL, &tm_out, tv_out);
jlu->jlu_base_line->set_time(tv_out);
}
else if (field_name == jlu->jlu_format->elf_level_field) {
jlu->jlu_base_line->set_level(logline::string2level((const char *)str, len, true));
}
else {
if (find_if(line_format.begin(), line_format.end(),
json_field_cmp(external_log_format::JLF_VARIABLE,
field_name)) == line_format.end()) {
jlu->jlu_sub_line_count += 1;
}
for (size_t lpc = 0; lpc < len; lpc++) {
if (str[lpc] == '\n') {
jlu->jlu_sub_line_count += 1;
}
}
}
return 1;
}
static int rewrite_json_field(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
{
json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata;
string field_name = ypc->get_path_fragment(0);
string val = string((const char *)str, len);
if (field_name == jlu->jlu_format->lf_timestamp_field) {
char time_buf[64];
sql_strftime(time_buf, sizeof(time_buf), jlu->jlu_line->get_timeval());
val = time_buf;
}
if (field_name == jlu->jlu_format->elf_body_field) {
jlu->jlu_format->jlf_line_values.push_back(logline_value("body", val));
}
jlu->jlu_format->jlf_line_values.push_back(logline_value(field_name, val));
return 1;
}
void external_log_format::get_subline(const logline &ll,
const char *line, size_t len,
ostringstream &stream_out)
{
if (!this->jlf_json) {
stream_out.write(line, len);
return;
}
if (this->jlf_cached_offset != ll.get_offset()) {
auto_mem<yajl_handle_t> handle(yajl_free);
yajlpp_parse_context ypc("", json_log_rewrite_handlers);
view_colors &vc = view_colors::singleton();
json_log_userdata jlu;
jlu.jlu_format = this;
jlu.jlu_line = &ll;
ypc.ypc_userdata = &jlu;
ypc.ypc_ignore_unused = true;
this->jlf_cached_line.clear();
this->jlf_line_values.clear();
this->jlf_line_offsets.clear();
this->jlf_line_attrs.clear();
ypc.ypc_alt_callbacks.yajl_start_array = json_array_start;
ypc.ypc_alt_callbacks.yajl_end_array = json_array_end;
ypc.ypc_alt_callbacks.yajl_start_map = json_array_start;
ypc.ypc_alt_callbacks.yajl_end_map = json_array_end;
handle = yajl_alloc(&ypc.ypc_callbacks, NULL, &ypc);
jlu.jlu_handle = handle.in();
jlu.jlu_line_value = line;
if (yajl_parse(handle.in(), (const unsigned char *)line, len) == yajl_status_ok &&
yajl_complete_parse(handle.in()) == yajl_status_ok) {
std::vector<logline_value>::iterator lv_iter;
std::vector<json_format_element>::iterator iter;
bool used_values[this->jlf_line_values.size()];
struct line_range lr;
ostringstream lines;
memset(used_values, 0, sizeof(used_values));
for (lv_iter = this->jlf_line_values.begin();
lv_iter != this->jlf_line_values.end();
++lv_iter) {
map<string, external_log_format::value_def>::iterator vd_iter;
vd_iter = this->elf_value_defs.find(lv_iter->lv_name);
if (vd_iter != this->elf_value_defs.end()) {
lv_iter->lv_identifier = vd_iter->second.vd_identifier;
lv_iter->lv_column = vd_iter->second.vd_column;
}
}
for (iter = this->jlf_line_format.begin();
iter != this->jlf_line_format.end();
++iter) {
switch (iter->jfe_type) {
case JLF_CONSTANT:
lines << iter->jfe_default_value;
break;
case JLF_VARIABLE:
lv_iter = find_if(this->jlf_line_values.begin(),
this->jlf_line_values.end(),
logline_value_cmp(&iter->jfe_value));
if (lv_iter != this->jlf_line_values.end()) {
string str = lv_iter->to_string();
size_t nl_pos = str.find('\n');
lr.lr_start = lines.tellp();
lines << str;
if (nl_pos == string::npos)
lr.lr_end = lines.tellp();
else
lr.lr_end = lr.lr_start + nl_pos;
if (lv_iter->lv_name == this->lf_timestamp_field)
this->jlf_line_attrs[lr].insert(make_string_attr("timestamp", 0));
else if (lv_iter->lv_name == this->elf_body_field)
this->jlf_line_attrs[lr].insert(make_string_attr("body", 0));
else if (lv_iter->lv_identifier) {
fprintf(stderr, "ident!! %s\n", lv_iter->lv_name.c_str());
this->jlf_line_attrs[lr].insert(make_string_attr("style", vc.attrs_for_ident(str.c_str(), lr.length())));
}
used_values[distance(this->jlf_line_values.begin(),
lv_iter)] = true;
}
else {
lines << iter->jfe_default_value;
}
break;
}
}
lines << endl;
for (size_t lpc = 0; lpc < this->jlf_line_values.size(); lpc++) {
const logline_value &lv = this->jlf_line_values[lpc];
if (used_values[lpc] ||
lv.lv_name == "body" ||
lv.lv_name == this->elf_level_field)
continue;
const std::string str = lv.to_string();
size_t curr_pos = 0, nl_pos, line_len = -1;
do {
nl_pos = str.find('\n', curr_pos);
if (nl_pos != std::string::npos) {
line_len = nl_pos - curr_pos;
}
lines << " "
<< lv.lv_name
<< ": "
<< str.substr(curr_pos, line_len)
<< endl;
curr_pos = nl_pos + 1;
line_len = -1;
} while (nl_pos != std::string::npos &&
nl_pos < str.size());
}
this->jlf_cached_line = lines.str();
this->jlf_line_offsets.push_back(0);
for (size_t lpc = 0; lpc < this->jlf_cached_line.size(); lpc++) {
if (this->jlf_cached_line[lpc] == '\n') {
this->jlf_line_offsets.push_back(lpc);
}
}
this->jlf_line_offsets.push_back(this->jlf_cached_line.size());
}
this->jlf_cached_offset = ll.get_offset();
}
off_t this_off, next_off;
this_off = this->jlf_line_offsets[ll.get_sub_offset()];
if (this->jlf_cached_line[this_off] == '\n')
this_off += 1;
next_off = this->jlf_line_offsets[ll.get_sub_offset() + 1];
stream_out.write(this->jlf_cached_line.c_str() + this_off,
next_off - this_off);
}
void external_log_format::build(std::vector<std::string> &errors)
{
try {
this->elf_filename_pcre = new pcrepp(this->elf_file_pattern.c_str());
}
catch (const pcrepp::error &e) {
errors.push_back("error:" +
this->elf_name + ".file-pattern:" +
e.what());
}
for (std::map<string, pattern>::iterator iter = this->elf_patterns.begin();
iter != this->elf_patterns.end();
++iter) {
@ -356,9 +807,14 @@ void external_log_format::build(std::vector<std::string> &errors)
value_iter = this->elf_value_defs.find(std::string(name_iter->pnc_name));
if (value_iter != this->elf_value_defs.end()) {
value_iter->second.vd_index = name_iter->index();
value_iter->second.vd_unit_field_index = iter->second.p_pcre->name_index(value_iter->second.vd_unit_field.c_str());
iter->second.p_value_by_index.push_back(value_iter->second);
value_def &vd = value_iter->second;
vd.vd_index = name_iter->index();
vd.vd_unit_field_index = iter->second.p_pcre->name_index(vd.vd_unit_field.c_str());
if (vd.vd_column == -1) {
vd.vd_column = this->elf_column_count++;
}
iter->second.p_value_by_index.push_back(vd);
}
}
@ -368,10 +824,19 @@ void external_log_format::build(std::vector<std::string> &errors)
this->elf_pattern_order.push_back(&iter->second);
}
if (this->elf_patterns.empty()) {
errors.push_back("error:" +
this->elf_name +
": no regexes specified for format");
if (this->jlf_json) {
if (!this->elf_patterns.empty()) {
errors.push_back("error:" +
this->elf_name +
": JSON logs cannot have regexes");
}
}
else {
if (this->elf_patterns.empty()) {
errors.push_back("error:" +
this->elf_name +
": no regexes specified for format");
}
}
for (std::map<logline::level_t, level_pattern>::iterator iter = this->elf_level_patterns.begin();
@ -386,7 +851,15 @@ void external_log_format::build(std::vector<std::string> &errors)
}
}
if (this->elf_samples.empty()) {
for (std::map<string, value_def>::iterator iter = this->elf_value_defs.begin();
iter != this->elf_value_defs.end();
++iter) {
if (iter->second.vd_column == -1) {
iter->second.vd_column = this->elf_column_count++;
}
}
if (!this->jlf_json && this->elf_samples.empty()) {
errors.push_back("error:" +
this->elf_name +
":no sample logs provided, all formats must have samples");
@ -459,21 +932,25 @@ public:
};
void get_columns(vector<vtab_column> &cols) {
std::vector<external_log_format::value_def>::const_iterator iter;
std::map<string, external_log_format::value_def>::const_iterator iter;
const external_log_format &elf = this->elt_format;
for (iter = elf.elf_pattern_order[0]->p_value_by_index.begin();
iter != elf.elf_pattern_order[0]->p_value_by_index.end();
cols.resize(elf.elf_value_defs.size());
for (iter = elf.elf_value_defs.begin();
iter != elf.elf_value_defs.end();
++iter) {
const external_log_format::value_def &vd = iter->second;
int type;
switch (iter->vd_kind) {
switch (vd.vd_kind) {
case logline_value::VALUE_NULL:
case logline_value::VALUE_TEXT:
type = SQLITE3_TEXT;
break;
case logline_value::VALUE_FLOAT:
type = SQLITE_FLOAT;
break;
case logline_value::VALUE_BOOLEAN:
case logline_value::VALUE_INTEGER:
type = SQLITE_INTEGER;
break;
@ -481,9 +958,9 @@ public:
assert(0);
break;
}
cols.push_back(vtab_column(iter->vd_name.c_str(),
type,
iter->vd_collate.c_str()));
cols[vd.vd_column].vc_name = vd.vd_name.c_str();
cols[vd.vd_column].vc_type = type;
cols[vd.vd_column].vc_collator = vd.vd_collate.c_str();
}
};

160
src/log_format.hh
View File

@ -43,6 +43,7 @@
#include <string>
#include <vector>
#include <memory>
#include <sstream>
#include "pcrepp.hh"
#include "lnav_util.hh"
@ -78,6 +79,10 @@ public:
virtual std::string to_command(void) = 0;
bool operator==(const std::string &rhs) {
return this->lf_id == rhs;
};
protected:
bool lf_enabled;
type_t lf_type;
@ -116,7 +121,7 @@ public:
static const char *level_names[LEVEL__MAX];
static level_t string2level(const char *levelstr, bool exact = false);
static level_t string2level(const char *levelstr, size_t len = -1, bool exact = false);
/**
* Construct a logline object with the given values.
@ -129,13 +134,13 @@ public:
logline(off_t off,
time_t t,
uint16_t millis,
level_t l,
uint8_t m = 0)
level_t l)
: ll_offset(off),
ll_time(t),
ll_millis(millis),
ll_level(l),
ll_filter_state(logfile_filter::MAYBE)
ll_filter_state(logfile_filter::MAYBE),
ll_sub_offset(0)
{
memset(this->ll_schema, 0, sizeof(this->ll_schema));
};
@ -146,7 +151,8 @@ public:
uint8_t m = 0)
: ll_offset(off),
ll_level(l),
ll_filter_state(logfile_filter::MAYBE)
ll_filter_state(logfile_filter::MAYBE),
ll_sub_offset(0)
{
this->set_time(tv);
memset(this->ll_schema, 0, sizeof(this->ll_schema));
@ -155,6 +161,10 @@ public:
/** @return The offset of the line in the file. */
off_t get_offset() const { return this->ll_offset; };
uint16_t get_sub_offset() const { return this->ll_sub_offset; };
void set_sub_offset(uint16_t suboff) { this->ll_sub_offset = suboff; };
/** @return The timestamp for the line. */
time_t get_time() const { return this->ll_time; };
@ -266,6 +276,7 @@ private:
uint16_t ll_millis;
uint8_t ll_level;
uint8_t ll_filter_state;
uint16_t ll_sub_offset;
char ll_schema[4];
};
@ -300,28 +311,42 @@ class logline_value {
public:
enum kind_t {
VALUE_UNKNOWN = -1,
VALUE_NULL,
VALUE_TEXT,
VALUE_INTEGER,
VALUE_FLOAT,
VALUE_BOOLEAN,
};
static kind_t string2kind(const char *kindstr);
logline_value(std::string name)
: lv_name(name), lv_kind(VALUE_NULL), lv_identifier(), lv_column(-1) { };
logline_value(std::string name, bool b)
: lv_name(name),
lv_kind(VALUE_BOOLEAN),
lv_number((int64_t)(b ? 1 : 0)),
lv_identifier(),
lv_column(-1) { };
logline_value(std::string name, int64_t i)
: lv_name(name), lv_kind(VALUE_INTEGER), lv_number(i) { };
: lv_name(name), lv_kind(VALUE_INTEGER), lv_number(i), lv_identifier(), lv_column(-1) { };
logline_value(std::string name, double i)
: lv_name(name), lv_kind(VALUE_FLOAT), lv_number(i) { };
: lv_name(name), lv_kind(VALUE_FLOAT), lv_number(i), lv_identifier(), lv_column(-1) { };
logline_value(std::string name, std::string s)
: lv_name(name), lv_kind(VALUE_TEXT), lv_string(s) { };
: lv_name(name), lv_kind(VALUE_TEXT), lv_string(s), lv_identifier(), lv_column(-1) { };
logline_value(std::string name, kind_t kind, std::string s,
bool ident=false, const scaling_factor *scaling=NULL)
: lv_name(name), lv_kind(kind), lv_identifier(ident)
bool ident=false, const scaling_factor *scaling=NULL,
int col=-1)
: lv_name(name), lv_kind(kind), lv_identifier(ident), lv_column(col)
{
switch (kind) {
case VALUE_TEXT:
this->lv_string = s;
break;
case VALUE_NULL:
break;
case VALUE_INTEGER:
sscanf(s.c_str(), "%" PRId64 "", &this->lv_number.i);
if (scaling != NULL) {
@ -336,17 +361,29 @@ public:
}
break;
case VALUE_BOOLEAN:
if (s == "true" || s == "yes") {
this->lv_number.i = 1;
}
else {
this->lv_number.i = 0;
}
break;
case VALUE_UNKNOWN:
assert(0);
break;
}
};
const std::string to_string()
const std::string to_string() const
{
char buffer[128];
switch (this->lv_kind) {
case VALUE_NULL:
return "null";
case VALUE_TEXT:
return this->lv_string;
@ -357,9 +394,18 @@ public:
case VALUE_FLOAT:
snprintf(buffer, sizeof(buffer), "%lf", this->lv_number.d);
break;
case VALUE_BOOLEAN:
if (this->lv_number.i) {
return "true";
}
else {
return "false";
}
break;
case VALUE_UNKNOWN:
assert(0);
break;
assert(0);
break;
}
return std::string(buffer);
@ -377,6 +423,28 @@ public:
} lv_number;
std::string lv_string;
bool lv_identifier;
int lv_column;
};
struct logline_value_cmp {
logline_value_cmp(const std::string *name = NULL, int col = -1)
: lvc_name(name), lvc_column(col) {
};
bool operator()(const logline_value &lv) {
bool retval = true;
if (this->lvc_name != NULL)
retval = retval && ((*this->lvc_name) == lv.lv_name);
if (this->lvc_column != -1)
retval = retval && (this->lvc_column == lv.lv_column);
return retval;
};
const std::string *lvc_name;
int lvc_column;
};
class log_vtab_impl;
@ -406,7 +474,7 @@ public:
};
};
log_format() : lf_fmt_lock(-1) {
log_format() : lf_fmt_lock(-1), lf_timestamp_field("timestamp") {
};
virtual ~log_format() { };
@ -423,6 +491,8 @@ public:
*/
virtual std::string get_name(void) const = 0;
virtual bool match_name(const std::string &filename) { return true; };
/**
* Scan a log line to see if it matches this log format.
*
@ -458,8 +528,15 @@ public:
return NULL;
};
virtual void get_subline(const logline &ll,
const char *line, size_t len,
std::ostringstream &stream_out) {
stream_out.write(line, len);
};
date_time_scanner lf_date_time;
int lf_fmt_lock;
std::string lf_timestamp_field;
protected:
static std::vector<log_format *> lf_root_formats;
@ -487,7 +564,8 @@ public:
vd_kind(logline_value::VALUE_UNKNOWN),
vd_identifier(false),
vd_foreign_key(false),
vd_unit_field_index(-1) {
vd_unit_field_index(-1),
vd_column(-1) {
};
@ -500,6 +578,7 @@ public:
std::string vd_unit_field;
int vd_unit_field_index;
std::map<std::string, scaling_factor> vd_unit_scaling;
int vd_column;
bool operator<(const value_def &rhs) const {
return this->vd_index < rhs.vd_index;
@ -522,12 +601,26 @@ public:
pcrepp *lp_pcre;
};
external_log_format(const std::string &name) : elf_name(name) { };
external_log_format(const std::string &name)
: elf_file_pattern(".*"),
elf_column_count(0),
elf_body_field("body"),
jlf_cached_offset(-1),
elf_name(name) {
this->jlf_line_offsets.reserve(128);
};
std::string get_name(void) const {
return this->elf_name;
};
bool match_name(const std::string &filename) {
pcre_context_static<10> pc;
pcre_input pi(filename);
return this->elf_filename_pcre->match(pc, pi);
};
bool scan(std::vector<logline> &dst,
off_t offset,
char *prefix,
@ -543,21 +636,50 @@ public:
std::auto_ptr<log_format> retval((log_format *)
new external_log_format(*this));
retval->lf_fmt_lock = this->lf_fmt_lock;
retval->lf_date_time = this->lf_date_time;
return retval;
};
void get_subline(const logline &ll,
const char *line, size_t len,
std::ostringstream &stream_out);
log_vtab_impl *get_vtab_impl(void) const;
std::string elf_file_pattern;
pcrepp *elf_filename_pcre;
std::map<std::string, pattern> elf_patterns;
std::vector<pattern *> elf_pattern_order;
std::vector<sample> elf_samples;
std::map<std::string, value_def> elf_value_defs;
int elf_column_count;
std::string elf_level_field;
std::string elf_body_field;
std::map<logline::level_t, level_pattern> elf_level_patterns;
enum json_log_field {
JLF_CONSTANT,
JLF_VARIABLE,
};
struct json_format_element {
json_format_element()
: jfe_type(JLF_CONSTANT), jfe_default_value("-"), jfe_min_width(0)
{ };
json_log_field jfe_type;
std::string jfe_value;
std::string jfe_default_value;
int jfe_min_width;
};
bool jlf_json;
std::vector<json_format_element> jlf_line_format;
std::vector<logline_value> jlf_line_values;
off_t jlf_cached_offset;
std::vector<off_t> jlf_line_offsets;
std::string jlf_cached_line;
string_attrs_t jlf_line_attrs;
private:
const std::string elf_name;

4
src/log_format_impls.cc
View File

@ -192,7 +192,7 @@ class generic_log_format : public log_format {
}
}
if (logline::string2level(level, true) == logline::LEVEL_UNKNOWN) {
if (logline::string2level(level, -1, true) == logline::LEVEL_UNKNOWN) {
prefix_len = lr.lr_end;
}
@ -308,7 +308,7 @@ class strace_log_format : public log_format {
{ "result", logline_value::VALUE_TEXT },
{ "duration", logline_value::VALUE_FLOAT },
{ NULL },
{ NULL, logline_value::VALUE_UNKNOWN },
};
pcre_context::iterator iter;

86
src/log_format_loader.cc
View File

@ -78,6 +78,8 @@ static int read_format_bool(yajlpp_parse_context *ypc, int val)
if (field_name == "local-time")
elf->lf_date_time.dts_local_time = val;
else if (field_name == "json")
elf->jlf_json = val;
return 1;
}
@ -88,8 +90,14 @@ static int read_format_field(yajlpp_parse_context *ypc, const unsigned char *str
string value = string((const char *)str, len);
string field_name = ypc->get_path_fragment(1);
if (field_name == "level-field")
if (field_name == "file-pattern")
elf->elf_file_pattern = value;
else if (field_name == "level-field")
elf->elf_level_field = value;
else if (field_name == "timestamp-field")
elf->lf_timestamp_field = value;
else if (field_name == "body-field")
elf->elf_body_field = value;
return 1;
}
@ -202,18 +210,76 @@ static int read_sample_line(yajlpp_parse_context *ypc, const unsigned char *str,
return 1;
}
static external_log_format::json_format_element &
ensure_json_format_element(external_log_format *elf, int index)
{
elf->jlf_line_format.resize(index + 1);
return elf->jlf_line_format[index];
}
static int read_json_constant(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
{
external_log_format *elf = ensure_format(ypc->get_path_fragment(0));
string val = string((const char *)str, len);
ypc->ypc_array_index.back() += 1;
int index = ypc->ypc_array_index.back();
external_log_format::json_format_element &jfe = ensure_json_format_element(elf, index);
jfe.jfe_type = external_log_format::JLF_CONSTANT;
jfe.jfe_default_value = val;
return 1;
}
static int read_json_variable(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
{
external_log_format *elf = ensure_format(ypc->get_path_fragment(0));
string val = string((const char *)str, len);
int index = ypc->ypc_array_index.back();
external_log_format::json_format_element &jfe = ensure_json_format_element(elf, index);
string field_name = ypc->get_path_fragment(3);
jfe.jfe_type = external_log_format::JLF_VARIABLE;
if (field_name == "field")
jfe.jfe_value = val;
else if (field_name == "default-value")
jfe.jfe_default_value = val;
return 1;
}
static int read_json_variable_num(yajlpp_parse_context *ypc, long long val)
{
external_log_format *elf = ensure_format(ypc->get_path_fragment(0));
int index = ypc->ypc_array_index.back();
external_log_format::json_format_element &jfe = ensure_json_format_element(elf, index);
string field_name = ypc->get_path_fragment(2);
jfe.jfe_type = external_log_format::JLF_VARIABLE;
jfe.jfe_min_width = val;
return 1;
}
static struct json_path_handler format_handlers[] = {
json_path_handler("/\\w+/regex/[^/]+/pattern", read_format_regex),
json_path_handler("/\\w+/local-time", read_format_bool),
json_path_handler("/\\w+/(level-field|url|title|description)", read_format_field),
json_path_handler("/\\w+/level/"
"(trace|debug|info|warning|error|critical|fatal)",
json_path_handler("^/\\w+/regex/[^/]+/pattern$", read_format_regex),
json_path_handler("^/\\w+/(json|local-time)$", read_format_bool),
json_path_handler("^/\\w+/(file-pattern|level-field|timestamp-field|body-field|url|title|description)$",
read_format_field),
json_path_handler("^/\\w+/level/"
"(trace|debug|info|warning|error|critical|fatal)$",
read_levels),
json_path_handler("/\\w+/value/\\w+/(kind|collate|unit/field)", read_value_def),
json_path_handler("/\\w+/value/\\w+/(identifier|foreign-key)", read_value_bool),
json_path_handler("/\\w+/value/\\w+/unit/scaling-factor/.*",
json_path_handler("^/\\w+/value/\\w+/(kind|collate|unit/field)$", read_value_def),
json_path_handler("^/\\w+/value/\\w+/(identifier|foreign-key)$", read_value_bool),
json_path_handler("^/\\w+/value/\\w+/unit/scaling-factor/.*$",
read_scaling),
json_path_handler("/\\w+/sample#/line", read_sample_line),
json_path_handler("^/\\w+/sample#/line$", read_sample_line),
json_path_handler("^/\\w+/line-format#/(field|default-value)$", read_json_variable),
json_path_handler("^/\\w+/line-format#/min-width$", read_json_variable_num),
json_path_handler("^/\\w+/line-format#$", read_json_constant),
json_path_handler()
};

22
src/log_vtab_impl.cc
View File

@ -75,6 +75,8 @@ static string declare_table_statement(log_vtab_impl *vi)
for (iter = cols.begin(); iter != cols.end(); iter++) {
auto_mem<char, sqlite3_free> coldecl;
assert(iter->vc_name != NULL);
coldecl = sqlite3_mprintf(" %Q %s collate %Q,\n",
iter->vc_name,
type_to_string(iter->vc_type),
@ -361,24 +363,30 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
}
size_t sub_col = col - VT_COL_MAX;
std::vector<logline_value>::iterator lv_iter;
if (sub_col < vc->line_values.size()) {
logline_value &lv = vc->line_values[sub_col];
lv_iter = find_if(vc->line_values.begin(), vc->line_values.end(),
logline_value_cmp(NULL, sub_col));
switch (lv.lv_kind) {
if (lv_iter != vc->line_values.end()) {
switch (lv_iter->lv_kind) {
case logline_value::VALUE_NULL:
sqlite3_result_null(ctx);
break;
case logline_value::VALUE_TEXT:
sqlite3_result_text(ctx,
lv.lv_string.c_str(),
lv.lv_string.length(),
lv_iter->lv_string.c_str(),
lv_iter->lv_string.length(),
SQLITE_TRANSIENT);
break;
case logline_value::VALUE_BOOLEAN:
case logline_value::VALUE_INTEGER:
sqlite3_result_int64(ctx, lv.lv_number.i);
sqlite3_result_int64(ctx, lv_iter->lv_number.i);
break;
case logline_value::VALUE_FLOAT:
sqlite3_result_double(ctx, lv.lv_number.d);
sqlite3_result_double(ctx, lv_iter->lv_number.d);
break;
case logline_value::VALUE_UNKNOWN:

4
src/log_vtab_impl.hh
View File

@ -58,7 +58,9 @@ struct log_cursor {
class log_vtab_impl {
public:
struct vtab_column {
vtab_column(const char *name, int type, const char *collator = NULL)
vtab_column(const char *name = NULL,
int type = SQLITE3_TEXT,
const char *collator = NULL)
: vc_name(name), vc_type(type), vc_collator(collator) { };
const char *vc_name;

29
src/logfile.cc
View File

@ -50,7 +50,7 @@
using namespace std;
static const int MAX_UNRECOGNIZED_LINES = 1000;
static const size_t MAX_UNRECOGNIZED_LINES = 1000;
logfile::logfile(string filename, auto_fd fd)
throw (error)
@ -141,6 +141,9 @@ void logfile::process_prefix(off_t offset, char *prefix, int len)
for (iter = root_formats.begin();
iter != root_formats.end() && !found;
++iter) {
if (!(*iter)->match_name(this->lf_filename))
continue;
(*iter)->clear();
(*iter)->lf_date_time.set_base_time(this->lf_line_buffer.get_file_time());
if ((*iter)->lf_date_time.dts_base_time == 0) {
@ -236,6 +239,9 @@ throw (line_buffer::error)
* Drop the last line we read since it might have been a partial
* read.
*/
while (this->lf_index.back().get_sub_offset() != 0) {
this->lf_index.pop_back();
}
this->lf_index.pop_back();
}
else {
@ -342,7 +348,15 @@ void logfile::read_line(logfile::iterator ll, string &line_out)
line_out.clear();
if ((line = this->lf_line_buffer.read_line(off, len)) != NULL) {
line_out.append(line, len);
ostringstream stream;
if (this->lf_format.get() != NULL) {
this->lf_format->get_subline(*ll, line, len, stream);
line_out = stream.str();
}
else {
line_out.append(line, len);
}
}
else {
/* XXX */
@ -357,18 +371,19 @@ void logfile::read_full_message(logfile::iterator ll,
string &msg_out,
int max_lines)
{
msg_out.clear();
ostringstream stream;
do {
try {
off_t off = ll->get_offset();
const char *line;
size_t len;
if (!msg_out.empty()) {
msg_out.append(1, '\n');
if (stream.tellp() > 0) {
stream.write("\n", 1);
}
if ((line = this->lf_line_buffer.read_line(off, len)) != NULL) {
msg_out.append(line, len);
this->lf_format->get_subline(*ll, line, len, stream);
}
else {
/* XXX */
@ -381,4 +396,6 @@ void logfile::read_full_message(logfile::iterator ll,
max_lines -= 1;
} while (ll != this->end() && ll->is_continued() &&
(max_lines == -1 || max_lines > 0));
msg_out = stream.str();
}

23
src/logfile_sub_source.cc
View File

@ -268,7 +268,7 @@ void logfile_sub_source::text_value_for_line(textview_curses &tc,
{ 60, "%qd%s", "s" },
{ 60, "%qd%s", "m" },
{ 0, "%qd%s", "h" },
{ 0, NULL }
{ 0, NULL, NULL }
};
struct rel_interval *curr_interval;
@ -420,6 +420,7 @@ void logfile_sub_source::text_attrs_for_line(textview_curses &lv,
bool logfile_sub_source::rebuild_index(observer *obs, bool force)
{
std::vector<logfile_data>::iterator iter;
size_t total_lines = 0;
bool retval = force;
int file_count = 0;
@ -437,6 +438,7 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force)
retval = true;
}
file_count += 1;
total_lines += iter->ld_file->size();
}
}
if (force) {
@ -489,13 +491,6 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force)
content_line_t con_line(file_index * MAX_LINES_PER_FILE +
line_index);
if (obs != NULL) {
obs->logfile_sub_source_filtering(
*this,
content_line_t(con_line % MAX_LINES_PER_FILE),
ld->ld_file->size());
}
if (!(lf_iter->get_level() & logline::LEVEL_CONTINUED)) {
if (action_for_prev_line == logfile_filter::INCLUDE) {
while (last_owner->ld_indexing.ld_start <=
@ -510,6 +505,13 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force)
ld->ld_indexing.ld_start = con_line;
}
if (obs != NULL) {
obs->logfile_sub_source_filtering(
*this,
vis_line_t(this->lss_index.size()),
total_lines);
}
ld->ld_indexing.ld_last = con_line;
action_for_prev_line = ld->ld_file->check_filter(
lf_iter, this->lss_filter_generation, this->lss_filters);
@ -518,6 +520,11 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force)
merge.next();
}
if (obs != NULL) {
obs->logfile_sub_source_filtering(*this,
vis_line_t(total_lines - 1),
total_lines);
}
if (action_for_prev_line == logfile_filter::INCLUDE) {
while (last_owner->ld_indexing.ld_start <=

2
src/logfile_sub_source.hh
View File

@ -59,7 +59,7 @@ public:
: public logfile_observer {
public:
virtual void logfile_sub_source_filtering(logfile_sub_source &lss,
content_line_t cl,
vis_line_t cl,
size_t total) = 0;
};

2
src/pcrepp.hh
View File

@ -224,7 +224,7 @@ public:
void reset(const std::string &str, size_t off = 0)
{
this->reset(str.c_str(), off, str.length());
}
};
size_t pi_offset;
size_t pi_next_offset;

2
src/piper_proc.cc
View File

@ -141,7 +141,7 @@ piper_proc::piper_proc(int pipefd, bool timestamp, const char *filename)
}
woff += wrc;
}
} while (lb.get_file_size() == -1);
} while (lb.get_file_size() == (size_t)-1);
if (timestamp) {
int wrc;

2
src/readline_curses.cc
View File

@ -683,7 +683,7 @@ void readline_curses::do_update(void)
{
if (this->rc_active_context == -1) {
int alt_start = -1;
struct line_range lr = { 0, };
struct line_range lr = { 0 };
attr_line_t al, alt_al;
wmove(this->vc_window, this->get_actual_y(), 0);

2
src/session_data.cc
View File

@ -201,7 +201,7 @@ static void cleanup_session_data(void)
session_info_list.sort();
int session_loops = 0;
size_t session_loops = 0;
while (session_info_list.size() > MAX_SESSION_FILE_COUNT) {
const session_file_info &front = session_info_list.front();

2
src/sql_util.hh
View File

@ -66,7 +66,7 @@ void sql_strftime(char *buffer, size_t buffer_size, time_t time, int millis);
inline void sql_strftime(char *buffer, size_t buffer_size,
const struct timeval &tv)
{
sql_strftime(buffer, buffer_size, tv.tv_sec, tv.tv_usec);
sql_strftime(buffer, buffer_size, tv.tv_sec, tv.tv_usec / 1000);
}
#endif

4
src/textview_curses.cc
View File

@ -85,7 +85,9 @@ void textview_curses::grep_match(grep_proc &gp,
this->tc_sub_source->text_mark(&BM_SEARCH, line, true);
}
listview_curses::reload_data();
if (this->get_top() <= line && line <= this->get_bottom()) {
listview_curses::reload_data();
}
}
void textview_curses::listview_value_for_row(const listview_curses &lv,

35
src/yajlpp.cc
View File

@ -36,6 +36,7 @@
int yajlpp_parse_context::map_start(void *ctx)
{
yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx;
int retval = 1;
ypc->ypc_path_index_stack.push_back(ypc->ypc_path.length());
@ -43,7 +44,10 @@ int yajlpp_parse_context::map_start(void *ctx)
ypc->ypc_array_index.back() += 1;
}
return 1;
if (ypc->ypc_alt_callbacks.yajl_start_map != NULL)
retval = ypc->ypc_alt_callbacks.yajl_start_map(ypc);
return retval;
}
int yajlpp_parse_context::map_key(void *ctx,
@ -51,12 +55,16 @@ int yajlpp_parse_context::map_key(void *ctx,
size_t len)
{
yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx;
int retval = 1;
ypc->ypc_path = ypc->ypc_path.substr(0, ypc->ypc_path_index_stack.back());
ypc->ypc_path += "/" + std::string((const char *)key, len);
if (ypc->ypc_alt_callbacks.yajl_map_key != NULL)
retval = ypc->ypc_alt_callbacks.yajl_map_key(ctx, key, len);
ypc->update_callbacks();
return 1;
return retval;
}
void yajlpp_parse_context::update_callbacks(void)
@ -68,6 +76,8 @@ void yajlpp_parse_context::update_callbacks(void)
for (int lpc = 0; this->ypc_handlers[lpc].jph_path[0]; lpc++) {
const json_path_handler &jph = this->ypc_handlers[lpc];
pi.reset(this->ypc_path);
if (jph.jph_regex.match(this->ypc_pcre_context, pi)) {
if (jph.jph_callbacks.yajl_null != NULL)
this->ypc_callbacks.yajl_null = jph.jph_callbacks.yajl_null;
@ -86,44 +96,59 @@ void yajlpp_parse_context::update_callbacks(void)
int yajlpp_parse_context::map_end(void *ctx)
{
yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx;
int retval = 1;
ypc->ypc_path = ypc->ypc_path.substr(0, ypc->ypc_path_index_stack.back());
ypc->ypc_path_index_stack.pop_back();
if (ypc->ypc_alt_callbacks.yajl_end_map != NULL)
retval = ypc->ypc_alt_callbacks.yajl_end_map(ctx);
ypc->update_callbacks();
return 1;
return retval;
}
int yajlpp_parse_context::array_start(void *ctx)
{
yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx;
int retval = 1;
ypc->ypc_path_index_stack.push_back(ypc->ypc_path.length());
ypc->ypc_path += "#";
ypc->ypc_array_index.push_back(-1);
if (ypc->ypc_alt_callbacks.yajl_start_array != NULL)
retval = ypc->ypc_alt_callbacks.yajl_start_array(ctx);
ypc->update_callbacks();
return 1;
return retval;
}
int yajlpp_parse_context::array_end(void *ctx)
{
yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx;
int retval = 1;
ypc->ypc_path = ypc->ypc_path.substr(0, ypc->ypc_path_index_stack.back());
ypc->ypc_path_index_stack.pop_back();
ypc->ypc_array_index.pop_back();
if (ypc->ypc_alt_callbacks.yajl_end_array != NULL)
retval = ypc->ypc_alt_callbacks.yajl_end_array(ctx);
ypc->update_callbacks();
return 1;
return retval;
}
int yajlpp_parse_context::handle_unused(void *ctx)
{
yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx;
if (ypc->ypc_ignore_unused)
return 1;
fprintf(stderr, "warning:%s:%s:unexpected data, expecting one of the following data types --\n",
ypc->ypc_source.c_str(),
ypc->ypc_path.c_str());

37
src/yajlpp.hh
View File

@ -105,7 +105,39 @@ struct json_path_handler : public json_path_handler_base {
this->jph_callbacks.yajl_string = (int (*)(void *, const unsigned char *, size_t))str_func;
}
json_path_handler(const char *path) : json_path_handler_base(path) { };
json_path_handler() : json_path_handler_base("") {};
json_path_handler &add_cb(int(*null_func)(yajlpp_parse_context *)) {
this->jph_callbacks.yajl_null = (int (*)(void *))null_func;
return *this;
};
json_path_handler &add_cb(int(*bool_func)(yajlpp_parse_context *, int))
{
this->jph_callbacks.yajl_boolean = (int (*)(void *, int))bool_func;
return *this;
}
json_path_handler &add_cb(int(*int_func)(yajlpp_parse_context *, long long))
{
this->jph_callbacks.yajl_integer = (int (*)(void *, long long))int_func;
return *this;
}
json_path_handler &add_cb(int(*double_func)(yajlpp_parse_context *, double))
{
this->jph_callbacks.yajl_double = (int (*)(void *, double))double_func;
return *this;
}
json_path_handler &add_cb(int(*str_func)(yajlpp_parse_context *, const unsigned char *, size_t))
{
this->jph_callbacks.yajl_string = (int (*)(void *, const unsigned char *, size_t))str_func;
return *this;
}
};
class yajlpp_parse_context {
@ -129,9 +161,10 @@ public:
yajlpp_parse_context(std::string source,
struct json_path_handler *handlers)
: ypc_source(source), ypc_handlers(handlers)
: ypc_source(source), ypc_handlers(handlers), ypc_ignore_unused(false)
{
this->ypc_callbacks = DEFAULT_CALLBACKS;
memset(&this->ypc_alt_callbacks, 0, sizeof(this->ypc_alt_callbacks));
};
std::string get_path_fragment(int offset) const
@ -155,10 +188,12 @@ public:
struct json_path_handler *ypc_handlers;
void * ypc_userdata;
yajl_callbacks ypc_callbacks;
yajl_callbacks ypc_alt_callbacks;
std::string ypc_path;
std::vector<size_t> ypc_path_index_stack;
std::vector<int> ypc_array_index;
pcre_context_static<30> ypc_pcre_context;
bool ypc_ignore_unused;
private:
static const yajl_callbacks DEFAULT_CALLBACKS;

10
test/Makefile.am
View File

@ -82,7 +82,13 @@ drive_listview_SOURCES = drive_listview.cc
drive_listview_LDADD = ../src/libdiag.a $(CURSES_LIB) -lz
drive_logfile_SOURCES = drive_logfile.cc
drive_logfile_LDADD = ../src/libdiag.a -lcrypto $(CURSES_LIB) $(SQLITE3_LIBS)
drive_logfile_LDADD = \
../src/libdiag.a \
-lcrypto \
$(CURSES_LIB) \
$(SQLITE3_LIBS) \
$(PCRE_LIBS) \
-lpcrecpp
drive_sequencer_SOURCES = drive_sequencer.cc
drive_sequencer_LDADD = ../src/libdiag.a -lcrypto $(CURSES_LIB) $(SQLITE3_LIBS)
@ -93,7 +99,9 @@ drive_data_scanner_LDADD = \
../src/libdiag.a \
../src/default-log-formats-json.o \
-lcrypto \
$(PCRE_LIBS) \
$(SQLITE3_LIBS) \
-lpcrecpp \
$(CURSES_LIB)
drive_view_colors_SOURCES = drive_view_colors.cc

15
test/Makefile.in
View File

@ -120,7 +120,7 @@ drive_data_scanner_OBJECTS = $(am_drive_data_scanner_OBJECTS)
am__DEPENDENCIES_1 =
drive_data_scanner_DEPENDENCIES = ../src/libdiag.a \
../src/default-log-formats-json.o $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1)
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1)
am_drive_grep_proc_OBJECTS = drive_grep_proc.$(OBJEXT)
drive_grep_proc_OBJECTS = $(am_drive_grep_proc_OBJECTS)
drive_grep_proc_DEPENDENCIES = ../src/libdiag.a $(am__DEPENDENCIES_1)
@ -134,7 +134,7 @@ drive_listview_DEPENDENCIES = ../src/libdiag.a $(am__DEPENDENCIES_1)
am_drive_logfile_OBJECTS = drive_logfile.$(OBJEXT)
drive_logfile_OBJECTS = $(am_drive_logfile_OBJECTS)
drive_logfile_DEPENDENCIES = ../src/libdiag.a $(am__DEPENDENCIES_1) \
$(am__DEPENDENCIES_1)
$(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1)
am_drive_readline_curses_OBJECTS = drive_readline_curses.$(OBJEXT)
drive_readline_curses_OBJECTS = $(am_drive_readline_curses_OBJECTS)
drive_readline_curses_DEPENDENCIES = ../src/libdiag.a \
@ -639,7 +639,14 @@ drive_grep_proc_LDADD = ../src/libdiag.a $(PCRE_LIBS) -lz
drive_listview_SOURCES = drive_listview.cc
drive_listview_LDADD = ../src/libdiag.a $(CURSES_LIB) -lz
drive_logfile_SOURCES = drive_logfile.cc
drive_logfile_LDADD = ../src/libdiag.a -lcrypto $(CURSES_LIB) $(SQLITE3_LIBS)
drive_logfile_LDADD = \
../src/libdiag.a \
-lcrypto \
$(CURSES_LIB) \
$(SQLITE3_LIBS) \
$(PCRE_LIBS) \
-lpcrecpp
drive_sequencer_SOURCES = drive_sequencer.cc
drive_sequencer_LDADD = ../src/libdiag.a -lcrypto $(CURSES_LIB) $(SQLITE3_LIBS)
drive_data_scanner_SOURCES = \
@ -649,7 +656,9 @@ drive_data_scanner_LDADD = \
../src/libdiag.a \
../src/default-log-formats-json.o \
-lcrypto \
$(PCRE_LIBS) \
$(SQLITE3_LIBS) \
-lpcrecpp \
$(CURSES_LIB)
drive_view_colors_SOURCES = drive_view_colors.cc