From c59acba758d6a783e792612231d6eab7962ce49c Mon Sep 17 00:00:00 2001 From: Timothy Stack Date: Tue, 10 Sep 2013 06:20:37 -0700 Subject: [PATCH] [format] initial support for json formats and some bug fixes --- src/lnav.cc | 34 ++- src/lnav_commands.cc | 33 ++- src/log_data_table.hh | 10 +- src/log_format.cc | 539 +++++++++++++++++++++++++++++++++++--- src/log_format.hh | 160 +++++++++-- src/log_format_impls.cc | 4 +- src/log_format_loader.cc | 86 +++++- src/log_vtab_impl.cc | 22 +- src/log_vtab_impl.hh | 4 +- src/logfile.cc | 29 +- src/logfile_sub_source.cc | 23 +- src/logfile_sub_source.hh | 2 +- src/pcrepp.hh | 2 +- src/piper_proc.cc | 2 +- src/readline_curses.cc | 2 +- src/session_data.cc | 2 +- src/sql_util.hh | 2 +- src/textview_curses.cc | 4 +- src/yajlpp.cc | 35 ++- src/yajlpp.hh | 37 ++- test/Makefile.am | 10 +- test/Makefile.in | 15 +- 22 files changed, 932 insertions(+), 125 deletions(-) diff --git a/src/lnav.cc b/src/lnav.cc index 17a372c6..b1e5a2c8 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -183,7 +183,19 @@ public: content_line_t cl = lss.at(lv.get_top()); logfile * lf = lss.find(cl); - std::string line = lf->read_line(lf->begin() + cl); + logfile::iterator ll = lf->begin() + cl; + + if (ll->is_continued()) { + if (this->fos_parser) { + delete this->fos_parser; + delete this->fos_scanner; + delete this->fos_namer; + } + this->fos_parser = NULL; + return 0; + } + + std::string line = lf->read_line(ll); struct line_range body; string_attrs_t sa; @@ -214,7 +226,7 @@ public: delete this->fos_scanner; delete this->fos_namer; } - + this->fos_scanner = new data_scanner(line, body.lr_start, body.lr_end); this->fos_parser = new data_parser(this->fos_scanner); this->fos_parser->parse(); @@ -506,10 +518,10 @@ public: }; void logfile_sub_source_filtering(logfile_sub_source &lss, - content_line_t cl, + vis_line_t cl, size_t total) { - if (std::abs(cl - this->lo_last_line) > 1024 || (size_t)cl == + if (std::abs(cl - this->lo_last_line) > (4 * 1024) || (size_t)cl == (total - 1)) { lnav_data.ld_bottom_source.update_loading(cl, (total - 1)); this->do_update(); @@ -531,7 +543,7 @@ private: }; off_t lo_last_offset; - content_line_t lo_last_line; + vis_line_t lo_last_line; }; static void rebuild_hist(size_t old_count, bool force) @@ -2007,7 +2019,7 @@ void execute_search(lnav_view_t view, const std::string ®ex) lnav_data.ld_last_search[view] = regex; } -static void rl_search(void *dummy, readline_curses *rc) +static void rl_search_internal(void *dummy, readline_curses *rc, bool complete = false) { string term_val; string name; @@ -2061,10 +2073,16 @@ static void rl_search(void *dummy, readline_curses *rc) textview_curses *tc = lnav_data.ld_view_stack.top(); lnav_view_t index = (lnav_view_t)(tc - lnav_data.ld_views); - tc->set_top(lnav_data.ld_search_start_line); + if (!complete) + tc->set_top(lnav_data.ld_search_start_line); execute_search(index, rc->get_value()); } +static void rl_search(void *dummy, readline_curses *rc) +{ + rl_search_internal(dummy, rc); +} + static void rl_abort(void *dummy, readline_curses *rc) { textview_curses *tc = lnav_data.ld_view_stack.top(); @@ -2112,7 +2130,7 @@ static void rl_callback(void *dummy, readline_curses *rc) case LNM_SEARCH: case LNM_CAPTURE: - rl_search(dummy, rc); + rl_search_internal(dummy, rc, true); if (rc->get_value().size() > 0) { lnav_data.ld_view_stack.top()->set_follow_search(false); lnav_data.ld_rl_view-> diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index f0293ee7..047bb6b5 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -349,24 +349,26 @@ static string com_highlight(string cmdline, vector &args) if (args.size() == 0) { } else if (args.size() > 1) { + textview_curses *tc = lnav_data.ld_view_stack.top(); + textview_curses::highlight_map_t &hm = tc->get_highlights(); const char *errptr; pcre * code; int eoff; args[1] = cmdline.substr(cmdline.find(args[1])); - if ((code = pcre_compile(args[1].c_str(), - PCRE_CASELESS, - &errptr, - &eoff, - NULL)) == NULL) { + if (hm.find(args[1]) != hm.end()) { + retval = "error: highlight already exists"; + } + else if ((code = pcre_compile(args[1].c_str(), + PCRE_CASELESS, + &errptr, + &eoff, + NULL)) == NULL) { retval = "error: " + string(errptr); } else { - textview_curses * tc = lnav_data.ld_view_stack.top(); textview_curses::highlighter hl(code, false); - textview_curses::highlight_map_t &hm = tc->get_highlights(); - hm[args[1]] = hl; retval = "info: highlight pattern now active"; @@ -491,20 +493,23 @@ static string com_filter(string cmdline, vector &args) args.push_back("filter"); } else if (args.size() > 1) { + logfile_sub_source &lss = lnav_data.ld_log_source; const char *errptr; pcre * code; int eoff; args[1] = cmdline.substr(cmdline.find(args[1])); - if ((code = pcre_compile(args[1].c_str(), - 0, - &errptr, - &eoff, - NULL)) == NULL) { + if (lss.get_filter(args[1]) != NULL) { + retval = "error: filter already exists"; + } + else if ((code = pcre_compile(args[1].c_str(), + 0, + &errptr, + &eoff, + NULL)) == NULL) { retval = "error: " + string(errptr); } else { - logfile_sub_source & lss = lnav_data.ld_log_source; logfile_filter::type_t lt = (args[0] == "filter-out") ? logfile_filter::EXCLUDE : logfile_filter::INCLUDE; diff --git a/src/log_data_table.hh b/src/log_data_table.hh index 3e7b87a6..ed5b1971 100644 --- a/src/log_data_table.hh +++ b/src/log_data_table.hh @@ -50,7 +50,8 @@ public: log_data_table(content_line_t template_line, std::string table_name = "logline") : log_vtab_impl(table_name), - ldt_template_line(template_line) { + ldt_template_line(template_line), + ldt_parent_column_count(0) { logfile *lf = lnav_data.ld_log_source.find(template_line); log_format *format = lf->get_format(); @@ -71,6 +72,7 @@ public: if (this->ldt_format_impl != NULL) { this->ldt_format_impl->get_columns(cols); } + this->ldt_parent_column_count = cols.size(); lf->read_full_message(lf->begin() + cl_copy, val); format->annotate(val, sa, line_values); body = find_string_attr_range(sa, "body"); @@ -132,7 +134,7 @@ public: logfile * lf = lss.find(cl); logfile::iterator lf_iter = lf->begin() + cl; - if (lf_iter->get_level() & logline::LEVEL_CONTINUED) { + if (lf_iter->is_continued()) { return false; } @@ -174,6 +176,8 @@ public: const std::string &line, std::vector &values) { + int next_column = this->ldt_parent_column_count; + this->ldt_format_impl->extract(lf, line, values); for (data_parser::element_list_t::iterator pair_iter = this->ldt_pairs.begin(); @@ -196,6 +200,7 @@ public: values.push_back(logline_value("", tmp)); break; } + values.back().lv_column = next_column++; } }; @@ -206,5 +211,6 @@ private: std::string ldt_current_line; data_parser::element_list_t ldt_pairs; log_vtab_impl *ldt_format_impl; + int ldt_parent_column_count; }; #endif diff --git a/src/log_format.cc b/src/log_format.cc index a4f2de4a..c7102d08 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -32,7 +32,10 @@ #include #include #include +#include +#include "yajlpp.hh" +#include "sql_util.hh" #include "log_format.hh" #include "log_vtab_impl.hh" @@ -64,46 +67,51 @@ const char *logline::level_names[LEVEL__MAX] = { "fatal", }; -static int strcasestr_i(const char *s1, const char *s2) +static int strncasestr_i(const char *s1, const char *s2, size_t len) { return strcasestr(s1, s2) == NULL; } -logline::level_t logline::string2level(const char *levelstr, bool exact) +logline::level_t logline::string2level(const char *levelstr, size_t len, bool exact) { logline::level_t retval = logline::LEVEL_UNKNOWN; - int (*cmpfunc)(const char *, const char *); + int (*cmpfunc)(const char *, const char *, size_t); + + assert(len == (size_t)-1 || (len != (size_t)-1 && exact)); + + if (len == (size_t)-1) + len = strlen(levelstr); if (exact) { - cmpfunc = strcasecmp; + cmpfunc = strncasecmp; } else{ - cmpfunc = strcasestr_i; + cmpfunc = strncasestr_i; } - if (cmpfunc(levelstr, "TRACE") == 0) { + if (cmpfunc(levelstr, "TRACE", len) == 0) { retval = logline::LEVEL_TRACE; } - else if (cmpfunc(levelstr, "VERBOSE") == 0) { + else if (cmpfunc(levelstr, "VERBOSE", len) == 0) { retval = logline::LEVEL_DEBUG; } - else if (cmpfunc(levelstr, "DEBUG") == 0) { + else if (cmpfunc(levelstr, "DEBUG", len) == 0) { retval = logline::LEVEL_DEBUG; } - else if (cmpfunc(levelstr, "INFO") == 0) { + else if (cmpfunc(levelstr, "INFO", len) == 0) { retval = logline::LEVEL_INFO; } - else if (cmpfunc(levelstr, "WARNING") == 0) { + else if (cmpfunc(levelstr, "WARNING", len) == 0) { retval = logline::LEVEL_WARNING; } - else if (cmpfunc(levelstr, "ERROR") == 0) { + else if (cmpfunc(levelstr, "ERROR", len) == 0) { retval = logline::LEVEL_ERROR; } - else if (cmpfunc(levelstr, "CRITICAL") == 0) { + else if (cmpfunc(levelstr, "CRITICAL", len) == 0) { retval = logline::LEVEL_CRITICAL; } - else if (cmpfunc(levelstr, "FATAL") == 0) { + else if (cmpfunc(levelstr, "FATAL", len) == 0) { retval = logline::LEVEL_FATAL; } @@ -121,6 +129,9 @@ logline_value::kind_t logline_value::string2kind(const char *kindstr) else if (strcmp(kindstr, "float") == 0) { return VALUE_FLOAT; } + else if (strcmp(kindstr, "boolean") == 0) { + return VALUE_BOOLEAN; + } return VALUE_UNKNOWN; } @@ -197,11 +208,242 @@ const char *log_format::log_scanf(const char *line, return retval; } +/* + * XXX This needs some cleanup. + */ +struct json_log_userdata { + json_log_userdata() : jlu_sub_line_count(1) { }; + + external_log_format *jlu_format; + const logline *jlu_line; + logline *jlu_base_line; + int jlu_sub_line_count; + yajl_handle jlu_handle; + const char *jlu_line_value; + size_t jlu_sub_start; +}; + +struct json_field_cmp { + json_field_cmp(external_log_format::json_log_field type, + const std::string &name) + : jfc_type(type), jfc_field_name(name) { + }; + + bool operator()(const external_log_format::json_format_element &jfe) const { + return (this->jfc_type == jfe.jfe_type && + this->jfc_field_name == jfe.jfe_value); + }; + + external_log_format::json_log_field jfc_type; + const std::string &jfc_field_name; +}; + +static int read_json_field(yajlpp_parse_context *ypc, const unsigned char *str, size_t len); + +static int read_json_null(yajlpp_parse_context *ypc) +{ + json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata; + vector &line_format = + jlu->jlu_format->jlf_line_format; + string field_name = ypc->get_path_fragment(0); + + if (find_if(line_format.begin(), line_format.end(), + json_field_cmp(external_log_format::JLF_VARIABLE, + field_name)) == line_format.end()) { + jlu->jlu_sub_line_count += 1; + } + + return 1; +} + +static int read_json_bool(yajlpp_parse_context *ypc, int val) +{ + json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata; + vector &line_format = + jlu->jlu_format->jlf_line_format; + string field_name = ypc->get_path_fragment(0); + + if (find_if(line_format.begin(), line_format.end(), + json_field_cmp(external_log_format::JLF_VARIABLE, + field_name)) == line_format.end()) { + jlu->jlu_sub_line_count += 1; + } + + return 1; +} + +static int read_json_int(yajlpp_parse_context *ypc, long long val) +{ + json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata; + vector &line_format = + jlu->jlu_format->jlf_line_format; + string field_name = ypc->get_path_fragment(0); + + if (find_if(line_format.begin(), line_format.end(), + json_field_cmp(external_log_format::JLF_VARIABLE, + field_name)) == line_format.end()) { + jlu->jlu_sub_line_count += 1; + } + + return 1; +} + +static int read_json_double(yajlpp_parse_context *ypc, double val) +{ + json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata; + vector &line_format = + jlu->jlu_format->jlf_line_format; + string field_name = ypc->get_path_fragment(0); + + if (find_if(line_format.begin(), line_format.end(), + json_field_cmp(external_log_format::JLF_VARIABLE, + field_name)) == line_format.end()) { + jlu->jlu_sub_line_count += 1; + } + + return 1; +} + +static int json_array_start(void *ctx) +{ + yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx; + json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata; + vector &line_format = + jlu->jlu_format->jlf_line_format; + + if (ypc->ypc_path_index_stack.size() == 2) { + string field_name = ypc->get_path_fragment(0); + + if (find_if(line_format.begin(), line_format.end(), + json_field_cmp(external_log_format::JLF_VARIABLE, + field_name)) == line_format.end()) { + jlu->jlu_sub_line_count += 1; + } + + jlu->jlu_sub_start = yajl_get_bytes_consumed(jlu->jlu_handle) - 1; + } + + return 1; +} + +static int json_array_end(void *ctx) +{ + yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx; + json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata; + + if (ypc->ypc_path_index_stack.size() == 1) { + string field_name = ypc->get_path_fragment(0); + size_t sub_end = yajl_get_bytes_consumed(jlu->jlu_handle); + string str = string(&jlu->jlu_line_value[jlu->jlu_sub_start], + sub_end - jlu->jlu_sub_start); + + jlu->jlu_format->jlf_line_values.push_back( + logline_value(field_name, str)); + } + + return 1; +} +static struct json_path_handler json_log_handlers[] = { + json_path_handler("^/\\w+$"). + add_cb(read_json_null). + add_cb(read_json_bool). + add_cb(read_json_int). + add_cb(read_json_double). + add_cb(read_json_field), + + json_path_handler() +}; + +static int rewrite_json_field(yajlpp_parse_context *ypc, const unsigned char *str, size_t len); + +static int rewrite_json_null(yajlpp_parse_context *ypc) +{ + json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata; + string field_name = ypc->get_path_fragment(0); + + jlu->jlu_format->jlf_line_values.push_back(logline_value(field_name)); + + return 1; +} + +static int rewrite_json_bool(yajlpp_parse_context *ypc, int val) +{ + json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata; + string field_name = ypc->get_path_fragment(0); + + jlu->jlu_format->jlf_line_values.push_back(logline_value(field_name, (bool)val)); + + return 1; +} + +static int rewrite_json_int(yajlpp_parse_context *ypc, long long val) +{ + json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata; + string field_name = ypc->get_path_fragment(0); + + jlu->jlu_format->jlf_line_values.push_back(logline_value(field_name, val)); + + return 1; +} + +static int rewrite_json_double(yajlpp_parse_context *ypc, double val) +{ + json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata; + string field_name = ypc->get_path_fragment(0); + + jlu->jlu_format->jlf_line_values.push_back(logline_value(field_name, val)); + + return 1; +} + +static struct json_path_handler json_log_rewrite_handlers[] = { + json_path_handler("^/\\w+$"). + add_cb(rewrite_json_null). + add_cb(rewrite_json_bool). + add_cb(rewrite_json_int). + add_cb(rewrite_json_double). + add_cb(rewrite_json_field), + + json_path_handler() +}; + bool external_log_format::scan(std::vector &dst, off_t offset, char *prefix, int len) { + if (this->jlf_json) { + auto_mem handle(yajl_free); + yajlpp_parse_context ypc("", json_log_handlers); + logline ll(offset, 0, 0, logline::LEVEL_INFO); + json_log_userdata jlu; + bool retval = false; + + ypc.ypc_userdata = &jlu; + ypc.ypc_ignore_unused = true; + ypc.ypc_alt_callbacks.yajl_start_array = json_array_start; + ypc.ypc_alt_callbacks.yajl_start_map = json_array_start; + handle = yajl_alloc(&ypc.ypc_callbacks, NULL, &ypc); // XXX + jlu.jlu_format = this; + jlu.jlu_base_line = ≪ + jlu.jlu_line_value = prefix; + jlu.jlu_handle = handle; + if (yajl_parse(handle.in(), (const unsigned char *)prefix, len) == yajl_status_ok && + yajl_complete_parse(handle.in()) == yajl_status_ok) { + for (int lpc = 0; lpc < jlu.jlu_sub_line_count; lpc++) { + ll.set_sub_offset(lpc); + if (lpc > 0) { + ll.set_level((logline::level_t) (ll.get_level() | + logline::LEVEL_CONTINUED)); + } + dst.push_back(ll); + } + retval = true; + } + + return retval; + } + pcre_input pi(prefix, 0, len); pcre_context_static<128> pc; bool retval = false; @@ -212,7 +454,7 @@ bool external_log_format::scan(std::vector &dst, continue; } - pcre_context::capture_t *ts = pc["timestamp"]; + pcre_context::capture_t *ts = pc[this->lf_timestamp_field]; pcre_context::capture_t *level_cap = pc[this->elf_level_field]; const char *ts_str = pi.get_substr_start(ts); const char *last; @@ -276,13 +518,19 @@ void external_log_format::annotate(const std::string &line, struct line_range lr; pcre_context::capture_t *cap; + if (this->jlf_json) { + values = this->jlf_line_values; + sa = this->jlf_line_attrs; + return; + } + pattern &pat = *this->elf_pattern_order[this->lf_fmt_lock]; if (!pat.p_pcre->match(pc, pi)) { return; } - cap = pc["timestamp"]; + cap = pc[this->lf_timestamp_field]; lr.lr_start = cap->c_begin; lr.lr_end = cap->c_end; sa[lr].insert(make_string_attr("timestamp", 0)); @@ -324,7 +572,8 @@ void external_log_format::annotate(const std::string &line, vd.vd_kind, pi.get_substr(pc[vd.vd_index]), vd.vd_identifier, - scaling)); + scaling, + vd.vd_column)); if (pc[vd.vd_index]->c_begin != -1 && vd.vd_identifier) { lr.lr_start = pc[vd.vd_index]->c_begin; @@ -334,8 +583,210 @@ void external_log_format::annotate(const std::string &line, } } +static int read_json_field(yajlpp_parse_context *ypc, const unsigned char *str, size_t len) +{ + json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata; + vector &line_format = + jlu->jlu_format->jlf_line_format; + string field_name = ypc->get_path_fragment(0); + struct tm tm_out; + struct timeval tv_out; + + if (field_name == jlu->jlu_format->lf_timestamp_field) { + jlu->jlu_format->lf_date_time.scan((const char *)str, NULL, &tm_out, tv_out); + jlu->jlu_base_line->set_time(tv_out); + } + else if (field_name == jlu->jlu_format->elf_level_field) { + jlu->jlu_base_line->set_level(logline::string2level((const char *)str, len, true)); + } + else { + if (find_if(line_format.begin(), line_format.end(), + json_field_cmp(external_log_format::JLF_VARIABLE, + field_name)) == line_format.end()) { + jlu->jlu_sub_line_count += 1; + } + for (size_t lpc = 0; lpc < len; lpc++) { + if (str[lpc] == '\n') { + jlu->jlu_sub_line_count += 1; + } + } + } + + return 1; +} + +static int rewrite_json_field(yajlpp_parse_context *ypc, const unsigned char *str, size_t len) +{ + json_log_userdata *jlu = (json_log_userdata *)ypc->ypc_userdata; + string field_name = ypc->get_path_fragment(0); + string val = string((const char *)str, len); + + if (field_name == jlu->jlu_format->lf_timestamp_field) { + char time_buf[64]; + + sql_strftime(time_buf, sizeof(time_buf), jlu->jlu_line->get_timeval()); + val = time_buf; + } + if (field_name == jlu->jlu_format->elf_body_field) { + jlu->jlu_format->jlf_line_values.push_back(logline_value("body", val)); + } + jlu->jlu_format->jlf_line_values.push_back(logline_value(field_name, val)); + + return 1; +} + +void external_log_format::get_subline(const logline &ll, + const char *line, size_t len, + ostringstream &stream_out) +{ + if (!this->jlf_json) { + stream_out.write(line, len); + return; + } + + if (this->jlf_cached_offset != ll.get_offset()) { + auto_mem handle(yajl_free); + yajlpp_parse_context ypc("", json_log_rewrite_handlers); + view_colors &vc = view_colors::singleton(); + json_log_userdata jlu; + + jlu.jlu_format = this; + jlu.jlu_line = ≪ + ypc.ypc_userdata = &jlu; + ypc.ypc_ignore_unused = true; + this->jlf_cached_line.clear(); + this->jlf_line_values.clear(); + this->jlf_line_offsets.clear(); + this->jlf_line_attrs.clear(); + ypc.ypc_alt_callbacks.yajl_start_array = json_array_start; + ypc.ypc_alt_callbacks.yajl_end_array = json_array_end; + ypc.ypc_alt_callbacks.yajl_start_map = json_array_start; + ypc.ypc_alt_callbacks.yajl_end_map = json_array_end; + handle = yajl_alloc(&ypc.ypc_callbacks, NULL, &ypc); + jlu.jlu_handle = handle.in(); + jlu.jlu_line_value = line; + if (yajl_parse(handle.in(), (const unsigned char *)line, len) == yajl_status_ok && + yajl_complete_parse(handle.in()) == yajl_status_ok) { + std::vector::iterator lv_iter; + std::vector::iterator iter; + bool used_values[this->jlf_line_values.size()]; + struct line_range lr; + ostringstream lines; + + memset(used_values, 0, sizeof(used_values)); + + for (lv_iter = this->jlf_line_values.begin(); + lv_iter != this->jlf_line_values.end(); + ++lv_iter) { + map::iterator vd_iter; + + vd_iter = this->elf_value_defs.find(lv_iter->lv_name); + if (vd_iter != this->elf_value_defs.end()) { + lv_iter->lv_identifier = vd_iter->second.vd_identifier; + lv_iter->lv_column = vd_iter->second.vd_column; + } + } + + for (iter = this->jlf_line_format.begin(); + iter != this->jlf_line_format.end(); + ++iter) { + switch (iter->jfe_type) { + case JLF_CONSTANT: + lines << iter->jfe_default_value; + break; + case JLF_VARIABLE: + lv_iter = find_if(this->jlf_line_values.begin(), + this->jlf_line_values.end(), + logline_value_cmp(&iter->jfe_value)); + if (lv_iter != this->jlf_line_values.end()) { + string str = lv_iter->to_string(); + size_t nl_pos = str.find('\n'); + + lr.lr_start = lines.tellp(); + lines << str; + if (nl_pos == string::npos) + lr.lr_end = lines.tellp(); + else + lr.lr_end = lr.lr_start + nl_pos; + if (lv_iter->lv_name == this->lf_timestamp_field) + this->jlf_line_attrs[lr].insert(make_string_attr("timestamp", 0)); + else if (lv_iter->lv_name == this->elf_body_field) + this->jlf_line_attrs[lr].insert(make_string_attr("body", 0)); + else if (lv_iter->lv_identifier) { + fprintf(stderr, "ident!! %s\n", lv_iter->lv_name.c_str()); + this->jlf_line_attrs[lr].insert(make_string_attr("style", vc.attrs_for_ident(str.c_str(), lr.length()))); + } + used_values[distance(this->jlf_line_values.begin(), + lv_iter)] = true; + } + else { + lines << iter->jfe_default_value; + } + break; + } + } + lines << endl; + for (size_t lpc = 0; lpc < this->jlf_line_values.size(); lpc++) { + const logline_value &lv = this->jlf_line_values[lpc]; + + if (used_values[lpc] || + lv.lv_name == "body" || + lv.lv_name == this->elf_level_field) + continue; + + const std::string str = lv.to_string(); + size_t curr_pos = 0, nl_pos, line_len = -1; + + do { + nl_pos = str.find('\n', curr_pos); + if (nl_pos != std::string::npos) { + line_len = nl_pos - curr_pos; + } + lines << " " + << lv.lv_name + << ": " + << str.substr(curr_pos, line_len) + << endl; + curr_pos = nl_pos + 1; + line_len = -1; + } while (nl_pos != std::string::npos && + nl_pos < str.size()); + } + this->jlf_cached_line = lines.str(); + + this->jlf_line_offsets.push_back(0); + for (size_t lpc = 0; lpc < this->jlf_cached_line.size(); lpc++) { + if (this->jlf_cached_line[lpc] == '\n') { + this->jlf_line_offsets.push_back(lpc); + } + } + this->jlf_line_offsets.push_back(this->jlf_cached_line.size()); + } + + this->jlf_cached_offset = ll.get_offset(); + } + + off_t this_off, next_off; + + this_off = this->jlf_line_offsets[ll.get_sub_offset()]; + if (this->jlf_cached_line[this_off] == '\n') + this_off += 1; + next_off = this->jlf_line_offsets[ll.get_sub_offset() + 1]; + + stream_out.write(this->jlf_cached_line.c_str() + this_off, + next_off - this_off); +} + void external_log_format::build(std::vector &errors) { + try { + this->elf_filename_pcre = new pcrepp(this->elf_file_pattern.c_str()); + } + catch (const pcrepp::error &e) { + errors.push_back("error:" + + this->elf_name + ".file-pattern:" + + e.what()); + } for (std::map::iterator iter = this->elf_patterns.begin(); iter != this->elf_patterns.end(); ++iter) { @@ -356,9 +807,14 @@ void external_log_format::build(std::vector &errors) value_iter = this->elf_value_defs.find(std::string(name_iter->pnc_name)); if (value_iter != this->elf_value_defs.end()) { - value_iter->second.vd_index = name_iter->index(); - value_iter->second.vd_unit_field_index = iter->second.p_pcre->name_index(value_iter->second.vd_unit_field.c_str()); - iter->second.p_value_by_index.push_back(value_iter->second); + value_def &vd = value_iter->second; + + vd.vd_index = name_iter->index(); + vd.vd_unit_field_index = iter->second.p_pcre->name_index(vd.vd_unit_field.c_str()); + if (vd.vd_column == -1) { + vd.vd_column = this->elf_column_count++; + } + iter->second.p_value_by_index.push_back(vd); } } @@ -368,10 +824,19 @@ void external_log_format::build(std::vector &errors) this->elf_pattern_order.push_back(&iter->second); } - if (this->elf_patterns.empty()) { - errors.push_back("error:" + - this->elf_name + - ": no regexes specified for format"); + if (this->jlf_json) { + if (!this->elf_patterns.empty()) { + errors.push_back("error:" + + this->elf_name + + ": JSON logs cannot have regexes"); + } + } + else { + if (this->elf_patterns.empty()) { + errors.push_back("error:" + + this->elf_name + + ": no regexes specified for format"); + } } for (std::map::iterator iter = this->elf_level_patterns.begin(); @@ -386,7 +851,15 @@ void external_log_format::build(std::vector &errors) } } - if (this->elf_samples.empty()) { + for (std::map::iterator iter = this->elf_value_defs.begin(); + iter != this->elf_value_defs.end(); + ++iter) { + if (iter->second.vd_column == -1) { + iter->second.vd_column = this->elf_column_count++; + } + } + + if (!this->jlf_json && this->elf_samples.empty()) { errors.push_back("error:" + this->elf_name + ":no sample logs provided, all formats must have samples"); @@ -459,21 +932,25 @@ public: }; void get_columns(vector &cols) { - std::vector::const_iterator iter; + std::map::const_iterator iter; const external_log_format &elf = this->elt_format; - for (iter = elf.elf_pattern_order[0]->p_value_by_index.begin(); - iter != elf.elf_pattern_order[0]->p_value_by_index.end(); + cols.resize(elf.elf_value_defs.size()); + for (iter = elf.elf_value_defs.begin(); + iter != elf.elf_value_defs.end(); ++iter) { + const external_log_format::value_def &vd = iter->second; int type; - switch (iter->vd_kind) { + switch (vd.vd_kind) { + case logline_value::VALUE_NULL: case logline_value::VALUE_TEXT: type = SQLITE3_TEXT; break; case logline_value::VALUE_FLOAT: type = SQLITE_FLOAT; break; + case logline_value::VALUE_BOOLEAN: case logline_value::VALUE_INTEGER: type = SQLITE_INTEGER; break; @@ -481,9 +958,9 @@ public: assert(0); break; } - cols.push_back(vtab_column(iter->vd_name.c_str(), - type, - iter->vd_collate.c_str())); + cols[vd.vd_column].vc_name = vd.vd_name.c_str(); + cols[vd.vd_column].vc_type = type; + cols[vd.vd_column].vc_collator = vd.vd_collate.c_str(); } }; diff --git a/src/log_format.hh b/src/log_format.hh index 796580cb..a7b77f56 100644 --- a/src/log_format.hh +++ b/src/log_format.hh @@ -43,6 +43,7 @@ #include #include #include +#include #include "pcrepp.hh" #include "lnav_util.hh" @@ -78,6 +79,10 @@ public: virtual std::string to_command(void) = 0; + bool operator==(const std::string &rhs) { + return this->lf_id == rhs; + }; + protected: bool lf_enabled; type_t lf_type; @@ -116,7 +121,7 @@ public: static const char *level_names[LEVEL__MAX]; - static level_t string2level(const char *levelstr, bool exact = false); + static level_t string2level(const char *levelstr, size_t len = -1, bool exact = false); /** * Construct a logline object with the given values. @@ -129,13 +134,13 @@ public: logline(off_t off, time_t t, uint16_t millis, - level_t l, - uint8_t m = 0) + level_t l) : ll_offset(off), ll_time(t), ll_millis(millis), ll_level(l), - ll_filter_state(logfile_filter::MAYBE) + ll_filter_state(logfile_filter::MAYBE), + ll_sub_offset(0) { memset(this->ll_schema, 0, sizeof(this->ll_schema)); }; @@ -146,7 +151,8 @@ public: uint8_t m = 0) : ll_offset(off), ll_level(l), - ll_filter_state(logfile_filter::MAYBE) + ll_filter_state(logfile_filter::MAYBE), + ll_sub_offset(0) { this->set_time(tv); memset(this->ll_schema, 0, sizeof(this->ll_schema)); @@ -155,6 +161,10 @@ public: /** @return The offset of the line in the file. */ off_t get_offset() const { return this->ll_offset; }; + uint16_t get_sub_offset() const { return this->ll_sub_offset; }; + + void set_sub_offset(uint16_t suboff) { this->ll_sub_offset = suboff; }; + /** @return The timestamp for the line. */ time_t get_time() const { return this->ll_time; }; @@ -266,6 +276,7 @@ private: uint16_t ll_millis; uint8_t ll_level; uint8_t ll_filter_state; + uint16_t ll_sub_offset; char ll_schema[4]; }; @@ -300,28 +311,42 @@ class logline_value { public: enum kind_t { VALUE_UNKNOWN = -1, + VALUE_NULL, VALUE_TEXT, VALUE_INTEGER, VALUE_FLOAT, + VALUE_BOOLEAN, }; static kind_t string2kind(const char *kindstr); + logline_value(std::string name) + : lv_name(name), lv_kind(VALUE_NULL), lv_identifier(), lv_column(-1) { }; + logline_value(std::string name, bool b) + : lv_name(name), + lv_kind(VALUE_BOOLEAN), + lv_number((int64_t)(b ? 1 : 0)), + lv_identifier(), + lv_column(-1) { }; logline_value(std::string name, int64_t i) - : lv_name(name), lv_kind(VALUE_INTEGER), lv_number(i) { }; + : lv_name(name), lv_kind(VALUE_INTEGER), lv_number(i), lv_identifier(), lv_column(-1) { }; logline_value(std::string name, double i) - : lv_name(name), lv_kind(VALUE_FLOAT), lv_number(i) { }; + : lv_name(name), lv_kind(VALUE_FLOAT), lv_number(i), lv_identifier(), lv_column(-1) { }; logline_value(std::string name, std::string s) - : lv_name(name), lv_kind(VALUE_TEXT), lv_string(s) { }; + : lv_name(name), lv_kind(VALUE_TEXT), lv_string(s), lv_identifier(), lv_column(-1) { }; logline_value(std::string name, kind_t kind, std::string s, - bool ident=false, const scaling_factor *scaling=NULL) - : lv_name(name), lv_kind(kind), lv_identifier(ident) + bool ident=false, const scaling_factor *scaling=NULL, + int col=-1) + : lv_name(name), lv_kind(kind), lv_identifier(ident), lv_column(col) { switch (kind) { case VALUE_TEXT: this->lv_string = s; break; + case VALUE_NULL: + break; + case VALUE_INTEGER: sscanf(s.c_str(), "%" PRId64 "", &this->lv_number.i); if (scaling != NULL) { @@ -336,17 +361,29 @@ public: } break; + case VALUE_BOOLEAN: + if (s == "true" || s == "yes") { + this->lv_number.i = 1; + } + else { + this->lv_number.i = 0; + } + break; + case VALUE_UNKNOWN: assert(0); break; } }; - const std::string to_string() + const std::string to_string() const { char buffer[128]; switch (this->lv_kind) { + case VALUE_NULL: + return "null"; + case VALUE_TEXT: return this->lv_string; @@ -357,9 +394,18 @@ public: case VALUE_FLOAT: snprintf(buffer, sizeof(buffer), "%lf", this->lv_number.d); break; + + case VALUE_BOOLEAN: + if (this->lv_number.i) { + return "true"; + } + else { + return "false"; + } + break; case VALUE_UNKNOWN: - assert(0); - break; + assert(0); + break; } return std::string(buffer); @@ -377,6 +423,28 @@ public: } lv_number; std::string lv_string; bool lv_identifier; + int lv_column; +}; + +struct logline_value_cmp { + logline_value_cmp(const std::string *name = NULL, int col = -1) + : lvc_name(name), lvc_column(col) { + + }; + + bool operator()(const logline_value &lv) { + bool retval = true; + + if (this->lvc_name != NULL) + retval = retval && ((*this->lvc_name) == lv.lv_name); + if (this->lvc_column != -1) + retval = retval && (this->lvc_column == lv.lv_column); + + return retval; + }; + + const std::string *lvc_name; + int lvc_column; }; class log_vtab_impl; @@ -406,7 +474,7 @@ public: }; }; - log_format() : lf_fmt_lock(-1) { + log_format() : lf_fmt_lock(-1), lf_timestamp_field("timestamp") { }; virtual ~log_format() { }; @@ -423,6 +491,8 @@ public: */ virtual std::string get_name(void) const = 0; + virtual bool match_name(const std::string &filename) { return true; }; + /** * Scan a log line to see if it matches this log format. * @@ -458,8 +528,15 @@ public: return NULL; }; + virtual void get_subline(const logline &ll, + const char *line, size_t len, + std::ostringstream &stream_out) { + stream_out.write(line, len); + }; + date_time_scanner lf_date_time; int lf_fmt_lock; + std::string lf_timestamp_field; protected: static std::vector lf_root_formats; @@ -487,7 +564,8 @@ public: vd_kind(logline_value::VALUE_UNKNOWN), vd_identifier(false), vd_foreign_key(false), - vd_unit_field_index(-1) { + vd_unit_field_index(-1), + vd_column(-1) { }; @@ -500,6 +578,7 @@ public: std::string vd_unit_field; int vd_unit_field_index; std::map vd_unit_scaling; + int vd_column; bool operator<(const value_def &rhs) const { return this->vd_index < rhs.vd_index; @@ -522,12 +601,26 @@ public: pcrepp *lp_pcre; }; - external_log_format(const std::string &name) : elf_name(name) { }; + external_log_format(const std::string &name) + : elf_file_pattern(".*"), + elf_column_count(0), + elf_body_field("body"), + jlf_cached_offset(-1), + elf_name(name) { + this->jlf_line_offsets.reserve(128); + }; std::string get_name(void) const { return this->elf_name; }; + bool match_name(const std::string &filename) { + pcre_context_static<10> pc; + pcre_input pi(filename); + + return this->elf_filename_pcre->match(pc, pi); + }; + bool scan(std::vector &dst, off_t offset, char *prefix, @@ -543,21 +636,50 @@ public: std::auto_ptr retval((log_format *) new external_log_format(*this)); - retval->lf_fmt_lock = this->lf_fmt_lock; - retval->lf_date_time = this->lf_date_time; - return retval; }; + void get_subline(const logline &ll, + const char *line, size_t len, + std::ostringstream &stream_out); + log_vtab_impl *get_vtab_impl(void) const; + std::string elf_file_pattern; + pcrepp *elf_filename_pcre; std::map elf_patterns; std::vector elf_pattern_order; std::vector elf_samples; std::map elf_value_defs; + int elf_column_count; std::string elf_level_field; + std::string elf_body_field; std::map elf_level_patterns; + enum json_log_field { + JLF_CONSTANT, + JLF_VARIABLE, + }; + + struct json_format_element { + json_format_element() + : jfe_type(JLF_CONSTANT), jfe_default_value("-"), jfe_min_width(0) + { }; + + json_log_field jfe_type; + std::string jfe_value; + std::string jfe_default_value; + int jfe_min_width; + }; + + bool jlf_json; + std::vector jlf_line_format; + std::vector jlf_line_values; + + off_t jlf_cached_offset; + std::vector jlf_line_offsets; + std::string jlf_cached_line; + string_attrs_t jlf_line_attrs; private: const std::string elf_name; diff --git a/src/log_format_impls.cc b/src/log_format_impls.cc index 9a53eeea..f288f2fb 100644 --- a/src/log_format_impls.cc +++ b/src/log_format_impls.cc @@ -192,7 +192,7 @@ class generic_log_format : public log_format { } } - if (logline::string2level(level, true) == logline::LEVEL_UNKNOWN) { + if (logline::string2level(level, -1, true) == logline::LEVEL_UNKNOWN) { prefix_len = lr.lr_end; } @@ -308,7 +308,7 @@ class strace_log_format : public log_format { { "result", logline_value::VALUE_TEXT }, { "duration", logline_value::VALUE_FLOAT }, - { NULL }, + { NULL, logline_value::VALUE_UNKNOWN }, }; pcre_context::iterator iter; diff --git a/src/log_format_loader.cc b/src/log_format_loader.cc index 9f11051f..e03a4673 100644 --- a/src/log_format_loader.cc +++ b/src/log_format_loader.cc @@ -78,6 +78,8 @@ static int read_format_bool(yajlpp_parse_context *ypc, int val) if (field_name == "local-time") elf->lf_date_time.dts_local_time = val; + else if (field_name == "json") + elf->jlf_json = val; return 1; } @@ -88,8 +90,14 @@ static int read_format_field(yajlpp_parse_context *ypc, const unsigned char *str string value = string((const char *)str, len); string field_name = ypc->get_path_fragment(1); - if (field_name == "level-field") + if (field_name == "file-pattern") + elf->elf_file_pattern = value; + else if (field_name == "level-field") elf->elf_level_field = value; + else if (field_name == "timestamp-field") + elf->lf_timestamp_field = value; + else if (field_name == "body-field") + elf->elf_body_field = value; return 1; } @@ -202,18 +210,76 @@ static int read_sample_line(yajlpp_parse_context *ypc, const unsigned char *str, return 1; } +static external_log_format::json_format_element & + ensure_json_format_element(external_log_format *elf, int index) +{ + elf->jlf_line_format.resize(index + 1); + + return elf->jlf_line_format[index]; +} + +static int read_json_constant(yajlpp_parse_context *ypc, const unsigned char *str, size_t len) +{ + external_log_format *elf = ensure_format(ypc->get_path_fragment(0)); + string val = string((const char *)str, len); + + ypc->ypc_array_index.back() += 1; + + int index = ypc->ypc_array_index.back(); + external_log_format::json_format_element &jfe = ensure_json_format_element(elf, index); + + jfe.jfe_type = external_log_format::JLF_CONSTANT; + jfe.jfe_default_value = val; + + return 1; +} + +static int read_json_variable(yajlpp_parse_context *ypc, const unsigned char *str, size_t len) +{ + external_log_format *elf = ensure_format(ypc->get_path_fragment(0)); + string val = string((const char *)str, len); + int index = ypc->ypc_array_index.back(); + external_log_format::json_format_element &jfe = ensure_json_format_element(elf, index); + string field_name = ypc->get_path_fragment(3); + + jfe.jfe_type = external_log_format::JLF_VARIABLE; + if (field_name == "field") + jfe.jfe_value = val; + else if (field_name == "default-value") + jfe.jfe_default_value = val; + + return 1; +} + +static int read_json_variable_num(yajlpp_parse_context *ypc, long long val) +{ + external_log_format *elf = ensure_format(ypc->get_path_fragment(0)); + int index = ypc->ypc_array_index.back(); + external_log_format::json_format_element &jfe = ensure_json_format_element(elf, index); + string field_name = ypc->get_path_fragment(2); + + jfe.jfe_type = external_log_format::JLF_VARIABLE; + jfe.jfe_min_width = val; + + return 1; +} + static struct json_path_handler format_handlers[] = { - json_path_handler("/\\w+/regex/[^/]+/pattern", read_format_regex), - json_path_handler("/\\w+/local-time", read_format_bool), - json_path_handler("/\\w+/(level-field|url|title|description)", read_format_field), - json_path_handler("/\\w+/level/" - "(trace|debug|info|warning|error|critical|fatal)", + json_path_handler("^/\\w+/regex/[^/]+/pattern$", read_format_regex), + json_path_handler("^/\\w+/(json|local-time)$", read_format_bool), + json_path_handler("^/\\w+/(file-pattern|level-field|timestamp-field|body-field|url|title|description)$", + read_format_field), + json_path_handler("^/\\w+/level/" + "(trace|debug|info|warning|error|critical|fatal)$", read_levels), - json_path_handler("/\\w+/value/\\w+/(kind|collate|unit/field)", read_value_def), - json_path_handler("/\\w+/value/\\w+/(identifier|foreign-key)", read_value_bool), - json_path_handler("/\\w+/value/\\w+/unit/scaling-factor/.*", + json_path_handler("^/\\w+/value/\\w+/(kind|collate|unit/field)$", read_value_def), + json_path_handler("^/\\w+/value/\\w+/(identifier|foreign-key)$", read_value_bool), + json_path_handler("^/\\w+/value/\\w+/unit/scaling-factor/.*$", read_scaling), - json_path_handler("/\\w+/sample#/line", read_sample_line), + json_path_handler("^/\\w+/sample#/line$", read_sample_line), + json_path_handler("^/\\w+/line-format#/(field|default-value)$", read_json_variable), + json_path_handler("^/\\w+/line-format#/min-width$", read_json_variable_num), + json_path_handler("^/\\w+/line-format#$", read_json_constant), json_path_handler() }; diff --git a/src/log_vtab_impl.cc b/src/log_vtab_impl.cc index c851e297..069f1063 100644 --- a/src/log_vtab_impl.cc +++ b/src/log_vtab_impl.cc @@ -75,6 +75,8 @@ static string declare_table_statement(log_vtab_impl *vi) for (iter = cols.begin(); iter != cols.end(); iter++) { auto_mem coldecl; + assert(iter->vc_name != NULL); + coldecl = sqlite3_mprintf(" %Q %s collate %Q,\n", iter->vc_name, type_to_string(iter->vc_type), @@ -361,24 +363,30 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col) } size_t sub_col = col - VT_COL_MAX; + std::vector::iterator lv_iter; - if (sub_col < vc->line_values.size()) { - logline_value &lv = vc->line_values[sub_col]; + lv_iter = find_if(vc->line_values.begin(), vc->line_values.end(), + logline_value_cmp(NULL, sub_col)); - switch (lv.lv_kind) { + if (lv_iter != vc->line_values.end()) { + switch (lv_iter->lv_kind) { + case logline_value::VALUE_NULL: + sqlite3_result_null(ctx); + break; case logline_value::VALUE_TEXT: sqlite3_result_text(ctx, - lv.lv_string.c_str(), - lv.lv_string.length(), + lv_iter->lv_string.c_str(), + lv_iter->lv_string.length(), SQLITE_TRANSIENT); break; + case logline_value::VALUE_BOOLEAN: case logline_value::VALUE_INTEGER: - sqlite3_result_int64(ctx, lv.lv_number.i); + sqlite3_result_int64(ctx, lv_iter->lv_number.i); break; case logline_value::VALUE_FLOAT: - sqlite3_result_double(ctx, lv.lv_number.d); + sqlite3_result_double(ctx, lv_iter->lv_number.d); break; case logline_value::VALUE_UNKNOWN: diff --git a/src/log_vtab_impl.hh b/src/log_vtab_impl.hh index af557ffb..06cec86c 100644 --- a/src/log_vtab_impl.hh +++ b/src/log_vtab_impl.hh @@ -58,7 +58,9 @@ struct log_cursor { class log_vtab_impl { public: struct vtab_column { - vtab_column(const char *name, int type, const char *collator = NULL) + vtab_column(const char *name = NULL, + int type = SQLITE3_TEXT, + const char *collator = NULL) : vc_name(name), vc_type(type), vc_collator(collator) { }; const char *vc_name; diff --git a/src/logfile.cc b/src/logfile.cc index 978d7401..ea66d196 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -50,7 +50,7 @@ using namespace std; -static const int MAX_UNRECOGNIZED_LINES = 1000; +static const size_t MAX_UNRECOGNIZED_LINES = 1000; logfile::logfile(string filename, auto_fd fd) throw (error) @@ -141,6 +141,9 @@ void logfile::process_prefix(off_t offset, char *prefix, int len) for (iter = root_formats.begin(); iter != root_formats.end() && !found; ++iter) { + if (!(*iter)->match_name(this->lf_filename)) + continue; + (*iter)->clear(); (*iter)->lf_date_time.set_base_time(this->lf_line_buffer.get_file_time()); if ((*iter)->lf_date_time.dts_base_time == 0) { @@ -236,6 +239,9 @@ throw (line_buffer::error) * Drop the last line we read since it might have been a partial * read. */ + while (this->lf_index.back().get_sub_offset() != 0) { + this->lf_index.pop_back(); + } this->lf_index.pop_back(); } else { @@ -342,7 +348,15 @@ void logfile::read_line(logfile::iterator ll, string &line_out) line_out.clear(); if ((line = this->lf_line_buffer.read_line(off, len)) != NULL) { - line_out.append(line, len); + ostringstream stream; + + if (this->lf_format.get() != NULL) { + this->lf_format->get_subline(*ll, line, len, stream); + line_out = stream.str(); + } + else { + line_out.append(line, len); + } } else { /* XXX */ @@ -357,18 +371,19 @@ void logfile::read_full_message(logfile::iterator ll, string &msg_out, int max_lines) { - msg_out.clear(); + ostringstream stream; + do { try { off_t off = ll->get_offset(); const char *line; size_t len; - if (!msg_out.empty()) { - msg_out.append(1, '\n'); + if (stream.tellp() > 0) { + stream.write("\n", 1); } if ((line = this->lf_line_buffer.read_line(off, len)) != NULL) { - msg_out.append(line, len); + this->lf_format->get_subline(*ll, line, len, stream); } else { /* XXX */ @@ -381,4 +396,6 @@ void logfile::read_full_message(logfile::iterator ll, max_lines -= 1; } while (ll != this->end() && ll->is_continued() && (max_lines == -1 || max_lines > 0)); + + msg_out = stream.str(); } diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc index 0a620bde..16d86d3f 100644 --- a/src/logfile_sub_source.cc +++ b/src/logfile_sub_source.cc @@ -268,7 +268,7 @@ void logfile_sub_source::text_value_for_line(textview_curses &tc, { 60, "%qd%s", "s" }, { 60, "%qd%s", "m" }, { 0, "%qd%s", "h" }, - { 0, NULL } + { 0, NULL, NULL } }; struct rel_interval *curr_interval; @@ -420,6 +420,7 @@ void logfile_sub_source::text_attrs_for_line(textview_curses &lv, bool logfile_sub_source::rebuild_index(observer *obs, bool force) { std::vector::iterator iter; + size_t total_lines = 0; bool retval = force; int file_count = 0; @@ -437,6 +438,7 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force) retval = true; } file_count += 1; + total_lines += iter->ld_file->size(); } } if (force) { @@ -489,13 +491,6 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force) content_line_t con_line(file_index * MAX_LINES_PER_FILE + line_index); - if (obs != NULL) { - obs->logfile_sub_source_filtering( - *this, - content_line_t(con_line % MAX_LINES_PER_FILE), - ld->ld_file->size()); - } - if (!(lf_iter->get_level() & logline::LEVEL_CONTINUED)) { if (action_for_prev_line == logfile_filter::INCLUDE) { while (last_owner->ld_indexing.ld_start <= @@ -510,6 +505,13 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force) ld->ld_indexing.ld_start = con_line; } + if (obs != NULL) { + obs->logfile_sub_source_filtering( + *this, + vis_line_t(this->lss_index.size()), + total_lines); + } + ld->ld_indexing.ld_last = con_line; action_for_prev_line = ld->ld_file->check_filter( lf_iter, this->lss_filter_generation, this->lss_filters); @@ -518,6 +520,11 @@ bool logfile_sub_source::rebuild_index(observer *obs, bool force) merge.next(); } + if (obs != NULL) { + obs->logfile_sub_source_filtering(*this, + vis_line_t(total_lines - 1), + total_lines); + } if (action_for_prev_line == logfile_filter::INCLUDE) { while (last_owner->ld_indexing.ld_start <= diff --git a/src/logfile_sub_source.hh b/src/logfile_sub_source.hh index d966e014..7fe9635a 100644 --- a/src/logfile_sub_source.hh +++ b/src/logfile_sub_source.hh @@ -59,7 +59,7 @@ public: : public logfile_observer { public: virtual void logfile_sub_source_filtering(logfile_sub_source &lss, - content_line_t cl, + vis_line_t cl, size_t total) = 0; }; diff --git a/src/pcrepp.hh b/src/pcrepp.hh index 6d4f80ee..353fe966 100644 --- a/src/pcrepp.hh +++ b/src/pcrepp.hh @@ -224,7 +224,7 @@ public: void reset(const std::string &str, size_t off = 0) { this->reset(str.c_str(), off, str.length()); - } + }; size_t pi_offset; size_t pi_next_offset; diff --git a/src/piper_proc.cc b/src/piper_proc.cc index 28422f74..797ca6d1 100644 --- a/src/piper_proc.cc +++ b/src/piper_proc.cc @@ -141,7 +141,7 @@ piper_proc::piper_proc(int pipefd, bool timestamp, const char *filename) } woff += wrc; } - } while (lb.get_file_size() == -1); + } while (lb.get_file_size() == (size_t)-1); if (timestamp) { int wrc; diff --git a/src/readline_curses.cc b/src/readline_curses.cc index cd21542e..097e5324 100644 --- a/src/readline_curses.cc +++ b/src/readline_curses.cc @@ -683,7 +683,7 @@ void readline_curses::do_update(void) { if (this->rc_active_context == -1) { int alt_start = -1; - struct line_range lr = { 0, }; + struct line_range lr = { 0 }; attr_line_t al, alt_al; wmove(this->vc_window, this->get_actual_y(), 0); diff --git a/src/session_data.cc b/src/session_data.cc index 5eaeb071..0afa3459 100644 --- a/src/session_data.cc +++ b/src/session_data.cc @@ -201,7 +201,7 @@ static void cleanup_session_data(void) session_info_list.sort(); - int session_loops = 0; + size_t session_loops = 0; while (session_info_list.size() > MAX_SESSION_FILE_COUNT) { const session_file_info &front = session_info_list.front(); diff --git a/src/sql_util.hh b/src/sql_util.hh index 21400912..d03734f6 100644 --- a/src/sql_util.hh +++ b/src/sql_util.hh @@ -66,7 +66,7 @@ void sql_strftime(char *buffer, size_t buffer_size, time_t time, int millis); inline void sql_strftime(char *buffer, size_t buffer_size, const struct timeval &tv) { - sql_strftime(buffer, buffer_size, tv.tv_sec, tv.tv_usec); + sql_strftime(buffer, buffer_size, tv.tv_sec, tv.tv_usec / 1000); } #endif diff --git a/src/textview_curses.cc b/src/textview_curses.cc index c213ef45..9a0d8bf7 100644 --- a/src/textview_curses.cc +++ b/src/textview_curses.cc @@ -85,7 +85,9 @@ void textview_curses::grep_match(grep_proc &gp, this->tc_sub_source->text_mark(&BM_SEARCH, line, true); } - listview_curses::reload_data(); + if (this->get_top() <= line && line <= this->get_bottom()) { + listview_curses::reload_data(); + } } void textview_curses::listview_value_for_row(const listview_curses &lv, diff --git a/src/yajlpp.cc b/src/yajlpp.cc index 8ed93d35..7f958e18 100644 --- a/src/yajlpp.cc +++ b/src/yajlpp.cc @@ -36,6 +36,7 @@ int yajlpp_parse_context::map_start(void *ctx) { yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx; + int retval = 1; ypc->ypc_path_index_stack.push_back(ypc->ypc_path.length()); @@ -43,7 +44,10 @@ int yajlpp_parse_context::map_start(void *ctx) ypc->ypc_array_index.back() += 1; } - return 1; + if (ypc->ypc_alt_callbacks.yajl_start_map != NULL) + retval = ypc->ypc_alt_callbacks.yajl_start_map(ypc); + + return retval; } int yajlpp_parse_context::map_key(void *ctx, @@ -51,12 +55,16 @@ int yajlpp_parse_context::map_key(void *ctx, size_t len) { yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx; + int retval = 1; ypc->ypc_path = ypc->ypc_path.substr(0, ypc->ypc_path_index_stack.back()); ypc->ypc_path += "/" + std::string((const char *)key, len); + if (ypc->ypc_alt_callbacks.yajl_map_key != NULL) + retval = ypc->ypc_alt_callbacks.yajl_map_key(ctx, key, len); + ypc->update_callbacks(); - return 1; + return retval; } void yajlpp_parse_context::update_callbacks(void) @@ -68,6 +76,8 @@ void yajlpp_parse_context::update_callbacks(void) for (int lpc = 0; this->ypc_handlers[lpc].jph_path[0]; lpc++) { const json_path_handler &jph = this->ypc_handlers[lpc]; + pi.reset(this->ypc_path); + if (jph.jph_regex.match(this->ypc_pcre_context, pi)) { if (jph.jph_callbacks.yajl_null != NULL) this->ypc_callbacks.yajl_null = jph.jph_callbacks.yajl_null; @@ -86,44 +96,59 @@ void yajlpp_parse_context::update_callbacks(void) int yajlpp_parse_context::map_end(void *ctx) { yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx; + int retval = 1; ypc->ypc_path = ypc->ypc_path.substr(0, ypc->ypc_path_index_stack.back()); ypc->ypc_path_index_stack.pop_back(); + if (ypc->ypc_alt_callbacks.yajl_end_map != NULL) + retval = ypc->ypc_alt_callbacks.yajl_end_map(ctx); + ypc->update_callbacks(); - return 1; + return retval; } int yajlpp_parse_context::array_start(void *ctx) { yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx; + int retval = 1; ypc->ypc_path_index_stack.push_back(ypc->ypc_path.length()); ypc->ypc_path += "#"; ypc->ypc_array_index.push_back(-1); + if (ypc->ypc_alt_callbacks.yajl_start_array != NULL) + retval = ypc->ypc_alt_callbacks.yajl_start_array(ctx); + ypc->update_callbacks(); - return 1; + return retval; } int yajlpp_parse_context::array_end(void *ctx) { yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx; + int retval = 1; ypc->ypc_path = ypc->ypc_path.substr(0, ypc->ypc_path_index_stack.back()); ypc->ypc_path_index_stack.pop_back(); ypc->ypc_array_index.pop_back(); + if (ypc->ypc_alt_callbacks.yajl_end_array != NULL) + retval = ypc->ypc_alt_callbacks.yajl_end_array(ctx); + ypc->update_callbacks(); - return 1; + return retval; } int yajlpp_parse_context::handle_unused(void *ctx) { yajlpp_parse_context *ypc = (yajlpp_parse_context *)ctx; + if (ypc->ypc_ignore_unused) + return 1; + fprintf(stderr, "warning:%s:%s:unexpected data, expecting one of the following data types --\n", ypc->ypc_source.c_str(), ypc->ypc_path.c_str()); diff --git a/src/yajlpp.hh b/src/yajlpp.hh index 79ea4c80..e6c42f44 100644 --- a/src/yajlpp.hh +++ b/src/yajlpp.hh @@ -105,7 +105,39 @@ struct json_path_handler : public json_path_handler_base { this->jph_callbacks.yajl_string = (int (*)(void *, const unsigned char *, size_t))str_func; } + json_path_handler(const char *path) : json_path_handler_base(path) { }; + json_path_handler() : json_path_handler_base("") {}; + + json_path_handler &add_cb(int(*null_func)(yajlpp_parse_context *)) { + this->jph_callbacks.yajl_null = (int (*)(void *))null_func; + return *this; + }; + + json_path_handler &add_cb(int(*bool_func)(yajlpp_parse_context *, int)) + { + this->jph_callbacks.yajl_boolean = (int (*)(void *, int))bool_func; + return *this; + } + + json_path_handler &add_cb(int(*int_func)(yajlpp_parse_context *, long long)) + { + this->jph_callbacks.yajl_integer = (int (*)(void *, long long))int_func; + return *this; + } + + json_path_handler &add_cb(int(*double_func)(yajlpp_parse_context *, double)) + { + this->jph_callbacks.yajl_double = (int (*)(void *, double))double_func; + return *this; + } + + json_path_handler &add_cb(int(*str_func)(yajlpp_parse_context *, const unsigned char *, size_t)) + { + this->jph_callbacks.yajl_string = (int (*)(void *, const unsigned char *, size_t))str_func; + return *this; + } + }; class yajlpp_parse_context { @@ -129,9 +161,10 @@ public: yajlpp_parse_context(std::string source, struct json_path_handler *handlers) - : ypc_source(source), ypc_handlers(handlers) + : ypc_source(source), ypc_handlers(handlers), ypc_ignore_unused(false) { this->ypc_callbacks = DEFAULT_CALLBACKS; + memset(&this->ypc_alt_callbacks, 0, sizeof(this->ypc_alt_callbacks)); }; std::string get_path_fragment(int offset) const @@ -155,10 +188,12 @@ public: struct json_path_handler *ypc_handlers; void * ypc_userdata; yajl_callbacks ypc_callbacks; + yajl_callbacks ypc_alt_callbacks; std::string ypc_path; std::vector ypc_path_index_stack; std::vector ypc_array_index; pcre_context_static<30> ypc_pcre_context; + bool ypc_ignore_unused; private: static const yajl_callbacks DEFAULT_CALLBACKS; diff --git a/test/Makefile.am b/test/Makefile.am index b0a3fd3f..63aae3cd 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -82,7 +82,13 @@ drive_listview_SOURCES = drive_listview.cc drive_listview_LDADD = ../src/libdiag.a $(CURSES_LIB) -lz drive_logfile_SOURCES = drive_logfile.cc -drive_logfile_LDADD = ../src/libdiag.a -lcrypto $(CURSES_LIB) $(SQLITE3_LIBS) +drive_logfile_LDADD = \ + ../src/libdiag.a \ + -lcrypto \ + $(CURSES_LIB) \ + $(SQLITE3_LIBS) \ + $(PCRE_LIBS) \ + -lpcrecpp drive_sequencer_SOURCES = drive_sequencer.cc drive_sequencer_LDADD = ../src/libdiag.a -lcrypto $(CURSES_LIB) $(SQLITE3_LIBS) @@ -93,7 +99,9 @@ drive_data_scanner_LDADD = \ ../src/libdiag.a \ ../src/default-log-formats-json.o \ -lcrypto \ + $(PCRE_LIBS) \ $(SQLITE3_LIBS) \ + -lpcrecpp \ $(CURSES_LIB) drive_view_colors_SOURCES = drive_view_colors.cc diff --git a/test/Makefile.in b/test/Makefile.in index 5749d081..290cd650 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -120,7 +120,7 @@ drive_data_scanner_OBJECTS = $(am_drive_data_scanner_OBJECTS) am__DEPENDENCIES_1 = drive_data_scanner_DEPENDENCIES = ../src/libdiag.a \ ../src/default-log-formats-json.o $(am__DEPENDENCIES_1) \ - $(am__DEPENDENCIES_1) + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) am_drive_grep_proc_OBJECTS = drive_grep_proc.$(OBJEXT) drive_grep_proc_OBJECTS = $(am_drive_grep_proc_OBJECTS) drive_grep_proc_DEPENDENCIES = ../src/libdiag.a $(am__DEPENDENCIES_1) @@ -134,7 +134,7 @@ drive_listview_DEPENDENCIES = ../src/libdiag.a $(am__DEPENDENCIES_1) am_drive_logfile_OBJECTS = drive_logfile.$(OBJEXT) drive_logfile_OBJECTS = $(am_drive_logfile_OBJECTS) drive_logfile_DEPENDENCIES = ../src/libdiag.a $(am__DEPENDENCIES_1) \ - $(am__DEPENDENCIES_1) + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) am_drive_readline_curses_OBJECTS = drive_readline_curses.$(OBJEXT) drive_readline_curses_OBJECTS = $(am_drive_readline_curses_OBJECTS) drive_readline_curses_DEPENDENCIES = ../src/libdiag.a \ @@ -639,7 +639,14 @@ drive_grep_proc_LDADD = ../src/libdiag.a $(PCRE_LIBS) -lz drive_listview_SOURCES = drive_listview.cc drive_listview_LDADD = ../src/libdiag.a $(CURSES_LIB) -lz drive_logfile_SOURCES = drive_logfile.cc -drive_logfile_LDADD = ../src/libdiag.a -lcrypto $(CURSES_LIB) $(SQLITE3_LIBS) +drive_logfile_LDADD = \ + ../src/libdiag.a \ + -lcrypto \ + $(CURSES_LIB) \ + $(SQLITE3_LIBS) \ + $(PCRE_LIBS) \ + -lpcrecpp + drive_sequencer_SOURCES = drive_sequencer.cc drive_sequencer_LDADD = ../src/libdiag.a -lcrypto $(CURSES_LIB) $(SQLITE3_LIBS) drive_data_scanner_SOURCES = \ @@ -649,7 +656,9 @@ drive_data_scanner_LDADD = \ ../src/libdiag.a \ ../src/default-log-formats-json.o \ -lcrypto \ + $(PCRE_LIBS) \ $(SQLITE3_LIBS) \ + -lpcrecpp \ $(CURSES_LIB) drive_view_colors_SOURCES = drive_view_colors.cc