Archives/lnav
2
0
Fork 0
mirror of https://github.com/tstack/lnav synced 2024-11-17 15:29:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

[sql] add a log_search table

Browse Source
This commit is contained in:
Timothy Stack 2015-08-05 23:18:19 -07:00
parent 6f26aa7f3e
commit 2b5447f59c
14 changed files with 355 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
NEWS
View File

@ -4,7 +4,10 @@ lnav v0.8.0:
* Integration with "papertrailapp.com" for querying and tailing
server log and syslog messages. See the Papertrail section in
the online help for more details.
* Remote files can be opened when linked with libcurl.
* Remote files can be opened when lnav is built with libcurl.
* SQL queries can now be done on lines that match a regular expression
using the "log_search" table or by creating custom tables with the
":create-search-table" command.
* Log formats that are "containers" for other log formats, like
syslog, are now supported. See the online help for more
information.

17
docs/source/commands.rst
View File

@ -63,6 +63,23 @@ Display
* redraw - Redraw the window to correct any corruption.
SQL
---
* create-logline-table <table-name> - Create an SQL table using the top line
of the log view as a template. See the :ref:`data-ext` section for more information.
* delete-logline-table <table-name> - Delete a table created by create-logline-table.
* create-search-table <table-name> [regex] - Create an SQL table that
extracts information from logs using the provided regular expression or the
last search that was done. Any captures in the expression will be used as
columns in the SQL table. If the capture is named, that name will be used as
the column name, otherwise the column name will be of the form 'col_N'.
* delete-search-table <table-name> - Delete a table that was created with create-search-table.
Output
------

1
src/CMakeLists.txt
View File

@ -96,6 +96,7 @@ set(diag_STAT_SRCS
log_data_helper.hh
log_data_table.hh
log_format_impls.cc
log_search_table.hh
logfile_stats.hh
plain_text_source.hh
pretty_printer.hh

1
src/Makefile.am
View File

@ -137,6 +137,7 @@ noinst_HEADERS = \
log_data_table.hh \
log_format.hh \
log_format_loader.hh \
log_search_table.hh \
logfile.hh \
logfile_sub_source.hh \
papertrail_proc.hh \

11
src/help.txt
View File

@ -487,6 +487,17 @@ COMMANDS
Delete an SQL table created by the 'create-logline-table'
command.
create-search-table <table-name> [<regex>]
Create an SQL table that extracts information from logs
using the provided regular expression or the last search
that was done. Any captures in the expression will be
used as columns in the SQL table. If the capture is named,
that name will be used as the column name, otherwise the
column name will be of the form 'col_N'.
delete-search-table <table-name>
Delete a table that was created with create-search-table.
switch-to-view <view-name>
Switch the display to the given view, which can be one of:
help, log, text, histogram, db, and schema.

13
src/lnav.cc
View File

@ -132,6 +132,7 @@
#include "readline_possibilities.hh"
#include "field_overlay_source.hh"
#include "url_loader.hh"
#include "log_search_table.hh"
using namespace std;
@ -936,10 +937,10 @@ void execute_search(lnav_view_t view, const std::string &regex_orig)
auto_ptr<grep_highlighter> &gc = lnav_data.ld_search_child[view];
textview_curses & tc = lnav_data.ld_views[view];
std::string regex = regex_orig;
pcre * code = NULL;
if ((gc.get() == NULL) || (regex != lnav_data.ld_last_search[view])) {
const char *errptr;
pcre * code = NULL;
int eoff;
bool quoted = false;
@ -1004,6 +1005,16 @@ void execute_search(lnav_view_t view, const std::string &regex_orig)
new grep_highlighter(gp, "$search", hm));
gc = gh;
}
if (view == LNV_LOG) {
static intern_string_t log_search_name = intern_string::lookup("log_search");
lnav_data.ld_vtab_manager->unregister_vtab(log_search_name);
if (code != NULL) {
lnav_data.ld_vtab_manager->register_vtab(new log_search_table(
regex.c_str(), log_search_name));
}
}
}
lnav_data.ld_last_search[view] = regex;

100
src/lnav_commands.cc
View File

@ -49,6 +49,7 @@
#include "command_executor.hh"
#include "url_loader.hh"
#include "readline_curses.hh"
#include "log_search_table.hh"
using namespace std;
@ -1014,7 +1015,7 @@ static string com_disable_word_wrap(string cmdline, vector<string> &args)
return retval;
}
static std::vector<string> custom_logline_tables;
static std::set<string> custom_logline_tables;
static string com_create_logline_table(string cmdline, vector<string> &args)
{
@ -1035,6 +1036,7 @@ static string com_create_logline_table(string cmdline, vector<string> &args)
errmsg = lnav_data.ld_vtab_manager->register_vtab(ldt);
if (errmsg.empty()) {
custom_logline_tables.insert(args[1]);
if (lnav_data.ld_rl_view != NULL) {
lnav_data.ld_rl_view->add_possibility(LNM_COMMAND,
"custom-table",
@ -1043,6 +1045,7 @@ static string com_create_logline_table(string cmdline, vector<string> &args)
retval = "info: created new log table -- " + args[1];
}
else {
delete ldt;
retval = "error: unable to create table -- " + errmsg;
}
}
@ -1059,6 +1062,10 @@ static string com_delete_logline_table(string cmdline, vector<string> &args)
args.push_back("custom-table");
}
else if (args.size() == 2) {
if (custom_logline_tables.find(args[1]) == custom_logline_tables.end()) {
return "error: unknown logline table -- " + args[1];
}
string rc = lnav_data.ld_vtab_manager->unregister_vtab(
intern_string::lookup(args[1]));
@ -1078,6 +1085,85 @@ static string com_delete_logline_table(string cmdline, vector<string> &args)
return retval;
}
static std::set<string> custom_search_tables;
static string com_create_search_table(string cmdline, vector<string> &args)
{
string retval = "error: expecting a table name";
if (args.size() == 0) {
}
else if (args.size() >= 2) {
log_search_table *lst;
string regex;
if (args.size() >= 3) {
regex = cmdline.substr(cmdline.find(args[2], args[0].size() + args[1].size()));
}
else {
regex = lnav_data.ld_last_search[LNV_LOG];
}
try {
lst = new log_search_table(regex.c_str(),
intern_string::lookup(args[1]));
} catch (pcrepp::error &e) {
return "error: unable to compile regex -- " + regex;
}
string errmsg;
errmsg = lnav_data.ld_vtab_manager->register_vtab(lst);
if (errmsg.empty()) {
custom_search_tables.insert(args[1]);
if (lnav_data.ld_rl_view != NULL) {
lnav_data.ld_rl_view->add_possibility(LNM_COMMAND,
"search-table",
args[1]);
}
retval = "info: created new search table -- " + args[1];
}
else {
delete lst;
retval = "error: unable to create table -- " + errmsg;
}
}
return retval;
}
static string com_delete_search_table(string cmdline, vector<string> &args)
{
string retval = "error: expecting a table name";
if (args.size() == 0) {
args.push_back("search-table");
}
else if (args.size() == 2) {
if (custom_search_tables.find(args[1]) == custom_search_tables.end()) {
return "error: unknown search table -- " + args[1];
}
string rc = lnav_data.ld_vtab_manager->unregister_vtab(
intern_string::lookup(args[1]));
if (rc.empty()) {
if (lnav_data.ld_rl_view != NULL) {
lnav_data.ld_rl_view->rem_possibility(LNM_COMMAND,
"search-table",
args[1]);
}
retval = "info: deleted search table";
}
else {
retval = "error: " + rc;
}
}
return retval;
}
static string com_session(string cmdline, vector<string> &args)
{
string retval = "error: expecting a command to save to the session file";
@ -1989,6 +2075,18 @@ readline_context::command_t STD_COMMANDS[] = {
"Delete a table created with create-logline-table",
com_delete_logline_table,
},
{
"create-search-table",
"<table-name> [<regex>]",
"Create an SQL table based on a regex search",
com_create_search_table,
},
{
"delete-search-table",
"<table-name>",
"Delete a table created with create-search-table",
com_delete_search_table,
},
{
"open",
"<filename>",

3
src/log_data_table.hh
View File

@ -27,9 +27,6 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file log_data_table.hh
*
* XXX This file has become a dumping ground for code and needs to be broken up
* a bit.
*/
#ifndef _log_data_table_hh

132
src/log_search_table.hh Normal file
View File

@ -0,0 +1,132 @@
/**
* Copyright (c) 2015, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file log_search_table.hh
*/
#ifndef _log_search_table_hh
#define _log_search_table_hh
#include <string>
#include <vector>
#include "lnav.hh"
#include "logfile.hh"
#include "data_parser.hh"
#include "column_namer.hh"
#include "log_vtab_impl.hh"
class log_search_table : public log_vtab_impl {
public:
log_search_table(const char *regex, intern_string_t table_name)
: log_vtab_impl(table_name),
lst_regex(regex) {
};
void get_columns(std::vector<vtab_column> &cols)
{
column_namer cn;
for (int lpc = 0; lpc < this->lst_regex.get_capture_count(); lpc++) {
// TODO: it would be nice to figure out the type and collator
// We could test the regex with some canned data to figure out
// what it will and will not accept. That way we don't have to
// check actual log data.
cols.push_back(vtab_column(cn.add_column(
this->lst_regex.name_for_capture(lpc))));
}
};
bool next(log_cursor &lc, logfile_sub_source &lss)
{
lc.lc_curr_line = lc.lc_curr_line + vis_line_t(1);
lc.lc_sub_index = 0;
if (lc.lc_curr_line == (int)lss.text_line_count()) {
return true;
}
content_line_t cl;
cl = lss.at(lc.lc_curr_line);
logfile * lf = lss.find(cl);
logfile::iterator lf_iter = lf->begin() + cl;
if (lf_iter->is_continued()) {
return false;
}
string_attrs_t sa;
std::vector<logline_value> line_values;
lf->read_full_message(lf_iter, this->lst_current_line);
lf->get_format()->annotate(this->lst_current_line, sa, line_values);
this->lst_body = find_string_attr_range(sa, &textview_curses::SA_BODY);
if (this->lst_body.lr_end == -1 || this->lst_body.length() == 0) {
return false;
}
pcre_input pi(&this->lst_current_line.get_data()[this->lst_body.lr_start],
0,
this->lst_body.length());
return this->lst_regex.match(this->lst_match_context, pi);
};
void extract(logfile *lf,
shared_buffer_ref &line,
std::vector<logline_value> &values)
{
pcre_input pi(&this->lst_current_line.get_data()[this->lst_body.lr_start],
0,
this->lst_body.length());
int next_column = 0;
for (int lpc = 0; lpc < this->lst_regex.get_capture_count(); lpc++) {
pcre_context::capture_t *cap = this->lst_match_context[lpc];
shared_buffer_ref value_sbr;
value_sbr.subset(line,
this->lst_body.lr_start + cap->c_begin,
cap->length());
values.push_back(logline_value(intern_string::lookup("", 0),
logline_value::VALUE_TEXT,
value_sbr));
values.back().lv_column = next_column++;
}
};
private:
pcrepp lst_regex;
shared_buffer_ref lst_current_line;
struct line_range lst_body;
pcre_context_static<128> lst_match_context;
};
#endif

12
src/log_vtab_impl.cc
View File

@ -78,9 +78,7 @@ std::string log_vtab_impl::get_table_statement(void)
auto_mem<char, sqlite3_free> coldecl;
auto_mem<char, sqlite3_free> colname;
require(iter->vc_name != NULL);
colname = sql_quote_ident(iter->vc_name);
colname = sql_quote_ident(iter->vc_name.c_str());
coldecl = sqlite3_mprintf(" %s %s %s collate %Q,\n",
colname.in(),
type_to_string(iter->vc_type),
@ -671,7 +669,7 @@ string log_vtab_manager::register_vtab(log_vtab_impl *vi)
if (this->vm_impls.find(vi->get_name()) == this->vm_impls.end()) {
auto_mem<char> errmsg(sqlite3_free);
char * sql;
auto_mem<char> sql(sqlite3_free);
int rc;
this->vm_impls[vi->get_name()] = vi;
@ -688,8 +686,6 @@ string log_vtab_manager::register_vtab(log_vtab_impl *vi)
if (rc != SQLITE_OK) {
retval = errmsg;
}
sqlite3_free(sql);
}
else {
retval = "a table with the given name already exists";
@ -706,7 +702,7 @@ string log_vtab_manager::unregister_vtab(intern_string_t name)
retval = "unknown log line table -- " + name.to_string();
}
else {
char *sql;
auto_mem<char> sql(sqlite3_free);
int rc;
sql = sqlite3_mprintf("DROP TABLE %s ", name.get());
@ -717,8 +713,6 @@ string log_vtab_manager::unregister_vtab(intern_string_t name)
NULL);
require(rc == SQLITE_OK);
sqlite3_free(sql);
this->vm_impls.erase(name);
}

4
src/log_vtab_impl.hh
View File

@ -70,13 +70,13 @@ struct log_cursor {
class log_vtab_impl {
public:
struct vtab_column {
vtab_column(const char *name = NULL,
vtab_column(const std::string name = "",
int type = SQLITE3_TEXT,
const char *collator = NULL,
bool hidden = false)
: vc_name(name), vc_type(type), vc_collator(collator), vc_hidden(hidden) { };
const char *vc_name;
std::string vc_name;
int vc_type;
const char *vc_collator;
bool vc_hidden;

23
src/pcrepp.hh
View File

@ -380,12 +380,28 @@ public:
int name_index(const char *name) const {
int retval = pcre_get_stringnumber(this->p_code, name);
if (retval == PCRE_ERROR_NOSUBSTRING)
if (retval == PCRE_ERROR_NOSUBSTRING) {
return retval;
}
return retval - 1;
};
const char *name_for_capture(int index) {
for (pcre_named_capture::iterator iter = this->named_begin();
iter != this->named_end();
++iter) {
if (iter->index() == index) {
return iter->pnc_name;
}
}
return "";
};
int get_capture_count() const {
return this->p_capture_count;
};
bool match(pcre_context &pc, pcre_input &pi, int options = 0) const
{
int length, startoffset, filtered_options = options;
@ -511,6 +527,10 @@ private:
pcre_assign_jit_stack(extra, NULL, jit_stack());
#endif
}
pcre_fullinfo(this->p_code,
this->p_code_extra,
PCRE_INFO_CAPTURECOUNT,
&this->p_capture_count);
pcre_fullinfo(this->p_code,
this->p_code_extra,
PCRE_INFO_NAMECOUNT,
@ -527,6 +547,7 @@ private:
pcre *p_code;
auto_mem<pcre_extra> p_code_extra;
int p_capture_count;
int p_named_count;
int p_name_len;
pcre_named_capture *p_named_entries;

1
src/readline_callbacks.cc
View File

@ -34,6 +34,7 @@
#include "plain_text_source.hh"
#include "command_executor.hh"
#include "readline_curses.hh"
#include "log_search_table.hh"
using namespace std;

50
test/test_sql.sh
View File

@ -519,3 +519,53 @@ check_output "multiline data is not right?" <<EOF
}
]
EOF
run_test ${lnav_test} -n \
-c ":create-search-table search_test1 (\w+), World!" \
-c ";select col_0 from search_test1" \
-c ":write-csv-to -" \
${test_dir}/logfile_multiline.0
check_output "create-search-table is not working?" <<EOF
col_0
Hello
Goodbye
EOF
run_test ${lnav_test} -n \
-c ":create-search-table search_test1 (?<word>\w+), World!" \
-c ";select word from search_test1" \
-c ":write-csv-to -" \
${test_dir}/logfile_multiline.0
check_output "create-search-table is not working?" <<EOF
word
Hello
Goodbye
EOF
run_test ${lnav_test} -n \
-c ":delete-search-table search_test1" \
${test_dir}/logfile_multiline.0
check_error_output "able to delete unknown table?" <<EOF
error: unknown search table -- search_test1
EOF
run_test ${lnav_test} -n \
-c ":create-logline-table search_test1" \
-c ":delete-search-table search_test1" \
${test_dir}/logfile_multiline.0
check_error_output "able to delete logline table?" <<EOF
error: unknown search table -- search_test1
EOF
run_test ${lnav_test} -n \
-c ":create-search-table search_test1 bad(" \
${test_dir}/logfile_multiline.0
check_error_output "able to create table with a bad regex?" <<EOF
error: unable to compile regex -- bad(
EOF