[cleanup] remove unused stuff and add a test

pull/824/merge
Timothy Stack 2 years ago
parent 317f2d32cf
commit b3ee092dc5

@ -321,7 +321,6 @@ add_library(
data_scanner.cc
data_scanner_re.cc
data_parser.cc
papertrail_proc.cc
pcap_manager.cc
plain_text_source.cc
pretty_printer.cc
@ -429,7 +428,6 @@ add_library(
md2attr_line.hh
md4cpp.hh
optional.hpp
papertrail_proc.hh
pcap_manager.hh
plain_text_source.hh
pretty_printer.hh

@ -235,7 +235,6 @@ noinst_HEADERS = \
md2attr_line.hh \
md4cpp.hh \
optional.hpp \
papertrail_proc.hh \
pcap_manager.hh \
piper_proc.hh \
plain_text_source.hh \
@ -405,7 +404,6 @@ libdiag_a_SOURCES = \
md4cpp.cc \
network-extension-functions.cc \
data_parser.cc \
papertrail_proc.cc \
pcap_manager.cc \
plain_text_source.cc \
pollable.cc \

@ -44,7 +44,6 @@
#include "lnav_config.hh"
#include "lnav_util.hh"
#include "log_format_loader.hh"
#include "papertrail_proc.hh"
#include "readline_highlighters.hh"
#include "service_tags.hh"
#include "shlex.hh"
@ -284,6 +283,18 @@ bind_sql_parameters(exec_context& ec, sqlite3_stmt* stmt)
return Ok(retval);
}
static void
execute_search(const std::string& search_cmd)
{
lnav_data.ld_view_stack.top() | [&search_cmd](auto tc) {
auto search_term
= string_fragment(search_cmd)
.find_right_boundary(0, string_fragment::tag1{'\n'})
.to_string();
tc->execute_search(search_term);
};
}
Result<std::string, lnav::console::user_message>
execute_sql(exec_context& ec, const std::string& sql, std::string& alt_msg)
{
@ -703,8 +714,7 @@ execute_from_file(exec_context& ec,
retval = TRY(execute_command(ec, cmdline.substr(1)));
break;
case '/':
lnav_data.ld_view_stack.top() |
[cmdline](auto tc) { tc->execute_search(cmdline.substr(1)); };
execute_search(cmdline.substr(1));
break;
case ';':
setup_logline_table(ec);
@ -746,8 +756,7 @@ execute_any(exec_context& ec, const std::string& cmdline_with_mode)
retval = TRY(execute_command(ec, cmdline));
break;
case '/':
lnav_data.ld_view_stack.top() |
[cmdline](auto tc) { tc->execute_search(cmdline.substr(1)); };
execute_search(cmdline);
break;
case ';':
setup_logline_table(ec);
@ -795,8 +804,7 @@ execute_init_commands(
alt_msg);
break;
case '/':
lnav_data.ld_view_stack.top() |
[cmd](auto tc) { tc->execute_search(cmd.substr(1)); };
execute_search(cmd.substr(1));
break;
case ';':
setup_logline_table(ec);
@ -813,20 +821,6 @@ execute_init_commands(
}
}
lnav_data.ld_commands.clear();
if (!lnav_data.ld_pt_search.empty()) {
#ifdef HAVE_LIBCURL
auto pt = std::make_shared<papertrail_proc>(
lnav_data.ld_pt_search.substr(3),
lnav_data.ld_pt_min_time,
lnav_data.ld_pt_max_time);
lnav_data.ld_active_files.fc_file_names[lnav_data.ld_pt_search].with_fd(
pt->copy_fd());
isc::to<curl_looper&, services::curl_streamer_t>().send(
[pt](auto& clooper) { clooper.add_request(pt); });
#endif
}
if (dls.dls_rows.size() > 1) {
ensure_view(&lnav_data.ld_views[LNV_DB]);
}

@ -3,7 +3,7 @@
"vmw_log": {
"title": "VMware Logs",
"description": "One of the log formats used in VMware's ESXi and vCenter software.",
"url": "http://kb.vmware.com/kb/2004201",
"url": "https://kb.vmware.com/kb/2004201",
"regex": {
"6.0+": {
"pattern": "^(?:\\[#\\d+\\] )?(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?<level>\\w+)(?:\\(\\d+\\)+)? (?<prc>[\\w\\-]+)\\[(?<tid>\\w+)\\]:? (?:\\w+ -\\[\\d+\\] )?\\[(?<src>\\w+@\\d+)(?:\\s+sub=(?<sub>.*?(?!\\w+=)))?(?:\\s+item=(?<item>[\\w\\.\\-@/:]+))?(?: req=(?<req>[^ \\]]+))?(?: opI(?:D|d)=(?<opid>(?:req=)?[\\w@ \\-\\.:]+?(?!\\w+=)))?(?: sid=(?<sid>[^ \\]]+))?(?: user=(?<user>[^ \\]<]+(?:<[^>]+>)?))?(?: update=(?<vpxa_update>\\d+))?(?:\\s+reason=(?<reason>[^\\]]+))?\\]\\s*(?<body>.*)$"

@ -995,28 +995,6 @@
----
.. _pt_max_time:
:pt-max-time
^^^^^^^^^^^^
(null)
----
.. _pt_min_time:
:pt-min-time
^^^^^^^^^^^^
(null)
----
.. _quit:
:quit

@ -59,6 +59,7 @@
static const ssize_t INITIAL_REQUEST_SIZE = 16 * 1024;
static const ssize_t DEFAULT_INCREMENT = 128 * 1024;
static const ssize_t INITIAL_COMPRESSED_BUFFER_SIZE = 5 * 1024 * 1024;
static const ssize_t MAX_COMPRESSED_BUFFER_SIZE = 32 * 1024 * 1024;
class io_looper : public isc::service<io_looper> {};
@ -376,6 +377,7 @@ line_buffer::set_fd(auto_fd& fd)
}
this->lb_compressed_offset
= lseek(this->lb_fd, 0, SEEK_CUR);
this->resize_buffer(INITIAL_COMPRESSED_BUFFER_SIZE);
}
#ifdef HAVE_BZLIB_H
else if (gz_id[0] == 'B' && gz_id[1] == 'Z')
@ -390,7 +392,7 @@ line_buffer::set_fd(auto_fd& fd)
* Loading data from a bzip2 file is pretty slow, so we try
* to keep as much in memory as possible.
*/
this->resize_buffer(MAX_COMPRESSED_BUFFER_SIZE);
this->resize_buffer(INITIAL_COMPRESSED_BUFFER_SIZE);
this->lb_compressed_offset = 0;
}
@ -512,7 +514,7 @@ line_buffer::load_next_buffer()
{
rc = 0;
} else {
// log_debug("decomp start");
// log_debug("async decomp start");
rc = gi->read(this->lb_alt_buffer.value().end(),
start + this->lb_alt_buffer.value().size(),
this->lb_alt_buffer.value().available());
@ -524,7 +526,12 @@ line_buffer::load_next_buffer()
this->lb_file_size
= (start + this->lb_alt_buffer.value().size() + rc);
}
// log_debug("decomp end");
#if 0
log_debug("async decomp end %d+%d:%d",
this->lb_alt_buffer->size(),
rc,
this->lb_alt_buffer->capacity());
#endif
}
}
#ifdef HAVE_BZLIB_H
@ -779,7 +786,12 @@ line_buffer::fill_range(file_off_t start, ssize_t max_length)
= (this->lb_file_offset + this->lb_buffer.size() + rc);
}
}
// log_debug("old decomp end");
#if 0
log_debug("old decomp end -- %d+%d:%d",
this->lb_buffer.size(),
rc,
this->lb_buffer.capacity());
#endif
}
#ifdef HAVE_BZLIB_H
else if (this->lb_bz_file)

@ -2602,13 +2602,6 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
if (file_path == "-") {
load_stdin = true;
} else if (startswith(file_path, "pt:")) {
#ifdef HAVE_LIBCURL
lnav_data.ld_pt_search = file_path;
#else
fprintf(stderr, "error: lnav is not compiled with libcurl\n");
retval = EXIT_FAILURE;
#endif
}
#ifdef HAVE_LIBCURL
else if (is_url(file_path))
@ -2821,7 +2814,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
if (retval == EXIT_SUCCESS
&& lnav_data.ld_active_files.fc_file_names.empty()
&& lnav_data.ld_commands.empty() && lnav_data.ld_pt_search.empty()
&& lnav_data.ld_commands.empty()
&& !(lnav_data.ld_show_help_view || mode_flags.mf_no_default))
{
lnav::console::print(

@ -172,9 +172,6 @@ struct lnav_data_t {
file_collection ld_active_files;
std::list<child_poller> ld_child_pollers;
std::list<std::pair<std::string, int>> ld_files_to_front;
std::string ld_pt_search;
time_t ld_pt_min_time;
time_t ld_pt_max_time;
bool ld_stdout_used;
sig_atomic_t ld_looping;
sig_atomic_t ld_winched;

@ -65,7 +65,6 @@
#include "log_data_helper.hh"
#include "log_data_table.hh"
#include "log_search_table.hh"
#include "papertrail_proc.hh"
#include "readline_callbacks.hh"
#include "readline_curses.hh"
#include "readline_highlighters.hh"
@ -170,47 +169,6 @@ combined_user_marks(vis_bookmarks& vb)
return retval;
}
static std::string
refresh_pt_search()
{
std::string retval;
if (!lnav_data.ld_cmd_init_done) {
return "";
}
#ifdef HAVE_LIBCURL
for (const auto& lf : lnav_data.ld_active_files.fc_files) {
if (startswith(lf->get_filename(), "pt:")) {
lf->close();
}
}
isc::to<curl_looper&, services::curl_streamer_t>().send(
[](auto& clooper) { clooper.close_request("papertrailapp.com"); });
if (lnav_data.ld_pt_search.empty()) {
return "info: no papertrail query is active";
}
auto pt
= std::make_shared<papertrail_proc>(lnav_data.ld_pt_search.substr(3),
lnav_data.ld_pt_min_time,
lnav_data.ld_pt_max_time);
lnav_data.ld_active_files.fc_file_names[lnav_data.ld_pt_search].with_fd(
pt->copy_fd());
isc::to<curl_looper&, services::curl_streamer_t>().send(
[pt](auto& clooper) { clooper.add_request(pt); });
ensure_view(&lnav_data.ld_views[LNV_LOG]);
retval = "info: opened papertrail query";
#else
retval = "error: lnav not compiled with libcurl";
#endif
return retval;
}
static Result<std::string, lnav::console::user_message>
com_adjust_log_time(exec_context& ec,
std::string cmdline,
@ -2412,15 +2370,6 @@ com_open(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
for (auto fn : split_args) {
int top = 0;
if (startswith(fn, "pt:")) {
if (!ec.ec_dry_run) {
lnav_data.ld_pt_search = fn;
refresh_pt_search();
}
continue;
}
if (access(fn.c_str(), R_OK) != 0
&& (colon_index = fn.rfind(':')) != std::string::npos)
{
@ -3248,80 +3197,6 @@ com_clear_partition(exec_context& ec,
return Ok(retval);
}
static Result<std::string, lnav::console::user_message>
com_pt_time(exec_context& ec,
std::string cmdline,
std::vector<std::string>& args)
{
std::string retval;
if (args.empty()) {
args.emplace_back("move-time");
retval = "";
} else if (args.size() == 1) {
char ftime[64];
if (args[0] == "pt-min-time") {
if (lnav_data.ld_pt_min_time == 0) {
retval
= "info: minimum time is not set, pass a time value to "
"this command to set it";
} else {
ctime_r(&lnav_data.ld_pt_min_time, ftime);
retval
= "info: papertrail minimum time is " + std::string(ftime);
}
}
if (args[0] == "pt-max-time") {
if (lnav_data.ld_pt_max_time == 0) {
retval
= "info: maximum time is not set, pass a time value to "
"this command to set it";
} else {
ctime_r(&lnav_data.ld_pt_max_time, ftime);
retval
= "info: papertrail maximum time is " + std::string(ftime);
}
}
} else if (args.size() >= 2) {
std::string all_args = remaining_args(cmdline, args);
struct timeval new_time = {0, 0};
date_time_scanner dts;
struct exttm tm;
time_t now;
struct tm base_tm;
auto parse_res = relative_time::from_str(all_args);
time(&now);
localtime_r(&now, &base_tm);
dts.dts_keep_base_tz = true;
dts.set_base_time(now, base_tm);
if (parse_res.isOk()) {
tm.et_tm = *gmtime(&now);
tm = parse_res.unwrap().adjust(tm);
new_time.tv_sec = timegm(&tm.et_tm);
} else {
dts.scan(args[1].c_str(), args[1].size(), nullptr, &tm, new_time);
}
if (ec.ec_dry_run) {
retval = "";
} else if (new_time.tv_sec != 0) {
if (args[0] == "pt-min-time") {
lnav_data.ld_pt_min_time = new_time.tv_sec;
retval = refresh_pt_search();
}
if (args[0] == "pt-max-time") {
lnav_data.ld_pt_max_time = new_time.tv_sec;
retval = refresh_pt_search();
}
}
} else {
return ec.make_error("expecting a time value");
}
return Ok(retval);
}
static Result<std::string, lnav::console::user_message>
com_summarize(exec_context& ec,
std::string cmdline,
@ -3814,10 +3689,8 @@ com_set_min_log_level(exec_context& ec,
} else if (ec.ec_dry_run) {
retval = "";
} else if (args.size() == 2) {
logfile_sub_source& lss = lnav_data.ld_log_source;
log_level_t new_level;
new_level = string2level(args[1].c_str(), args[1].size(), false);
auto& lss = lnav_data.ld_log_source;
auto new_level = string2level(args[1].c_str(), args[1].size(), false);
lss.set_min_log_level(new_level);
retval = ("info: minimum log level is now -- "
@ -5518,18 +5391,6 @@ readline_context::command_t STD_COMMANDS[] = {
help_text(":clear-partition")
.with_summary("Clear the partition the top line is a part of")
.with_opposites({"partition-name"})},
{
"pt-min-time",
com_pt_time,
help_text(":pt-min-time"),
},
{
"pt-max-time",
com_pt_time,
help_text(":pt-max-time"),
},
{"session",
com_session,

@ -123,6 +123,7 @@ log_search_table::next(log_cursor& lc, logfile_sub_source& lss)
if (this->lst_regex.match(
this->lst_match_context, this->lst_input, PCRE_NO_UTF8_CHECK))
{
// log_debug("matched within line");
this->lst_match_index += 1;
return true;
}

@ -292,8 +292,18 @@ struct vtab {
};
struct vtab_cursor {
void cache_msg(logfile* lf, logfile::const_iterator ll)
{
if (this->log_msg_line == this->log_cursor.lc_curr_line) {
return;
}
lf->read_full_message(ll, this->log_msg);
this->log_msg_line = this->log_cursor.lc_curr_line;
}
sqlite3_vtab_cursor base;
struct log_cursor log_cursor;
vis_line_t log_msg_line{-1_vl};
shared_buffer_ref log_msg;
std::vector<logline_value> line_values;
};
@ -430,7 +440,7 @@ populate_indexed_columns(vtab_cursor* vc, vtab* vt)
lf = (*ld)->get_file_ptr();
auto ll = lf->begin() + line_number;
lf->read_full_message(ll, vc->log_msg);
vc->cache_msg(lf, ll);
vt->vi->extract(lf, line_number, vc->log_msg, vc->line_values);
}
@ -561,6 +571,12 @@ vt_next_no_rowid(sqlite3_vtab_cursor* cur)
}
} while (!done);
#ifdef DEBUG_INDEXING
log_debug("vt_next_no_rowid() -> %d:%d",
vc->log_cursor.lc_curr_line,
vc->log_cursor.lc_end_line);
#endif
return SQLITE_OK;
}
@ -637,7 +653,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
if (ll->is_time_skewed()) {
if (vc->line_values.empty()) {
lf->read_full_message(ll, vc->log_msg);
vc->cache_msg(lf, ll);
vt->vi->extract(
lf, line_number, vc->log_msg, vc->line_values);
}
@ -790,7 +806,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
switch (footer_column) {
case log_footer_columns::opid: {
if (vc->line_values.empty()) {
lf->read_full_message(ll, vc->log_msg);
vc->cache_msg(lf, ll);
vt->vi->extract(
lf, line_number, vc->log_msg, vc->line_values);
}
@ -857,7 +873,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
}
case log_footer_columns::body: {
if (vc->line_values.empty()) {
lf->read_full_message(ll, vc->log_msg);
vc->cache_msg(lf, ll);
vt->vi->extract(
lf, line_number, vc->log_msg, vc->line_values);
}
@ -926,7 +942,7 @@ vt_column(sqlite3_vtab_cursor* cur, sqlite3_context* ctx, int col)
}
} else {
if (vc->line_values.empty()) {
lf->read_full_message(ll, vc->log_msg);
vc->cache_msg(lf, ll);
vt->vi->extract(
lf, line_number, vc->log_msg, vc->line_values);
}

@ -732,6 +732,11 @@ logfile::read_full_message(logfile::const_iterator ll,
{
require(ll->get_sub_offset() == 0);
#if 0
log_debug(
"%s: reading msg at %d", this->lf_filename.c_str(), ll->get_offset());
#endif
msg_out.disown();
try {
auto read_result

@ -1,210 +0,0 @@
/**
* Copyright (c) 2015, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file papertrail_proc.cc
*/
#include "config.h"
#ifdef HAVE_LIBCURL
# include <curl/curl.h>
# include "papertrail_proc.hh"
# include "yajl/api/yajl_parse.h"
# include "yajlpp/yajlpp_def.hh"
const char* papertrail_proc::PT_SEARCH_URL
= "https://papertrailapp.com/api/v1/events/search.json?min_id=%s&";
static int
read_max_id(yajlpp_parse_context* ypc, const unsigned char* str, size_t len)
{
papertrail_proc* ptp = (papertrail_proc*) ypc->ypc_userdata;
ptp->ptp_last_max_id = std::string((const char*) str, len);
return 1;
}
static int
read_partial(yajlpp_parse_context* ypc, int val)
{
papertrail_proc* ptp = (papertrail_proc*) ypc->ypc_userdata;
if (val) {
ptp->ptp_partial_read = true;
}
return 1;
}
static int
read_limit(yajlpp_parse_context* ypc, int val)
{
papertrail_proc* ptp = (papertrail_proc*) ypc->ypc_userdata;
if (val) {
ptp->ptp_partial_read = true;
}
return 1;
}
static int
ignore_bool(yajlpp_parse_context* ypc, int val)
{
return 1;
}
static int
ignore_str(yajlpp_parse_context* ypc, const unsigned char* str, size_t len)
{
return 1;
}
static int
read_event_int(yajlpp_parse_context* ypc, long long val)
{
papertrail_proc* ptp = (papertrail_proc*) ypc->ypc_userdata;
yajl_gen_string(ptp->ptp_gen, ypc->get_path_fragment(2));
yajl_gen_integer(ptp->ptp_gen, val);
return 1;
}
static int
read_event_field(yajlpp_parse_context* ypc,
const unsigned char* str,
size_t len)
{
papertrail_proc* ptp = (papertrail_proc*) ypc->ypc_userdata;
yajl_gen_string(ptp->ptp_gen, ypc->get_path_fragment(2));
yajl_gen_string(ptp->ptp_gen, str, len);
return 1;
}
int
papertrail_proc::json_map_start(void* ctx)
{
yajlpp_parse_context* ypc = (yajlpp_parse_context*) ctx;
papertrail_proc* ptp = (papertrail_proc*) ypc->ypc_userdata;
if (ypc->ypc_path_index_stack.size() == 3) {
yajl_gen_map_open(ptp->ptp_gen);
}
return 1;
}
int
papertrail_proc::json_map_end(void* ctx)
{
yajlpp_parse_context* ypc = (yajlpp_parse_context*) ctx;
papertrail_proc* ptp = (papertrail_proc*) ypc->ypc_userdata;
if (ypc->ypc_path_index_stack.size() == 2) {
yajl_gen_map_close(ptp->ptp_gen);
yajl_gen_reset(ptp->ptp_gen, "\n");
}
return 1;
}
struct json_path_container papertrail_proc::FORMAT_HANDLERS
= json_path_container{
json_path_handler("/max_id", read_max_id),
json_path_handler("/(partial_results)", read_partial),
json_path_handler("/(reached_record_limit|reached_time_limit)",
read_limit),
json_path_handler("/(min_id|min_time_at|max_time_at|"
"reached_beginning|reached_end|tail|no_events)")
.add_cb(ignore_bool)
.add_cb(ignore_str),
json_path_handler("/events#/\\w+")
.add_cb(read_event_field)
.add_cb(read_event_int)};
size_t
papertrail_proc::write_cb(void* contents,
size_t size,
size_t nmemb,
void* userp)
{
yajl_handle handle = (yajl_handle) userp;
size_t realsize = size * nmemb;
if (yajl_parse(handle, (const unsigned char*) contents, realsize)
!= yajl_status_ok)
{
return -1;
}
return realsize;
}
void
papertrail_proc::yajl_writer(void* context, const char* str, size_t len)
{
papertrail_proc* ptp = (papertrail_proc*) context;
write(ptp->ptp_fd, str, len);
}
long
papertrail_proc::complete(CURLcode result)
{
curl_request::complete(result);
yajl_reset(this->ptp_jhandle.in());
if (result != CURLE_OK) {
static const char* err_msg = "Unable to execute papertrail search -- ";
write(this->ptp_fd, err_msg, strlen(err_msg));
write(
this->ptp_fd, this->cr_error_buffer, strlen(this->cr_error_buffer));
return -1;
}
if (this->ptp_max_time) {
return -1;
}
this->set_url();
if (this->ptp_partial_read) {
this->ptp_partial_read = false;
return 1;
}
return 3000;
}
#endif

@ -1,179 +0,0 @@
/**
* Copyright (c) 2015, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* @file papertrail_proc.hh
*/
#ifndef LNAV_PAPERTRAIL_PROC_HH
#define LNAV_PAPERTRAIL_PROC_HH
#include "config.h"
#ifdef HAVE_LIBCURL
# include <memory>
# include <string>
# include <fcntl.h>
# include <paths.h>
# include <signal.h>
# include <sys/wait.h>
# include <unistd.h>
# include "base/auto_fd.hh"
# include "base/auto_mem.hh"
# include "curl_looper.hh"
# include "line_buffer.hh"
# include "yajlpp/yajlpp.hh"
class papertrail_proc : public curl_request {
public:
papertrail_proc(const std::string& search, time_t min_time, time_t max_time)
: curl_request("papertrailapp.com"),
ptp_jcontext(intern_string::lookup(this->cr_name), &FORMAT_HANDLERS),
ptp_search(search), ptp_min_time(min_time), ptp_max_time(max_time)
{
char piper_tmpname[PATH_MAX];
const char* tmpdir;
if ((tmpdir = getenv("TMPDIR")) == NULL) {
tmpdir = _PATH_VARTMP;
}
snprintf(
piper_tmpname, sizeof(piper_tmpname), "%s/lnav.pt.XXXXXX", tmpdir);
if ((this->ptp_fd = mkstemp(piper_tmpname)) == -1) {
return;
}
unlink(piper_tmpname);
this->ptp_jcontext.ypc_alt_callbacks.yajl_start_map = json_map_start;
this->ptp_jcontext.ypc_alt_callbacks.yajl_end_map = json_map_end;
this->ptp_jcontext.ypc_userdata = this;
this->ptp_jhandle = yajl_alloc(
&this->ptp_jcontext.ypc_callbacks, NULL, &this->ptp_jcontext);
this->ptp_gen = yajl_gen_alloc(NULL);
yajl_gen_config(
this->ptp_gen, yajl_gen_print_callback, yajl_writer, this);
curl_easy_setopt(this->cr_handle, CURLOPT_WRITEFUNCTION, write_cb);
curl_easy_setopt(
this->cr_handle, CURLOPT_WRITEDATA, this->ptp_jhandle.in());
curl_easy_setopt(this->cr_handle, CURLOPT_FAILONERROR, 1L);
this->ptp_api_key = getenv("PAPERTRAIL_API_TOKEN");
if (this->ptp_api_key == NULL) {
this->ptp_error
= "papertrail search requested, but PAPERTRAIL_API_TOKEN is "
"not set";
}
this->ptp_quoted_search = curl_easy_escape(
this->cr_handle, this->ptp_search.c_str(), this->ptp_search.size());
log_perror(asprintf(this->ptp_token_header.out(),
"X-Papertrail-Token: %s",
this->ptp_api_key));
this->ptp_header_list = curl_slist_append(this->ptp_header_list,
this->ptp_token_header.in());
curl_easy_setopt(
this->cr_handle, CURLOPT_HTTPHEADER, this->ptp_header_list.in());
this->set_url();
}
~papertrail_proc() {}
auto_fd copy_fd() const { return this->ptp_fd.dup(); }
long complete(CURLcode result);
void set_url()
{
char base_url[1024];
snprintf(base_url,
sizeof(base_url),
PT_SEARCH_URL,
this->ptp_last_max_id.c_str());
if (this->ptp_min_time) {
size_t base_len = strlen(base_url);
snprintf(&base_url[base_len],
sizeof(base_url) - base_len,
"min_time=%ld&",
this->ptp_min_time);
}
if (this->ptp_max_time) {
size_t base_len = strlen(base_url);
snprintf(&base_url[base_len],
sizeof(base_url) - base_len,
"max_time=%ld&",
this->ptp_max_time);
}
log_perror(asprintf(this->ptp_url.out(),
"%sq=%s",
base_url,
this->ptp_quoted_search.in()));
curl_easy_setopt(this->cr_handle, CURLOPT_URL, this->ptp_url.in());
}
static size_t write_cb(void* contents,
size_t size,
size_t nmemb,
void* userp);
static int json_map_start(void* ctx);
static int json_map_end(void* ctx);
static void yajl_writer(void* context, const char* str, size_t len);
static struct json_path_container FORMAT_HANDLERS;
static const char* PT_SEARCH_URL;
yajlpp_parse_context ptp_jcontext;
auto_mem<yajl_handle_t> ptp_jhandle{yajl_free};
auto_mem<yajl_gen_t> ptp_gen{yajl_gen_free};
const char* ptp_api_key;
const std::string ptp_search;
auto_mem<const char> ptp_quoted_search{curl_free};
auto_mem<char> ptp_url;
auto_mem<char> ptp_token_header;
auto_mem<struct curl_slist> ptp_header_list{curl_slist_free_all};
auto_fd ptp_fd;
std::string ptp_last_max_id;
bool ptp_partial_read{false};
std::string ptp_error;
time_t ptp_min_time;
time_t ptp_max_time;
};
#endif
#endif // LNAV_PAPERTRAIL_PROC_HH

@ -61,7 +61,7 @@ shared_buffer_ref::share(shared_buffer& sb, char* data, size_t len)
this->sb_data = data;
this->sb_length = len;
ensure(this->sb_length < (5 * 1024 * 1024));
ensure(this->sb_length < (10 * 1024 * 1024));
}
bool

@ -396,8 +396,12 @@ EXPECTED_FILES = \
$(srcdir)/%reldir%/test_sessions.sh_903b41c950f5f90d7786d7a09bb6e2f217654b15.out \
$(srcdir)/%reldir%/test_sessions.sh_92a98a3e4e3a10bf1f2371d21a8282c5d3d4baa5.err \
$(srcdir)/%reldir%/test_sessions.sh_92a98a3e4e3a10bf1f2371d21a8282c5d3d4baa5.out \
$(srcdir)/%reldir%/test_sessions.sh_9978aaa475513f9981840e612f853a7707ffcf90.err \
$(srcdir)/%reldir%/test_sessions.sh_9978aaa475513f9981840e612f853a7707ffcf90.out \
$(srcdir)/%reldir%/test_sessions.sh_a92822d121a836140a401fd71535dc4a7a8d5b48.err \
$(srcdir)/%reldir%/test_sessions.sh_a92822d121a836140a401fd71535dc4a7a8d5b48.out \
$(srcdir)/%reldir%/test_sessions.sh_b3d71a87fcb4e3487f71ccad8c6ce681db220572.err \
$(srcdir)/%reldir%/test_sessions.sh_b3d71a87fcb4e3487f71ccad8c6ce681db220572.out \
$(srcdir)/%reldir%/test_sessions.sh_b932b33dd087b94d4306dd179c5d4f9ddd394960.err \
$(srcdir)/%reldir%/test_sessions.sh_b932b33dd087b94d4306dd179c5d4f9ddd394960.out \
$(srcdir)/%reldir%/test_sessions.sh_ddf45811e9906de9f3930fe802ac7b2cc6e48106.err \

@ -0,0 +1,11 @@
view_name filter_id enabled type language pattern
log 1 1 out regex blah
name search
log foobar
text
help
histogram
db
schema
pretty
spectro

@ -32,6 +32,25 @@ run_cap_test ${lnav_test} -nq \
-c ":export-session-to -" \
${test_dir}/logfile_access_log.0
run_cap_test ${lnav_test} -nq \
-c ";update access_log set log_mark = 1 where sc_bytes > 60000" \
-c ":set-min-log-level debug" \
-c ":hide-lines-before 2005" \
-c ":hide-lines-after 2030" \
-c ":filter-out blah" \
-c "/foobar" \
-c ":goto 1" \
-c ":export-session-to exported-session.0.lnav" \
${test_dir}/logfile_access_log.0
run_cap_test ${lnav_test} -n \
-c "|exported-session.0.lnav" \
-c ";SELECT * FROM lnav_view_filters" \
-c ":write-screen-to -" \
-c ";SELECT name,search FROM lnav_views" \
-c ":write-screen-to -" \
${test_dir}/logfile_access_log.0
# log mark was not saved in session
run_cap_test ${lnav_test} -n \
-c ":load-session" \

Loading…
Cancel
Save