Archives/lnav
2
0
Fork 0
mirror of https://github.com/tstack/lnav synced 2024-11-05 21:21:19 +00:00
Code Issues Packages Projects Releases Wiki Activity
1c8c7089cb
lnav/test/drive_data_scanner.cc
Tim Stack 1c8c7089cb [log_format] try to do a better job matching JSON formats to logs 
Try all formats and find the one with the highest quality
match.  For JSON logs, this means the format that matched
the most fields in the line-format.

Related to #1136
2023-06-20 09:38:11 -07:00

313 lines
11 KiB
C++
Raw Blame History

/**
* Copyright (c) 2007-2012, Timothy Stack
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* * Neither the name of Timothy Stack nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifdef __CYGWIN__
# include <alloca.h>
#endif
#include <fstream>
#include <iostream>
#include <stdio.h>
#include <stdlib.h>
#include "base/injector.hh"
#include "config.h"
#include "data_parser.hh"
#include "data_scanner.hh"
#include "elem_to_json.hh"
#include "log_format.hh"
#include "log_format_loader.hh"
#include "logfile.hh"
#include "pretty_printer.hh"
#include "shared_buffer.hh"
#include "view_curses.hh"
const char* TMP_NAME = "scanned.tmp";
int
main(int argc, char* argv[])
{
int c, retval = EXIT_SUCCESS;
bool prompt = false, is_log = false, pretty_print = false;
bool scanner_details = false;
{
static auto builtin_formats
= injector::get<std::vector<std::shared_ptr<log_format>>>();
auto& root_formats = log_format::get_root_formats();
log_format::get_root_formats().insert(root_formats.begin(),
builtin_formats.begin(),
builtin_formats.end());
builtin_formats.clear();
}
{
std::vector<ghc::filesystem::path> paths;
std::vector<lnav::console::user_message> errors;
load_formats(paths, errors);
}
while ((c = getopt(argc, argv, "pPls")) != -1) {
switch (c) {
case 'p':
prompt = true;
break;
case 'P':
pretty_print = true;
break;
case 'l':
is_log = true;
break;
case 's':
scanner_details = true;
break;
default:
retval = EXIT_FAILURE;
break;
}
}
argc -= optind;
argv += optind;
if (retval != EXIT_SUCCESS) {
} else if (argc < 1) {
fprintf(stderr, "error: expecting file name argument(s)\n");
retval = EXIT_FAILURE;
} else {
for (int lpc = 0; lpc < argc; lpc++) {
std::unique_ptr<std::ifstream> in_ptr;
std::istream* in;
FILE* out;
if (strcmp(argv[lpc], "-") == 0) {
in = &std::cin;
} else {
auto ifs = std::make_unique<std::ifstream>(argv[lpc]);
if (!ifs->is_open()) {
fprintf(stderr, "error: unable to open file\n");
retval = EXIT_FAILURE;
} else {
in_ptr = std::move(ifs);
in = in_ptr.get();
}
}
if ((out = fopen(TMP_NAME, "w")) == nullptr) {
fprintf(stderr,
"error: unable to temporary file for writing\n");
retval = EXIT_FAILURE;
} else {
std::shared_ptr<log_format> format;
char* log_line;
bool found = false;
char cmd[2048];
std::string line;
int rc;
getline(*in, line);
if (strcmp(argv[lpc], "-") == 0) {
line = " " + line;
}
log_line = (char*) alloca(line.length());
strcpy(log_line, &line[13]);
auto sub_line = line.substr(13);
struct line_range body(0, sub_line.length());
shared_buffer share_manager;
logline_value_vector ll_values;
auto& sbr = ll_values.lvv_sbr;
sbr.share(
share_manager, (char*) sub_line.c_str(), sub_line.size());
auto& root_formats = log_format::get_root_formats();
std::vector<std::shared_ptr<log_format>>::iterator iter;
std::vector<logline> index;
if (is_log) {
logfile_open_options loo;
auto open_res = logfile::open(argv[lpc], loo);
auto lf = open_res.unwrap();
ArenaAlloc::Alloc<char> allocator;
scan_batch_context sbc{allocator};
for (iter = root_formats.begin();
iter != root_formats.end() && !found;
++iter)
{
line_info li = {{13}};
(*iter)->clear();
if ((*iter)
->scan(*lf, index, li, sbr, sbc)
.is<log_format::scan_match>())
{
format = (*iter)->specialized();
found = true;
}
}
if (!found) {
fprintf(stderr, "error: log sample does not match\n");
return EXIT_FAILURE;
}
}
string_attrs_t sa;
if (format.get() != nullptr) {
format->annotate(0, sa, ll_values);
body = find_string_attr_range(sa, &SA_BODY);
}
data_parser::TRACE_FILE = fopen("scanned.dpt", "w");
data_scanner ds(sub_line, body.lr_start);
if (scanner_details) {
fprintf(out,
" %s\n",
ds.get_input().to_string().c_str());
while (true) {
auto tok_res = ds.tokenize2();
if (!tok_res) {
break;
}
fprintf(out,
"%4s %3d:%-3d ",
data_scanner::token2name(tok_res->tr_token),
tok_res->tr_capture.c_begin,
tok_res->tr_capture.c_end);
size_t cap_index = 0;
for (; cap_index < tok_res->tr_capture.c_end;
cap_index++)
{
if (cap_index == tok_res->tr_capture.c_begin) {
fputc('^', out);
} else if (cap_index
== (tok_res->tr_capture.c_end - 1))
{
fputc('^', out);
} else if (cap_index > tok_res->tr_capture.c_begin)
{
fputc('-', out);
} else {
fputc(' ', out);
}
}
for (; cap_index < (int) ds.get_input().length();
cap_index++)
{
fputc(' ', out);
}
auto sub = tok_res->to_string();
fprintf(out, " %s\n", sub.c_str());
}
}
ds.reset();
data_parser dp(&ds);
std::string msg_format;
dp.dp_msg_format = &msg_format;
dp.parse();
dp.print(out, dp.dp_pairs);
fprintf(
out, "msg :%s\n", sub_line.c_str() + body.lr_start);
fprintf(out, "format :%s\n", msg_format.c_str());
if (pretty_print) {
data_scanner ds2(sub_line, body.lr_start);
pretty_printer pp(&ds2, sa);
attr_line_t pretty_out;
pp.append_to(pretty_out);
fprintf(out, "\n--\n%s", pretty_out.get_string().c_str());
}
auto_mem<yajl_gen_t> gen(yajl_gen_free);
gen = yajl_gen_alloc(nullptr);
yajl_gen_config(gen.in(), yajl_gen_beautify, true);
elements_to_json(gen, dp, &dp.dp_pairs);
const unsigned char* buf;
size_t len;
yajl_gen_get_buf(gen, &buf, &len);
fwrite(buf, 1, len, out);
fclose(out);
snprintf(
cmd, sizeof(cmd), "diff -u %s %s", argv[lpc], TMP_NAME);
rc = system(cmd);
if (rc != 0) {
if (prompt) {
char resp[4];
printf("\nOriginal line:\n%s\n",
sub_line.c_str() + body.lr_start);
printf(
"Would you like to update the original file? "
"(y/N) ");
fflush(stdout);
log_perror(scanf("%3s", resp));
if (strcasecmp(resp, "y") == 0) {
rename(TMP_NAME, argv[lpc]);
} else {
retval = EXIT_FAILURE;
}
} else {
fprintf(stderr, "error: mismatch\n");
retval = EXIT_FAILURE;
}
}
fclose(data_parser::TRACE_FILE);
data_parser::TRACE_FILE = nullptr;
}
}
}
return retval;
}
Reference in New Issue View Git Blame Copy Permalink