Archives
/
lnav
mirror of https://github.com/tstack/lnav
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[anchor] fix anchors for json files

Browse Source
pull/1235/head
Tim Stack 4 months ago
parent 5c0772e9af
commit 0eb394f4b8
14 changed files with 296 additions and 57 deletions
Show Stats Download Patch File Download Diff File

49
src/base/intern_string.cc
Unescape Escape View File

@ -305,6 +305,55 @@ string_fragment::to_string_with_case_style(case_style style) const
return retval;
}
std::string
string_fragment::to_unquoted_string() const
{
auto sub_sf = *this;
if (sub_sf.startswith("r") || sub_sf.startswith("u")) {
sub_sf = sub_sf.consume_n(1).value();
}
if (sub_sf.length() >= 2
&& ((sub_sf.startswith("\"") && sub_sf.endswith("\""))
|| (sub_sf.startswith("'") && sub_sf.endswith("'"))))
{
std::string retval;
sub_sf.sf_begin += 1;
sub_sf.sf_end -= 1;
retval.reserve(this->length());
auto in_escape = false;
for (auto ch : sub_sf) {
if (in_escape) {
switch (ch) {
case 'n':
retval.push_back('\n');
break;
case 't':
retval.push_back('\t');
break;
case 'r':
retval.push_back('\r');
break;
default:
retval.push_back(ch);
break;
}
in_escape = false;
} else if (ch == '\\') {
in_escape = true;
} else {
retval.push_back(ch);
}
}
return retval;
}
return this->to_string();
}
uint32_t
string_fragment::front_codepoint() const
{

5
src/base/intern_string.hh
Unescape Escape View File

@ -548,6 +548,8 @@ struct string_fragment {
return {this->data(), (size_t) this->length()};
}
std::string to_unquoted_string() const;
void clear()
{
this->sf_begin = 0;
@ -854,7 +856,8 @@ operator==(const string_fragment& left, const intern_string_t& right)
&& (memcmp(left.data(), right.get(), left.length()) == 0);
}
inline string_fragment operator"" _frag(const char* str, std::size_t len)
inline string_fragment
operator"" _frag(const char* str, std::size_t len)
{
return string_fragment::from_byte_range(str, 0, len);
}

5
src/document.sections.cc
Unescape Escape View File

@ -328,7 +328,8 @@ public:
this->sw_interval_state.back().is_line_number
= this->sw_line_number;
this->sw_interval_state.back().is_name
= tokenize_res->to_string();
= tokenize_res->to_string_fragment()
.to_unquoted_string();
this->sw_depth += 1;
this->sw_interval_state.resize(this->sw_depth + 1);
this->sw_hier_nodes.push_back(
@ -609,7 +610,7 @@ private:
this->sw_interval_state.back().is_name
= this->sw_scanner
.to_string_fragment(last_key.value())
.to_string();
.to_unquoted_string();
if (!this->sw_interval_state.back().is_name.empty()) {
this->sw_interval_state.back().is_start
= static_cast<ssize_t>(

14
src/log_format.cc
Unescape Escape View File

@ -2107,7 +2107,7 @@ external_log_format::get_subline(const logline& ll,
= jlu.jlu_opid_frag->to_string();
}
int sub_offset = 1 + this->jlf_line_format_init_count;
int sub_offset = this->jlf_line_format_init_count;
for (const auto& jfe : this->jlf_line_format) {
static const intern_string_t ts_field
= intern_string::lookup("__timestamp__", -1);
@ -2213,6 +2213,7 @@ external_log_format::get_subline(const logline& ll,
lr, logline::L_OPID.value());
}
lv_iter->lv_origin = lr;
lv_iter->lv_sub_offset = sub_offset;
used_values[std::distance(
this->jlf_line_values.lvv_values.begin(),
lv_iter)]
@ -2333,6 +2334,7 @@ external_log_format::get_subline(const logline& ll,
}
}
this->json_append_to_cache("\n", 1);
sub_offset += 1;
for (size_t lpc = 0; lpc < this->jlf_line_values.lvv_values.size();
lpc++)
@ -3872,11 +3874,6 @@ external_log_format::value_line_count(const intern_string_t ist,
return retval;
}
if (iter->second->vd_meta.is_hidden()) {
retval.vlcr_count = 0;
return retval;
}
if (str != nullptr && !val) {
auto frag = string_fragment::from_bytes(str, len);
while (frag.endswith("\n")) {
@ -3907,6 +3904,11 @@ external_log_format::value_line_count(const intern_string_t ist,
}
}
if (iter->second->vd_meta.is_hidden()) {
retval.vlcr_count = 0;
return retval;
}
if (std::find_if(this->jlf_line_format.begin(),
this->jlf_line_format.end(),
json_field_cmp(json_log_field::VARIABLE, ist))

143
src/textfile_sub_source.cc
Unescape Escape View File

@ -40,6 +40,7 @@
#include "config.h"
#include "lnav.events.hh"
#include "md2attr_line.hh"
#include "scn/scn.h"
#include "sql_util.hh"
#include "sqlitepp.hh"
@ -1016,7 +1017,7 @@ nonstd::optional<vis_line_t>
textfile_sub_source::row_for_anchor(const std::string& id)
{
auto lf = this->current_file();
if (!lf) {
if (!lf || id.empty()) {
return nonstd::nullopt;
}
@ -1033,27 +1034,98 @@ textfile_sub_source::row_for_anchor(const std::string& id)
const auto& meta = iter->second.ms_metadata;
nonstd::optional<vis_line_t> retval;
auto is_ptr = startswith(id, "#/");
if (is_ptr) {
auto hier_sf = string_fragment::from_str(id).consume_n(2).value();
std::vector<lnav::document::section_key_t> path;
while (!hier_sf.empty()) {
auto comp_pair = hier_sf.split_when(string_fragment::tag1{'/'});
auto scan_res
= scn::scan_value<int64_t>(comp_pair.first.to_string_view());
if (scan_res && scan_res.empty()) {
path.emplace_back(scan_res.value());
} else {
path.emplace_back(json_ptr::decode(comp_pair.first));
}
hier_sf = comp_pair.second;
}
auto lookup_res = lnav::document::hier_node::lookup_path(
meta.m_sections_root.get(), path);
if (lookup_res) {
auto ll_opt = lf->line_for_offset(lookup_res.value()->hn_start);
if (ll_opt != lf->end()) {
retval
= vis_line_t(std::distance(lf->cbegin(), ll_opt.value()));
}
}
return retval;
}
lnav::document::hier_node::depth_first(
meta.m_sections_root.get(),
[lf, &id, &retval](const lnav::document::hier_node* node) {
for (const auto& child_pair : node->hn_named_children) {
auto child_anchor
const auto& child_anchor
= text_anchors::to_anchor_string(child_pair.first);
if (child_anchor == id) {
auto ll_opt
= lf->line_for_offset(child_pair.second->hn_start);
if (ll_opt != lf->end()) {
retval = vis_line_t(
std::distance(lf->cbegin(), ll_opt.value()));
}
if (child_anchor != id) {
continue;
}
auto ll_opt = lf->line_for_offset(child_pair.second->hn_start);
if (ll_opt != lf->end()) {
retval = vis_line_t(
std::distance(lf->cbegin(), ll_opt.value()));
}
break;
}
});
return retval;
}
static void
anchor_generator(std::unordered_set<std::string>& retval,
std::vector<std::string>& comps,
size_t& max_depth,
lnav::document::hier_node* hn)
{
if (hn->hn_named_children.empty()) {
if (hn->hn_children.empty()) {
if (retval.size() >= 250 || comps.empty()) {
} else if (comps.size() == 1) {
retval.emplace(text_anchors::to_anchor_string(comps.front()));
} else {
retval.emplace(
fmt::format(FMT_STRING("#/{}"),
fmt::join(comps.begin(), comps.end(), "/")));
}
max_depth = std::max(max_depth, comps.size());
} else {
int index = 0;
for (const auto& child : hn->hn_children) {
comps.emplace_back(fmt::to_string(index));
anchor_generator(retval, comps, max_depth, child.get());
comps.pop_back();
}
}
} else {
for (const auto& child : hn->hn_named_children) {
comps.emplace_back(child.first);
anchor_generator(retval, comps, max_depth, child.second);
comps.pop_back();
}
if (max_depth > 1) {
retval.emplace(
fmt::format(FMT_STRING("#/{}"),
fmt::join(comps.begin(), comps.end(), "/")));
}
}
}
std::unordered_set<std::string>
textfile_sub_source::get_anchors()
{
@ -1076,18 +1148,13 @@ textfile_sub_source::get_anchors()
const auto& meta = iter->second.ms_metadata;
lnav::document::hier_node::depth_first(
meta.m_sections_root.get(),
[&retval](const lnav::document::hier_node* node) {
if (retval.size() > 100) {
return;
}
if (meta.m_sections_root == nullptr) {
return retval;
}
for (const auto& child_pair : node->hn_named_children) {
retval.emplace(
text_anchors::to_anchor_string(child_pair.first));
}
});
std::vector<std::string> comps;
size_t max_depth = 0;
anchor_generator(retval, comps, max_depth, meta.m_sections_root.get());
return retval;
}
@ -1095,11 +1162,9 @@ textfile_sub_source::get_anchors()
nonstd::optional<std::string>
textfile_sub_source::anchor_for_row(vis_line_t vl)
{
nonstd::optional<std::string> retval;
auto lf = this->current_file();
if (!lf) {
return retval;
return nonstd::nullopt;
}
auto rend_iter = this->tss_rendered_files.find(lf->get_filename());
@ -1109,31 +1174,43 @@ textfile_sub_source::anchor_for_row(vis_line_t vl)
auto iter = this->tss_doc_metadata.find(lf->get_filename());
if (iter == this->tss_doc_metadata.end()) {
return retval;
return nonstd::nullopt;
}
auto* lfo = dynamic_cast<line_filter_observer*>(lf->get_logline_observer());
if (vl >= lfo->lfo_filter_state.tfs_index.size()) {
return retval;
return nonstd::nullopt;
}
auto ll_iter = lf->begin() + lfo->lfo_filter_state.tfs_index[vl];
auto ll_next_iter = ll_iter + 1;
auto end_offset = (ll_next_iter == lf->end())
? lf->get_index_size() - 1
: ll_next_iter->get_offset() - 1;
std::vector<std::string> collector;
iter->second.ms_metadata.m_sections_tree.visit_overlapping(
ll_iter->get_offset(),
end_offset,
[&retval](const lnav::document::section_interval_t& iv) {
retval = iv.value.match(
[](const std::string& str) {
return nonstd::make_optional(
text_anchors::to_anchor_string(str));
},
[](size_t) { return nonstd::nullopt; });
[&collector](const lnav::document::section_interval_t& iv) {
collector.emplace_back(iv.value.match(
[](const std::string& str) { return str; },
[](size_t index) { return fmt::to_string(index); }));
});
return retval;
if (collector.empty()) {
return nonstd::nullopt;
}
if (collector.size() == 1) {
return text_anchors::to_anchor_string(collector.front());
}
for (auto& elem : collector) {
elem = json_ptr::encode_str(elem);
}
return fmt::format(FMT_STRING("#/{}"),
fmt::join(collector.begin(), collector.end(), "/"));
}
bool

33
src/yajlpp/json_ptr.cc
Unescape Escape View File

@ -281,6 +281,39 @@ json_ptr::decode(char* dst, const char* src, ssize_t src_len)
return retval;
}
std::string
json_ptr::decode(const string_fragment& sf)
{
std::string retval;
auto in_escape = false;
retval.reserve(sf.length());
for (const auto ch : sf) {
if (in_escape) {
switch (ch) {
case '0':
retval.push_back('~');
break;
case '1':
retval.push_back('/');
break;
case '2':
retval.push_back('#');
break;
default:
break;
}
in_escape = false;
} else if (ch == '~') {
in_escape = true;
} else {
retval.push_back(ch);
}
}
return retval;
}
bool
json_ptr::expect_map(int32_t& depth, int32_t& index)
{

2
src/yajlpp/json_ptr.hh
Unescape Escape View File

@ -41,6 +41,7 @@
#include <sys/types.h>
#include "base/auto_mem.hh"
#include "base/intern_string.hh"
#include "yajl/api/yajl_parse.h"
#include "yajl/api/yajl_tree.h"
@ -116,6 +117,7 @@ public:
}
static size_t decode(char* dst, const char* src, ssize_t src_len = -1);
static std::string decode(const string_fragment& sf);
json_ptr(const char* value) : jp_value(value), jp_pos(value) {}

2
test/expected/expected.am
Unescape Escape View File

@ -1210,6 +1210,8 @@ EXPECTED_FILES = \
$(srcdir)/%reldir%/test_text_file.sh_6a24078983cf1b7a80b6fb65d5186cd125498136.out \
$(srcdir)/%reldir%/test_text_file.sh_73f69c883f60761bff9f8874f61d21a189e92912.err \
$(srcdir)/%reldir%/test_text_file.sh_73f69c883f60761bff9f8874f61d21a189e92912.out \
$(srcdir)/%reldir%/test_text_file.sh_78f252288519c8f767bb2759ea32959dab2ebc46.err \
$(srcdir)/%reldir%/test_text_file.sh_78f252288519c8f767bb2759ea32959dab2ebc46.out \
$(srcdir)/%reldir%/test_text_file.sh_7b00f32a3fff7fc2d78a87045ae842e58be88480.err \
$(srcdir)/%reldir%/test_text_file.sh_7b00f32a3fff7fc2d78a87045ae842e58be88480.out \
$(srcdir)/%reldir%/test_text_file.sh_87943c6be50d701a03e901f16493314c839af1ab.err \

26
test/expected/test_json_format.sh_469f005b0708d629bc95f0c48a5e390f440c1fef.out
Unescape Escape View File

@ -1,29 +1,29 @@
[2013-09-06T20:00:48.124] TRACE trace test
[2013-09-06T20:00:48.124] ⋮ trace test
[2013-09-06T20:00:49.124] INFO Starting up service
[2013-09-06T20:00:49.124] ⋮ Starting up service
[2013-09-06T22:00:49.124] INFO Shutting down service
[2013-09-06T22:00:49.124] ⋮ Shutting down service
user: steve@example.com
[2013-09-06T22:00:59.124] DEBUG5 Details...
[2013-09-06T22:00:59.124] ⋮ Details...
[2013-09-06T22:00:59.124] DEBUG4 Details...
[2013-09-06T22:00:59.124] ⋮ Details...
[2013-09-06T22:00:59.124] DEBUG3 Details...
[2013-09-06T22:00:59.124] ⋮ Details...
[2013-09-06T22:00:59.124] DEBUG2 Details...
[2013-09-06T22:00:59.124] ⋮ Details...
[2013-09-06T22:01:00.000] DEBUG Details...
[2013-09-06T22:01:00.000] ⋮ Details...
[2013-09-06T22:01:49.124] STATS 1 beat per second
[2013-09-06T22:01:49.124] ⋮ 1 beat per second
[2013-09-06T22:01:49.124] WARNING not looking good
[2013-09-06T22:01:49.124] ⋮ not looking good
[2013-09-06T22:01:49.124] ERROR looking bad
[2013-09-06T22:01:49.124] ⋮ looking bad
[2013-09-06T22:01:49.124] CRITICAL sooo bad
[2013-09-06T22:01:49.124] ⋮ sooo bad
[2013-09-06T22:01:49.124] FATAL shoot
[2013-09-06T22:01:49.124] ⋮ shoot
 obj: { "field1" : "hi", "field2": 2 }
 arr: ["hi", {"sub1": true}]

2
test/expected/test_json_format.sh_f740026626ab554dacb249762d8be7d6539b8c6e.out
Unescape Escape View File

@ -1,2 +1,2 @@
[2013-09-06T20:00:49.124] INFO Starting up service
[2013-09-06T20:00:49.124] ⋮ Starting up service

2
test/expected/test_text_file.sh_1ce4056d72b871f8bb844c86aade2a9b1da58030.out
Unescape Escape View File

@ -2,7 +2,7 @@
{
"top_meta": {
"file": "stdin",
"anchor": "#distclean-local-",
"anchor": "#/test~1Makefile.am/distclean-local:",
"breadcrumbs": [
{
"display_value": "stdin",

0
test/expected/test_text_file.sh_78f252288519c8f767bb2759ea32959dab2ebc46.err
Unescape Escape View File

64
test/expected/test_text_file.sh_78f252288519c8f767bb2759ea32959dab2ebc46.out
Unescape Escape View File

@ -0,0 +1,64 @@
[
{
"top_meta": {
"file": "{test_dir}/formats/jsontest/format.json",
"anchor": "#/test_log/timestamp-field",
"breadcrumbs": [
{
"display_value": "format.json",
"search_placeholder": "",
"possibilities": [
{
"display_value": "format.json"
}
]
},
{
"display_value": "test_log",
"search_placeholder": "",
"possibilities": [
{
"display_value": "$schema"
},
{
"display_value": "test_log"
}
]
},
{
"display_value": "timestamp-field",
"search_placeholder": "",
"possibilities": [
{
"display_value": "body-field"
},
{
"display_value": "description"
},
{
"display_value": "file-pattern"
},
{
"display_value": "json"
},
{
"display_value": "level-field"
},
{
"display_value": "line-format"
},
{
"display_value": "timestamp-field"
},
{
"display_value": "title"
},
{
"display_value": "value"
}
]
}
]
}
}
]

6
test/test_text_file.sh
Unescape Escape View File

@ -73,3 +73,9 @@ run_cap_test ${lnav_test} -n \
run_cap_test ${lnav_test} -n \
-c ';SELECT content FROM lnav_file' \
${test_dir}/textfile_nonl.txt
run_cap_test ${lnav_test} -n \
-c ':goto 23' \
-c ';SELECT top_meta FROM lnav_top_view' \
-c ':write-json-to -' \
${test_dir}/formats/jsontest/format.json

Write Preview
Loading…
Cancel
Save