From 926d73afd03f74be84d1e76abc4f726da0acf348 Mon Sep 17 00:00:00 2001 From: Tim Stack Date: Wed, 7 Feb 2024 20:58:53 -0800 Subject: [PATCH] [crumbs] try to fix crumb for json path --- src/logfile.cc | 4 +- src/textfile_sub_source.cc | 16 ++- test/CMakeLists.txt | 3 + test/Makefile.am | 5 + test/drive_doc_discovery.cc | 116 ++++++++++++++++++ test/expected/expected.am | 2 + ...fd274911e45a743b4de616888a64183d07cb76.err | 0 ...fd274911e45a743b4de616888a64183d07cb76.out | 2 + test/test_text_file.sh | 4 + test/textfile_nonl.txt | 1 + 10 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 test/drive_doc_discovery.cc create mode 100644 test/expected/test_text_file.sh_11fd274911e45a743b4de616888a64183d07cb76.err create mode 100644 test/expected/test_text_file.sh_11fd274911e45a743b4de616888a64183d07cb76.out create mode 100644 test/textfile_nonl.txt diff --git a/src/logfile.cc b/src/logfile.cc index f18457d7..4521ed3f 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -1022,7 +1022,9 @@ logfile::read_file() retval.append(22, '\x16'); } retval.append(sbr.get_data(), sbr.length()); - retval.push_back('\n'); + if (retval.size() < this->lf_stat.st_size) { + retval.push_back('\n'); + } } return Ok(std::move(retval)); diff --git a/src/textfile_sub_source.cc b/src/textfile_sub_source.cc index b4589b20..f85e5fd6 100644 --- a/src/textfile_sub_source.cc +++ b/src/textfile_sub_source.cc @@ -593,6 +593,7 @@ textfile_sub_source::text_crumbs_for_line( const auto initial_size = crumbs.size(); meta_iter->second.ms_metadata.m_sections_tree.visit_overlapping( + lf->get_line_content_offset(ll_iter), end_offset, [&crumbs, initial_size, @@ -893,6 +894,14 @@ textfile_sub_source::rescan_files( if (st.st_mtime != ms_iter->second.ms_mtime || st.st_size != ms_iter->second.ms_file_size) { + log_debug( + "text file has changed, invalidating metadata. " + "old: {mtime: %d size: %zu}, new: {mtime: %d " + "size: %zu}", + ms_iter->second.ms_mtime, + ms_iter->second.ms_file_size, + st.st_mtime, + st.st_size); this->tss_doc_metadata.erase(ms_iter); ms_iter = this->tss_doc_metadata.end(); } @@ -904,8 +913,9 @@ textfile_sub_source::rescan_files( if (read_res.isOk()) { auto content = attr_line_t(read_res.unwrap()); - log_info("generating metadata for: %s", - lf->get_filename().c_str()); + log_info("generating metadata for: %s (size=%zu)", + lf->get_filename().c_str(), + content.length()); scrub_ansi_string(content.get_string(), &content.get_attrs()); @@ -920,7 +930,7 @@ textfile_sub_source::rescan_files( this->tss_doc_metadata[lf->get_filename()] = metadata_state{ st.st_mtime, - static_cast(st.st_size), + static_cast(content.length()), lnav::document::discover_structure( content, line_range{0, -1}, diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c98dddc5..9dbec24d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -85,5 +85,8 @@ target_link_libraries(drive_sql_anno diag) add_executable(drive_data_scanner drive_data_scanner.cc test_stubs.cc) target_link_libraries(drive_data_scanner diag logfmt) +add_executable(drive_doc_discovery drive_doc_discovery.cc test_stubs.cc) +target_link_libraries(drive_doc_discovery diag logfmt) + add_executable(scripty scripty.cc test_stubs.cc) target_link_libraries(scripty diag) diff --git a/test/Makefile.am b/test/Makefile.am index b003e551..e01c0424 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -52,6 +52,7 @@ DUMMY_OBJS = \ check_PROGRAMS = \ drive_data_scanner \ + drive_doc_discovery \ drive_line_buffer \ drive_grep_proc \ drive_listview \ @@ -164,6 +165,9 @@ drive_shlexer_SOURCES = drive_shlexer.cc drive_data_scanner_SOURCES = \ drive_data_scanner.cc +drive_doc_discovery_SOURCES = \ + drive_doc_discovery.cc + drive_mvwattrline_SOURCES = drive_mvwattrline.cc drive_view_colors_SOURCES = drive_view_colors.cc @@ -383,6 +387,7 @@ dist_noinst_DATA = \ textfile_ansi_expanding.0 \ textfile_json_indented.0 \ textfile_json_one_line.0 \ + textfile_nonl.txt \ textfile_quoted_json.0 \ toplevel.lnav \ UTF-8-test.txt \ diff --git a/test/drive_doc_discovery.cc b/test/drive_doc_discovery.cc new file mode 100644 index 00000000..ec932942 --- /dev/null +++ b/test/drive_doc_discovery.cc @@ -0,0 +1,116 @@ +/** + * Copyright (c) 2024, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +#include "base/fs_util.hh" +#include "document.sections.hh" +#include "fmt/color.h" + +int +main(int argc, char* argv[]) +{ + int retval = EXIT_SUCCESS; + + if (argc < 2) { + fprintf(stderr, "error: expecting file to discover\n"); + retval = EXIT_FAILURE; + } else { + const auto fn = ghc::filesystem::path(argv[1]); + auto read_res = lnav::filesystem::read_file(fn); + if (read_res.isErr()) { + fprintf(stderr, + "error: unable to read %s -- %s\n", + fn.c_str(), + read_res.unwrapErr().c_str()); + retval = EXIT_FAILURE; + } else { + auto content = attr_line_t(read_res.unwrap()); + const auto& content_sf + = string_fragment::from_str(content.get_string()); + auto tf = detect_text_format(content_sf, fn); + auto lr = line_range{0, static_cast(content.length())}; + auto meta = lnav::document::discover_structure(content, lr, tf); + + auto remaining = content_sf; + while (!remaining.empty()) { + auto line_pair + = remaining.split_when(string_fragment::tag1{'\n'}); + auto line_sf = line_pair.first; + fmt::print(FMT_STRING("{}\n"), line_sf); + size_t indent = 0; + meta.m_sections_tree.visit_overlapping( + line_sf.sf_begin, + line_sf.sf_end, + [&line_sf, &indent](const auto& iv) { + if (iv.start < line_sf.sf_begin) { + return; + } + auto this_indent = iv.start - line_sf.sf_begin; + if (this_indent < indent) { + return; + } + auto indent_diff = this_indent - indent; + indent = this_indent; + fmt::print(FMT_STRING("{}^"), + std::string(indent_diff, ' ')); + if (iv.stop >= line_sf.sf_end + 1) { + fmt::print( + FMT_STRING(" [{}:{})"), iv.start, iv.stop); + return; + } + auto dot_len = iv.stop - iv.start - 1; + fmt::print(FMT_STRING("{}^"), + std::string(dot_len, '-')); + fmt::print(FMT_STRING(" [{}:{})"), iv.start, iv.stop); + }); + fmt::print(FMT_STRING("\n")); + meta.m_sections_tree.visit_overlapping( + line_sf.sf_begin, + line_sf.sf_end, + [&line_sf](const auto& iv) { + fmt::print( + fmt::fg(iv.start < line_sf.sf_begin + ? fmt::terminal_color::yellow + : fmt::terminal_color::green), + FMT_STRING("/{}"), + iv.value.match( + [](const std::string& str) { return str; }, + [](size_t ind) { + return fmt::to_string(ind); + })); + }); + fmt::print(FMT_STRING("\n")); + remaining = line_pair.second; + } + } + } + + return retval; +} diff --git a/test/expected/expected.am b/test/expected/expected.am index 944d9e90..24e856e7 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -1196,6 +1196,8 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_sql_yaml_func.sh_dc189d02e8979b7ed245d5d750f68b9965984699.out \ $(srcdir)/%reldir%/test_text_file.sh_0bba304f34ae07c4fa9e91e0b42f5fe98654a6a8.err \ $(srcdir)/%reldir%/test_text_file.sh_0bba304f34ae07c4fa9e91e0b42f5fe98654a6a8.out \ + $(srcdir)/%reldir%/test_text_file.sh_11fd274911e45a743b4de616888a64183d07cb76.err \ + $(srcdir)/%reldir%/test_text_file.sh_11fd274911e45a743b4de616888a64183d07cb76.out \ $(srcdir)/%reldir%/test_text_file.sh_1ce4056d72b871f8bb844c86aade2a9b1da58030.err \ $(srcdir)/%reldir%/test_text_file.sh_1ce4056d72b871f8bb844c86aade2a9b1da58030.out \ $(srcdir)/%reldir%/test_text_file.sh_4226123565a53b4e3f80e602c1f294721e8e07bf.err \ diff --git a/test/expected/test_text_file.sh_11fd274911e45a743b4de616888a64183d07cb76.err b/test/expected/test_text_file.sh_11fd274911e45a743b4de616888a64183d07cb76.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_text_file.sh_11fd274911e45a743b4de616888a64183d07cb76.out b/test/expected/test_text_file.sh_11fd274911e45a743b4de616888a64183d07cb76.out new file mode 100644 index 00000000..77618337 --- /dev/null +++ b/test/expected/test_text_file.sh_11fd274911e45a743b4de616888a64183d07cb76.out @@ -0,0 +1,2 @@ +content  + abc diff --git a/test/test_text_file.sh b/test/test_text_file.sh index 9365b0a7..0cff3b17 100644 --- a/test/test_text_file.sh +++ b/test/test_text_file.sh @@ -69,3 +69,7 @@ run_cap_test ${lnav_test} -n \ run_cap_test ${lnav_test} -n \ < ${top_srcdir}/autogen.sh + +run_cap_test ${lnav_test} -n \ + -c ';SELECT content FROM lnav_file' \ + ${test_dir}/textfile_nonl.txt diff --git a/test/textfile_nonl.txt b/test/textfile_nonl.txt new file mode 100644 index 00000000..f2ba8f84 --- /dev/null +++ b/test/textfile_nonl.txt @@ -0,0 +1 @@ +abc \ No newline at end of file