[doc.sections] fixes for man pages

src/base/ansi_scrubber.cc

@@ -45,7 +45,7 @@ static const lnav::pcre2pp::code&
 ansi_regex()
 {
     static const auto retval = lnav::pcre2pp::code::from_const(
-        R"(\x1b\[([\d=;\?]*)([a-zA-Z])|\x1b\](\d+);(.*?)(?:\x07|\x1b\\)|(?:\X\x08\X)+)");
+        R"(\x1b\[([\d=;\?]*)([a-zA-Z])|\x1b\](\d+);(.*?)(?:\x07|\x1b\\)|(?:\X\x08\X)+|(\x16+))");
 
     return retval;
 }
@@ -370,7 +370,7 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
                 }
             }
         }
-        if (md[1] || md[3]) {
+        if (md[1] || md[3] || md[5]) {
             str.erase(str.begin() + sf.sf_begin, str.begin() + sf.sf_end);
             if (sa != nullptr) {
                 shift_string_attrs(*sa, sf.sf_begin, -sf.length());

src/data_scanner.cc

@@ -203,6 +203,9 @@ static struct {
     {
         "gbg",
     },
+    {
+        "zwsp",
+    },
 };
 
 const char* DNT_NAMES[DNT_MAX - DNT_KEY] = {

src/data_scanner.hh

@@ -100,8 +100,9 @@ enum data_token_t {
     DT_CSI,
 
     DT_GARBAGE,
+    DT_ZERO_WIDTH_SPACE,
 
-    DT_TERMINAL_MAX = DT_GARBAGE + 1,
+    DT_TERMINAL_MAX = DT_ZERO_WIDTH_SPACE + 1,
 
     DNT_KEY = 50,
     DNT_PAIR,

src/data_scanner_re.re

@@ -120,6 +120,7 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
        NUM = [0-9];
        ALPHANUM = [a-zA-Z0-9_];
        EOF = "\x00";
+       SYN = "\x16";
        IPV4SEG  = ("25"[0-5]|("2"[0-4]|"1"{0,1}[0-9]){0,1}[0-9]);
        IPV4ADDR = (IPV4SEG"."){3,3}IPV4SEG;
        IPV6SEG  = [0-9a-fA-F]{1,4};
@@ -140,7 +141,11 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
 
        EOF { return nonstd::nullopt; }
 
-       ("f"|"u"|"r")?'"'('\\'.|[^\x00\x1b"\\]|'""')*'"' {
+       SYN+ {
+           RET(DT_ZERO_WIDTH_SPACE);
+       }
+
+       ("f"|"u"|"r")?'"'('\\'.|[^\x00\x16\x1b"\\]|'""')*'"' {
            CAPTURE(DT_QUOTED_STRING);
            switch (this->ds_input[cap_inner.c_begin]) {
            case 'f':
@@ -153,7 +158,7 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
            cap_inner.c_end -= 1;
            return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
        }
-       ("f"|"u"|"r")?'"""'[^\x00\x1b]*'"""' {
+       ("f"|"u"|"r")?'"""'[^\x00\x16\x1b]*'"""' {
            CAPTURE(DT_QUOTED_STRING);
            switch (this->ds_input[cap_inner.c_begin]) {
            case 'f':
@@ -175,7 +180,7 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
        [a-qstv-zA-QSTV-Z]"'" {
            CAPTURE(DT_WORD);
        }
-       ("f"|"u"|"r")?"'"('\\'.|"''"|[^\x00\x1b'\\])*"'"/[^sS] {
+       ("f"|"u"|"r")?"'"('\\'.|"''"|[^\x00\x16\x1b'\\])*"'"/[^sS] {
            CAPTURE(DT_QUOTED_STRING);
            if (tf == text_format_t::TF_RUST) {
                auto sf = this->to_string_fragment(cap_all);
@@ -196,7 +201,7 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
            cap_inner.c_end -= 1;
            return tokenize_result{token_out, cap_all, cap_inner, this->ds_input.data()};
        }
-       [a-zA-Z0-9]+":/""/"?[^\x00\x1b\r\n\t '"[\](){}]+[/a-zA-Z0-9\-=&?%] { RET(DT_URL); }
+       [a-zA-Z0-9]+":/""/"?[^\x00\x16\x1b\r\n\t '"[\](){}]+[/a-zA-Z0-9\-=&?%] { RET(DT_URL); }
        ("/"|"./"|"../"|[A-Z]":\\"|"\\\\")("Program Files"(" (x86)")?)?[a-zA-Z0-9_\.\-\~/\\!@#$%^&*()]* { RET(DT_PATH); }
        (SPACE|NUM)NUM":"NUM{2}/[^:] { RET(DT_TIME); }
        (SPACE|NUM)NUM?":"NUM{2}":"NUM{2}("."NUM{3,6})?/[^:] { RET(DT_TIME); }
@@ -292,7 +297,7 @@ nonstd::optional<data_scanner::tokenize_result> data_scanner::tokenize2(text_for
 
        ("re-")?[a-zA-Z][a-z']+/([\r\n\t \(\)!\*:;'\"\?,]|[\.\!,\?]SPACE|EOF) { RET(DT_WORD); }
 
-       [^\x00\x1b"; \t\r\n:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\.\\][^\x00\x1b"; \t\r\n:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\\]*("::"[^\x00\x1b"; \r\n\t:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\\]+)* {
+       [^\x00\x16\x1b"; \t\r\n:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\.\\][^\x00\x16\x1b"; \t\r\n:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\\]*("::"[^\x00\x16\x1b"; \r\n\t:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\\]+)* {
            RET(DT_SYMBOL);
        }
 

src/document.sections.cc

@@ -265,7 +265,8 @@ public:
             }
 
             auto dt = tokenize_res->tr_token;
-            element el(tokenize_res->tr_token, tokenize_res->tr_capture);
+
+            element el(dt, tokenize_res->tr_capture);
 
 #if 0
             log_debug("tok %s %s",
@@ -410,6 +411,8 @@ public:
                         this->sw_at_start = false;
                     }
                     break;
+                case DT_ZERO_WIDTH_SPACE:
+                    break;
                 default:
                     if (dt == DT_GARBAGE) {
                         garbage_count += 1;

src/line_buffer.hh

@@ -260,6 +260,13 @@ public:
 
     void enable_cache();
 
+    file_size_t get_piper_header_size() const
+    {
+        return this->lb_piper_header_size;
+    }
+
+    bool is_piper() const { return this->lb_piper_header_size > 0; }
+
     static void cleanup_cache();
 
 private:
@@ -333,7 +340,7 @@ private:
     safe_gz_indexed lb_gz_file; /*< File reader for gzipped files. */
     bool lb_bz_file{false}; /*< Flag set for bzip2 compressed files. */
     bool lb_line_metadata{false};
-    size_t lb_piper_header_size{0};
+    file_size_t lb_piper_header_size{0};
 
     auto_buffer lb_buffer{auto_buffer::alloc(DEFAULT_LINE_BUFFER_SIZE)};
     nonstd::optional<auto_buffer> lb_alt_buffer;

src/logfile.cc

@@ -954,10 +954,22 @@ logfile::read_file()
         return Err(std::string("file is too large to read"));
     }
 
-    auto retval
-        = TRY(this->lf_line_buffer.read_range({0, this->lf_stat.st_size}));
+    auto retval = std::string();
+    retval.reserve(this->lf_stat.st_size);
+
+    retval.append(this->lf_line_buffer.get_piper_header_size(), '\x16');
+    for (auto iter = this->begin(); iter != this->end(); ++iter) {
+        auto fr = this->get_file_range(iter);
+        auto sbr = TRY(this->lf_line_buffer.read_range(fr));
+
+        if (this->lf_line_buffer.is_piper()) {
+            retval.append(22, '\x16');
+        }
+        retval.append(sbr.get_data(), sbr.length());
+        retval.push_back('\n');
+    }
 
-    return Ok(to_string(retval));
+    return Ok(std::move(retval));
 }
 
 void

src/piper.looper.cc

@@ -54,7 +54,7 @@ write_timestamp(int fd, log_level_t level, off_t woff)
     gettimeofday(&tv, nullptr);
     auto fmt_res = fmt::format_to_n(time_str,
                                     sizeof(time_str),
-                                    FMT_STRING("{}.{}:{};"),
+                                    FMT_STRING("{: 12}.{:06}:{};"),
                                     tv.tv_sec,
                                     tv.tv_usec,
                                     level_names[level][0]);

src/textfile_sub_source.cc

@@ -213,6 +213,9 @@ textfile_sub_source::text_attrs_for_line(textview_curses& tc,
                             (int) (indent + 1),
                             line_range::unit::codepoint,
                         };
+                        if (this->tas_display_time_offset) {
+                            guide_lr.shift(0, 13);
+                        }
                         value_out.emplace_back(
                             guide_lr,
                             VC_BLOCK_ELEM.value(block_elem_t{

test/expected/expected.am

@@ -1166,8 +1166,12 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_text_file.sh_4226123565a53b4e3f80e602c1f294721e8e07bf.out \
     $(srcdir)/%reldir%/test_text_file.sh_5b51b55dff7332c5bee2c9b797c401c5614d574a.err \
     $(srcdir)/%reldir%/test_text_file.sh_5b51b55dff7332c5bee2c9b797c401c5614d574a.out \
+    $(srcdir)/%reldir%/test_text_file.sh_5e9320f18d066e6fc930dbbffc357af64312bd4b.err \
+    $(srcdir)/%reldir%/test_text_file.sh_5e9320f18d066e6fc930dbbffc357af64312bd4b.out \
     $(srcdir)/%reldir%/test_text_file.sh_6a24078983cf1b7a80b6fb65d5186cd125498136.err \
     $(srcdir)/%reldir%/test_text_file.sh_6a24078983cf1b7a80b6fb65d5186cd125498136.out \
+    $(srcdir)/%reldir%/test_text_file.sh_73f69c883f60761bff9f8874f61d21a189e92912.err \
+    $(srcdir)/%reldir%/test_text_file.sh_73f69c883f60761bff9f8874f61d21a189e92912.out \
     $(srcdir)/%reldir%/test_text_file.sh_7b00f32a3fff7fc2d78a87045ae842e58be88480.err \
     $(srcdir)/%reldir%/test_text_file.sh_7b00f32a3fff7fc2d78a87045ae842e58be88480.out \
     $(srcdir)/%reldir%/test_text_file.sh_87943c6be50d701a03e901f16493314c839af1ab.err \
@@ -1184,4 +1188,6 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_text_file.sh_d59b67113864ef5e77267d7fd8ad4072f5aef0fc.out \
     $(srcdir)/%reldir%/test_text_file.sh_e088ea61a5382458cc48a2607e2639e52b0be1da.err \
     $(srcdir)/%reldir%/test_text_file.sh_e088ea61a5382458cc48a2607e2639e52b0be1da.out \
+    $(srcdir)/%reldir%/test_text_file.sh_f586ef080a86dfe1f981b345bcf8d7a279b2b247.err \
+    $(srcdir)/%reldir%/test_text_file.sh_f586ef080a86dfe1f981b345bcf8d7a279b2b247.out \
     $()

test/expected/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.err

@@ -1,4 +1,4 @@
 ⓘ info: the following piper captures were found in:
 	piper-tmp/lnav-user-{uid}-work/piper
- = note: The captures currently consume 31B of disk space.  File sizes include associated metadata.
+ = note: The captures currently consume 33B of disk space.  File sizes include associated metadata.
  = help: You can reopen a capture by passing the piper URL to lnav

test/expected/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.out

@@ -1 +1 @@
-          just now  piper://p-e25e2eb68547f31e42da0818b4d0084f-000   31.0 B “[0] echo hi”
+          just now  piper://p-e25e2eb68547f31e42da0818b4d0084f-000   33.0 B “[0] echo hi”

test/expected/test_text_file.sh_5e9320f18d066e6fc930dbbffc357af64312bd4b.out

@@ -0,0 +1,42 @@
+[
+    {
+        "top_meta": {
+            "file": "{test_dir}/man_echo.txt",
+            "breadcrumbs": [
+                {
+                    "display_value": "man_echo.txt",
+                    "search_placeholder": "",
+                    "possibilities": [
+                        {
+                            "display_value": "man_echo.txt"
+                        }
+                    ]
+                },
+                {
+                    "display_value": "⋯",
+                    "search_placeholder": "",
+                    "possibilities": [
+                        {
+                            "display_value": "DESCRIPTION"
+                        },
+                        {
+                            "display_value": "EXIT STATUS"
+                        },
+                        {
+                            "display_value": "NAME"
+                        },
+                        {
+                            "display_value": "SEE ALSO"
+                        },
+                        {
+                            "display_value": "STANDARDS"
+                        },
+                        {
+                            "display_value": "SYNOPSIS"
+                        }
+                    ]
+                }
+            ]
+        }
+    }
+]

test/expected/test_text_file.sh_73f69c883f60761bff9f8874f61d21a189e92912.out

@@ -0,0 +1,43 @@
+[
+    {
+        "top_meta": {
+            "file": "stdin",
+            "anchor": "#description",
+            "breadcrumbs": [
+                {
+                    "display_value": "stdin",
+                    "search_placeholder": "",
+                    "possibilities": [
+                        {
+                            "display_value": "stdin"
+                        }
+                    ]
+                },
+                {
+                    "display_value": "DESCRIPTION",
+                    "search_placeholder": "",
+                    "possibilities": [
+                        {
+                            "display_value": "DESCRIPTION"
+                        },
+                        {
+                            "display_value": "EXIT STATUS"
+                        },
+                        {
+                            "display_value": "NAME"
+                        },
+                        {
+                            "display_value": "SEE ALSO"
+                        },
+                        {
+                            "display_value": "STANDARDS"
+                        },
+                        {
+                            "display_value": "SYNOPSIS"
+                        }
+                    ]
+                }
+            ]
+        }
+    }
+]

test/expected/test_text_file.sh_f586ef080a86dfe1f981b345bcf8d7a279b2b247.out

@@ -0,0 +1,38 @@
+ECHO(1)                     General Commands Manual                    ECHO(1)
+
+NAME
+     echo – write arguments to the standard output
+
+SYNOPSIS
+     echo [-n] [string ...]
+
+DESCRIPTION
+     The echo utility writes any specified operands, separated by single blank
+     (‘ ’) characters and followed by a newline (‘\n’) character, to the
+     standard output.
+
+     The following option is available:
+
+     -n    Do not print the trailing newline character.  This may also be
+           achieved by appending ‘\c’ to the end of the string, as is done by
+           iBCS2 compatible systems.  Note that this option as well as the
+           effect of ‘\c’ are implementation-defined in IEEE Std 1003.1-2001
+           (“POSIX.1”) as amended by Cor. 1-2002.  Applications aiming for
+           maximum portability are strongly encouraged to use printf(1) to
+           suppress the newline character.
+
+     Some shells may provide a builtin echo command which is similar or
+     identical to this utility.  Most notably, the builtin echo in sh(1) does
+     not accept the -n option.  Consult the builtin(1) manual page.
+
+EXIT STATUS
+     The echo utility exits 0 on success, and >0 if an error occurs.
+
+SEE ALSO
+     builtin(1), csh(1), printf(1), sh(1)
+
+STANDARDS
+     The echo utility conforms to IEEE Std 1003.1-2001 (“POSIX.1”) as amended
+     by Cor. 1-2002.
+
+macOS 13.5                      April 12, 2003                      macOS 13.5

test/man_echo.txt

@@ -0,0 +1,38 @@
+ECHO(1)                     General Commands Manual                    ECHO(1)
+
+NNAAMMEE
+     eecchhoo – write arguments to the standard output
+
+SSYYNNOOPPSSIISS
+     eecchhoo [--nn] [_s_t_r_i_n_g _._._.]
+
+DDEESSCCRRIIPPTTIIOONN
+     The eecchhoo utility writes any specified operands, separated by single blank
+     (‘ ’) characters and followed by a newline (‘\n’) character, to the
+     standard output.
+
+     The following option is available:
+
+     --nn    Do not print the trailing newline character.  This may also be
+           achieved by appending ‘\c’ to the end of the string, as is done by
+           iBCS2 compatible systems.  Note that this option as well as the
+           effect of ‘\c’ are implementation-defined in IEEE Std 1003.1-2001
+           (“POSIX.1”) as amended by Cor. 1-2002.  Applications aiming for
+           maximum portability are strongly encouraged to use printf(1) to
+           suppress the newline character.
+
+     Some shells may provide a builtin eecchhoo command which is similar or
+     identical to this utility.  Most notably, the builtin eecchhoo in sh(1) does
+     not accept the --nn option.  Consult the builtin(1) manual page.
+
+EEXXIITT SSTTAATTUUSS
+     The eecchhoo utility exits 0 on success, and >0 if an error occurs.
+
+SSEEEE AALLSSOO
+     builtin(1), csh(1), printf(1), sh(1)
+
+SSTTAANNDDAARRDDSS
+     The eecchhoo utility conforms to IEEE Std 1003.1-2001 (“POSIX.1”) as amended
+     by Cor. 1-2002.
+
+macOS 13.5                      April 12, 2003                      macOS 13.5

test/test_text_file.sh

@@ -40,3 +40,17 @@ run_cap_test ${lnav_test} -n \
 
 run_cap_test ${lnav_test} -n \
     ${test_dir}/pyfile_0.py
+
+run_cap_test ${lnav_test} -n \
+    ${test_dir}/man_echo.txt
+
+run_cap_test ${lnav_test} -n \
+    -c ";SELECT top_meta FROM lnav_views WHERE name = 'text'" \
+    -c ':write-json-to -' \
+    ${test_dir}/man_echo.txt
+
+run_cap_test ${lnav_test} -n \
+    -c ':goto 8' \
+    -c ";SELECT top_meta FROM lnav_views WHERE name = 'text'" \
+    -c ':write-json-to -' \
+    < ${test_dir}/man_echo.txt