[files] add a comment about log format match

Related to #466

.circleci/config.yml

@@ -0,0 +1,32 @@
+# This config was automatically generated from your source code
+# Stacks detected: cicd:github-actions:.github/workflows,deps:python:docs,deps:ruby:docs,deps:rust:src/third-party/prqlc-c
+version: 2.1
+orbs:
+  macos: circleci/macos@2.2.0
+jobs:
+  macos-apple-clang:
+    macos:
+      xcode: 14.2.0
+    resource_class: macos.m1.medium.gen1
+    steps:
+      - checkout
+      - run: brew install pcre2 sqlite ncurses xz zstd readline libarchive curl autoconf automake
+      - run: ./autogen.sh
+      - run: >-
+          ./configure \
+            --with-libcurl=$(brew --prefix curl) \
+            --with-pcre2=$(brew --prefix pcre2) \
+            --with-sqlite3=$(brew --prefix sqlite3) \
+            "CXXFLAGS=-I$(brew --prefix ncurses)/include -g2 -O2" \
+            'CFLAGS=-O2 -g2' \
+            "LDFLAGS=-L$(brew --prefix ncurses)/lib -L$(brew --prefix xz)/lib -L$(brew --prefix lz4)/lib -L$(brew --prefix zstd)/lib/" \
+            --with-readline=$(brew --prefix readline) \
+            --with-libarchive=$(brew --prefix libarchive) \
+            || cat config.log
+      - run: make -j2 || true
+      - run: env DUMP_CRASH=1 src/lnav -V
+      - run: make check -j2 || (test -e test/test-suite.log && cat test/test-suite.log && false)
+workflows:
+  build-and-test:
+    jobs:
+      - macos-apple-clang

.github/actions/muslbuilder/Dockerfile

@@ -176,7 +176,9 @@ RUN apk add --no-cache \
     autoconf \
     automake \
     build-base \
+    cargo \
     git \
+    rust \
     zip
 
 COPY entrypoint.sh /entrypoint.sh

.github/workflows/bins.yml

@@ -43,7 +43,7 @@ jobs:
         run: >-
           mkdir ${{ env.LNAV_BASENAME }} &&
           cd ${{ env.LNAV_BASENAME }} &&
-          cp ../NEWS.md ../README . &&
+          cp ../NEWS.md ../README ../lnav.1 . &&
           cp ../lbuild/src/lnav . &&
           cd .. &&
           zip -r ${{ env.LNAV_ZIPNAME }} ${{ env.LNAV_BASENAME }}
@@ -67,7 +67,7 @@ jobs:
 
   build-arm:
     runs-on: ubuntu-latest
-    if: ${{ inputs.lnav_version_number != '' }}
+    if: false
     strategy:
       matrix:
         include:
@@ -96,7 +96,7 @@ jobs:
           cd ${{ github.workspace }} &&
           mkdir ${{ env.LNAV_BASENAME }} &&
           cd ${{ env.LNAV_BASENAME }} &&
-          cp ../NEWS.md ../README . &&
+          cp ../NEWS.md ../README ../lnav.1 . &&
           cp ../lbuild/src/lnav . &&
           cd .. &&
           zip -r ${{ env.LNAV_ZIPNAME }} ${{ env.LNAV_BASENAME }}
@@ -149,7 +149,7 @@ jobs:
         run: >-
           mkdir ${{ env.LNAV_BASENAME }} &&
           cd ${{ env.LNAV_BASENAME }} &&
-          cp ../NEWS.md ../README . &&
+          cp ../NEWS.md ../README ../lnav.1 . &&
           cp ../src/lnav . &&
           cd .. &&
           zip -r ${{ env.LNAV_ZIPNAME }} ${{ env.LNAV_BASENAME }}

.github/workflows/c-cpp.yml

@@ -65,14 +65,16 @@ jobs:
           autoconf
           g++
           libpcre2-dev
-          libpcre3-dev
           libncurses-dev
           libsqlite3-dev
           libbz2-dev
           libcurl4-openssl-dev
           libreadline-dev
+          pipx
           tshark
           zlib1g-dev
+      - name: Install python packages
+        run: pipx install check-jsonschema
       - name: autogen
         run: ./autogen.sh
       - name: configure

.github/workflows/coverity.yml

@@ -15,6 +15,9 @@ on:
   # Allow for manual triggers from the web.
   workflow_dispatch:
 
+env:
+  COVERITY_SUPPRESS_ASSERT: 2564
+
 jobs:
   coverity:
     runs-on: ubuntu-latest
@@ -29,14 +32,17 @@ jobs:
           automake
           autoconf
           g++
-          libpcre3-dev
+          libpcre2-dev
           libncurses-dev
           libsqlite3-dev
           libbz2-dev
           libcurl4-openssl-dev
           libreadline-dev
+          pipx
           tshark
           zlib1g-dev
+      - name: Install python packages
+        run: pipx install check-jsonschema
       - name: autogen
         run: ./autogen.sh
       - name: configure
@@ -46,3 +52,10 @@ jobs:
           command: make -j$(getconf _NPROCESSORS_CONF)
           email: ${{ secrets.COVERITY_SCAN_EMAIL }}
           token: ${{ secrets.COVERITY_SCAN_TOKEN }}
+      - name: Upload a Build Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          # Artifact name
+          name: lnav-cov-int.zip
+          # A file, directory or wildcard pattern that describes what to upload
+          path: cov-int/build-log.txt

.gitignore

@@ -32,6 +32,7 @@ missing
 mkinstalldirs
 test-driver
 docs/build
+release/lnav.spec
 release/release-NEWS.md
 release/linux-pkg/
 release/osx-build-dir/

CMakeLists.txt

@@ -2,11 +2,11 @@ cmake_minimum_required(VERSION 3.14)
 
 include(cmake/prelude.cmake)
 
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
 project(
         lnav
-        VERSION 0.12.0
-        DESCRIPTION "An advanced log file viewer for the small-scale."
+        VERSION 0.12.3
+        DESCRIPTION "An advanced log file viewer for the terminal."
         HOMEPAGE_URL "https://lnav.org/"
         LANGUAGES CXX C
 )
@@ -14,24 +14,24 @@ project(
 include(cmake/project-is-top-level.cmake)
 include(cmake/variables.cmake)
 
-find_package(SQLite3 REQUIRED)
+set(CURSES_NEED_NCURSES TRUE)
+find_package(PkgConfig REQUIRED)
+find_package(unofficial-sqlite3 CONFIG REQUIRED)
 find_package(BZip2 REQUIRED)
 find_package(LibArchive REQUIRED)
 find_package(ZLIB REQUIRED)
-find_package(pcre REQUIRED)
-find_package(pcre2 REQUIRED)
-find_package(readline REQUIRED)
-find_package(ncurses REQUIRED)
+find_package(pcre2 CONFIG REQUIRED)
+pkg_check_modules(readline REQUIRED IMPORTED_TARGET readline)
+find_package(Curses REQUIRED)
 find_package(CURL REQUIRED)
 
 set(lnav_LIBS
         CURL::libcurl
-        SQLite::SQLite3
+        unofficial::sqlite3::sqlite3
         BZip2::BZip2
-        ncurses::libcurses
-        pcre::libpcre
-        pcre2::pcre2
-        readline::readline
+        ${CURSES_LIBRARIES}
+        PCRE2::8BIT PCRE2::16BIT PCRE2::32BIT PCRE2::POSIX
+        PkgConfig::readline
         LibArchive::LibArchive
         ZLIB::ZLIB
 )

CMakePresets.json

@@ -1,142 +1,13 @@
 {
-  "version": 2,
-  "cmakeMinimumRequired": {
-    "major": 3,
-    "minor": 14,
-    "patch": 0
-  },
-  "configurePresets": [
-    {
-      "name": "cmake-pedantic",
-      "hidden": true,
-      "warnings": {
-        "dev": true,
-        "deprecated": true,
-        "uninitialized": true,
-        "unusedCli": true,
-        "systemVars": false
-      },
-      "errors": {
-        "dev": true,
-        "deprecated": true
-      }
-    },
-    {
-      "name": "dev-mode",
-      "hidden": true,
-      "inherits": "cmake-pedantic",
-      "cacheVariables": {
-        "lnav_DEVELOPER_MODE": "ON"
-      }
-    },
-    {
-      "name": "conan",
-      "hidden": true,
-      "cacheVariables": {
-        "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/conan/conan_toolchain.cmake"
-      }
-    },
-    {
-      "name": "cppcheck",
-      "hidden": true,
-      "cacheVariables": {
-        "CMAKE_CXX_CPPCHECK": "cppcheck;--inline-suppr"
-      }
-    },
-    {
-      "name": "clang-tidy",
-      "hidden": true,
-      "cacheVariables": {
-        "CMAKE_CXX_CLANG_TIDY": "clang-tidy;--header-filter=${sourceDir}/*"
-      }
-    },
-    {
-      "name": "ci-std",
-      "description": "This preset makes sure the project actually builds with at least the specified standard",
-      "hidden": true,
-      "cacheVariables": {
-        "CMAKE_CXX_EXTENSIONS": "OFF",
-        "CMAKE_CXX_STANDARD": "14",
-        "CMAKE_CXX_STANDARD_REQUIRED": "ON"
-      }
-    },
-    {
-      "name": "flags-unix",
-      "hidden": true,
-      "cacheVariables": {
-        "CMAKE_CXX_FLAGS": ""
-      }
-    },
-    {
-      "name": "flags-windows",
-      "hidden": true,
-      "cacheVariables": {
-        "CMAKE_CXX_FLAGS": "/W4 /permissive- /utf-8 /volatile:iso /EHsc /Zc:__cplusplus /Zc:throwingNew"
-      }
-    },
-    {
-      "name": "ci-unix",
-      "generator": "Unix Makefiles",
-      "hidden": true,
-      "inherits": ["flags-unix", "ci-std"],
-      "cacheVariables": {
-        "CMAKE_BUILD_TYPE": "Release"
-      }
-    },
-    {
-      "name": "ci-win64",
-      "inherits": ["flags-windows", "ci-std"],
-      "generator": "Visual Studio 17 2022",
-      "architecture": "x64",
-      "hidden": true
-    },
-    {
-      "name": "coverage-unix",
-      "binaryDir": "${sourceDir}/build/coverage",
-      "inherits": "ci-unix",
-      "hidden": true,
-      "cacheVariables": {
-        "ENABLE_COVERAGE": "ON",
-        "CMAKE_BUILD_TYPE": "Coverage",
-        "CMAKE_CXX_FLAGS_COVERAGE": "-Og -g --coverage -fkeep-inline-functions -fkeep-static-functions",
-        "CMAKE_EXE_LINKER_FLAGS_COVERAGE": "--coverage",
-        "CMAKE_SHARED_LINKER_FLAGS_COVERAGE": "--coverage",
-        "CMAKE_MAP_IMPORTED_CONFIG_SANITIZE": "Coverage;RelWithDebInfo;Release;Debug;"
-      }
-    },
-    {
-      "name": "ci-coverage",
-      "inherits": ["coverage-unix", "dev-mode", "conan"],
-      "cacheVariables": {
-        "COVERAGE_HTML_COMMAND": ""
-      }
-    },
-    {
-      "name": "ci-sanitize",
-      "binaryDir": "${sourceDir}/build/sanitize",
-      "inherits": ["ci-unix", "dev-mode", "conan"],
-      "cacheVariables": {
-        "CMAKE_BUILD_TYPE": "Sanitize",
-        "CMAKE_CXX_FLAGS_SANITIZE": "-O2 -g -fsanitize=address,undefined -fno-omit-frame-pointer -fno-common",
-        "CMAKE_MAP_IMPORTED_CONFIG_SANITIZE": "Sanitize;RelWithDebInfo;Release;Debug;"
-      }
-    },
-    {
-      "name": "ci-build",
-      "binaryDir": "${sourceDir}/build",
-      "hidden": true
-    },
-    {
-      "name": "ci-macos",
-      "inherits": ["ci-build", "ci-unix", "dev-mode", "conan"]
-    },
-    {
-      "name": "ci-ubuntu",
-      "inherits": ["ci-build", "ci-unix", "clang-tidy", "conan", "dev-mode"]
-    },
-    {
-      "name": "ci-windows",
-      "inherits": ["ci-build", "ci-win64", "dev-mode", "conan"]
-    }
-  ]
-}
+    "version": 2,
+    "configurePresets": [
+        {
+            "name": "default",
+            "generator": "Ninja",
+            "binaryDir": "${sourceDir}/build",
+            "cacheVariables": {
+                "CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake"
+            }
+        }
+    ]
+}

Makefile.am

@@ -10,6 +10,12 @@ dist_man_MANS = lnav.1
 EXTRA_DIST = \
     docs/lnav-architecture.png \
     docs/lnav-tui.png \
+    docs/schemas/config-v1.schema.json \
+    docs/schemas/event-file-format-detected-v1.schema.json \
+    docs/schemas/event-file-open-v1.schema.json \
+    docs/schemas/event-log-msg-detected-v1.schema.json \
+    docs/schemas/event-session-loaded-v1.schema.json \
+    docs/schemas/format-v1.schema.json \
     ARCHITECTURE.md \
 	AUTHORS \
 	LICENSE \

NEWS.md

@@ -1,3 +1,233 @@
+## lnav v0.12.3
+
+Features:
+* Files that contain a mixture of log messages from separate
+  services (e.g. docker logs) can now be automatically
+  de-multiplexed into separate files that lnav can digest.
+* The `log_opid` column on log vtables can now be `UPDATE`d
+  so that you can manually set an opid on log messages that
+  don't have one.  Setting an opid allows messages to show
+  up in the timeline view.
+* The Files panel now has a details view on the right side
+  that shows extra information about the selected file.
+  You can look here for details of why lnav selected a
+  particular log format.
+* Add support for GitHub Markdown Alerts.
+* Added the `:xopen` command that will open the given paths
+  using an external opener like `open` or `xdg-open`.
+* Clicking on a link in a markdown file will open the Actions
+  with the following options:
+  - opening the link target in lnav or, if it's an lnav script,
+    executing the script;
+  - opening the target with `:xopen`;
+  - or, copying the link to the clipboard.
+* Added a `crash upload` command to the management CLI that will
+  upload crash logs to a server for analysis.
+* Added a `:set-text-view-mode` command that controls whether
+  file contents, such as markdown, are rendered or shown in
+  their raw state.
+* Text files with lines longer than 240 characters will be
+  automatically pretty-printed.  You can revert to the raw view
+  using the `:set-text-view-mode` command.
+* Added a `pretty_print()` SQL function that provides the same
+  functionality as the PRETTY view.
+* Keymap definitions can now bind to a function key using an
+  identifier that starts with `f` followed by the number of the
+  function key.
+
+Interface Changes:
+* The "Gantt Chart" view has been renamed to "timeline".
+* In the timeline view, pressing `ENTER` will focus on
+  the preview pane, so you can scroll through messages
+  with the selected Op ID.
+* With mouse mode enabled, `CTRL` can be used as an alternate
+  to `SHIFT` when clicking/dragging in the main view to
+  highlight lines.  A few terminals capture shift+clicks as a
+  way to select text and do not pass them to the application.
+* Clicking on an internal link in a Markdown document will move 
+  to that section.
+
+Bug Fixes:
+* Log messages in formats with custom timestamp formats were
+  not being converted to the local timezone.
+* The timezone offset is now shown in the parser details
+  overlay for log messages.
+* If a theme does not define `cursor-line` or `selected-text`
+  styles, the styles from the default theme will be used.
+* The first argument to a script is now the full path of the
+  script and not just the script name.
+
+Maintenance:
+* You can now do an `UPDATE` on the `lnav_top_view` SQL view.
+  This makes it easier to write queries that manipulate the
+  current view.
+* Upgrade to C++17
+
+
+## lnav v0.12.2
+
+Features:
+* Added mouse support that can be toggled with `F2` or enabled
+  by default with: `:config /ui/mouse/mode enabled`.  With
+  mouse support enabled, many of the UI elements will respond to
+  mouse inputs:
+  - clicking on the main view will move the cursor to the given
+    row and dragging will scroll the view as needed;
+  - shift + clicking/dragging in the main view will highlight
+    lines and then toggle their bookmark status on release;
+  - double-clicking in the main view will select the underlying 
+    text and drag-selecting within a line will select the given
+    text;
+  - when double-clicking text: if the mouse pointer is inside
+    a quoted string, the contents of the string will be selected;
+    if the mouse pointer is on the quote, the quote will be included
+    in the selection; if the mouse pointer is over a bracket
+    (e.g. [],{},()) where the matching bracket is on the same line,
+    the selection will span from one bracket to the other;
+  - when text is selected, a menu will pop up that can be used
+    to filter based on the current text, search for it, or copy
+    it to the clipboard;
+  - right-clicking the start of a log message in the main view
+    will open the parser details overlay;
+  - the parser details now displays a diamond next to fields to
+    indicate whether they are shown/hidden and this can be
+    clicked to toggle the state;
+  - the parser details will show a bar chart icon for fields with
+    values which, when clicked, will open either the spectrogram
+    view for the given field or open the DB query prompt with a
+    PRQL query to generate a histogram of the field values;
+  - clicking in the scroll area will move the view by a page,
+    double-clicking will move the view to that area, and
+    dragging the scrollbar will move the view to the given spot;
+  - clicking on the breadcrumb bar will select a crumb and
+    selecting a possibility from the popup will move to that
+    location in the view;
+  - clicking on portions of the bottom status bar will trigger
+    a relevant action (e.g. clicking the line number will open
+    the command prompt with `:goto <current-line>`);
+  - clicking on the configuration panel tabs (i.e. Files/Filters)
+    will open the selected panel and clicking parts of the
+    display in there will perform the relevant action (e.g.
+    clicking the diamond will enable/disable the file/filter);
+  - clicking in a prompt will move the cursor to the location;
+  - clicking on a column in the spectrogram view will select it.
+
+  (Note that this is new work, so there are likely to be some 
+  glitches.)
+* Added a `journald://` URL handler that will call `journalctl`
+  and pass any query parameters as options.  For example, the
+  following command:
+
+  ```
+  $ lnav 'journal://?since=yesterday'
+  ```
+
+  Will execute the following and capture the output:
+
+  ```
+  journalctl --output=json -f --since=yesterday
+  ```
+* Added the "last-word" line-format field shortening algorithm
+  from @flicus.
+* Added a `stats.hist` PRQL transform that produces a histogram
+  of values over time.
+* The preview for the `:open` command will now show a listing
+  of archive contents.
+* Added `humanize_id` SQL function that colorizes a string using
+  ANSI escape codes.
+* Added a `selected_text` column to the `lnav_views` table that
+  reports information about text that was selected with a mouse.
+  This makes it possible to script operations that use the
+  selected text as an input.
+* Added `breadcrumb` as an option to the `:prompt` command so
+  that the breadcrumb hotkey can be configured.
+
+Interface changes:
+* The bar charts in the DB view have now been moved to their
+  individual columns instead of occupying the whole width of
+  the view.  The result is much cleaner, so the charts are
+  now enabled by default again.
+* Cursor mode in the main view is now the default instead of
+  using the top line as the focus.  You can change back by
+  running:
+
+  `:config /ui/movement/mode top`
+* In the parser details panel (opened by pressing `p`), you
+  can now hide/show fields by moving the cursor line to the
+  given field and pressing the space bar or by clicking on
+  the diamond with the mouse.
+* The `sv` keymap binds `§` to focus the breadcrumb bar.
+
+Bug Fixes:
+* With the recent xz backdoor shenanigans, it seems like a good
+  time to add some checks for data being hidden by escape codes:
+  - File names with escape sequences are now displayed in quotes
+    with backslash escapes.
+  - Text that has the same foreground and background colors will
+    have the background set to a contrasting color.
+* Sub-millisecond time values should now be preserved when
+  displaying JSON-lines logs.
+* A crash during initialization on Apple Silicon and MacOS 12
+  has been fixed.
+* A crash when previewing non-text files.
+* Optimized ANSI-escape processing.
+* Various fixes to make lnav usable as a `PAGER`.
+
+## lnav v0.12.1
+
+Features:
+* Database queries can now be written in
+  [PRQL](https://prql-lang.org).  When executing a query with `;`,
+  if the query starts with `from`, it will be treated as PRQL.
+  The pipeline structure of PRQL queries is more desirable for
+  interactive use since lnav can make better suggestions and
+  show previews of the stages of the pipeline.
+* Log partitions can automatically be created by defining a log
+  message pattern in a log format.  Under a format definition,
+  add an entry into the "partitions" object in a format definition.
+  The "pattern" property specifies the regular expression to match
+  against a line in a file that matches the format.  If a match is
+  found, the partition name will be set to the value(s) captured
+  by the regex.  To restrict matches to certain files, you can add
+  a "paths" array whose object elements contain a "glob" property
+  that will be matched against file names.
+
+Interface changes:
+* When using PRQL in the database query prompt (`;`),
+  the preview pane will show the results for the pipeline
+  stage the cursor is within along with the results of
+  the previous stage (if there is one).  The preview
+  works on a limited data set, so the preview results
+  may differ from the final results.
+* Changed the breadcrumb bar styling to space things out
+  more and make the divisions between items clearer.
+* The `ESC` key can now be used to exit the files/filters
+  configuration panel instead of `q`.  This should make
+  it easier to avoid accidentally exiting lnav.
+* Added some default help text for the command prompt.
+* Suggestions are now shown for some commands and can
+  be accepted by pressing the right arrow key.  For
+  example, after typing in `:filter-in` the current
+  search term for the view will be suggested (if
+  one is active).
+* The focused line should be preserved more reliably in
+  the LOG/TEXT views.
+* In the LOG view, the current partition name (as set
+  with the `:partition-name` command) is shown as the
+  first breadcrumb in the breadcrumb bar.  And, when
+  that breadcrumb is selected, you can select another
+  partition to jump to.
+* The `{` / `}` hotkeys, `:next-section`, and `:prev-section`
+  commands now work in the LOG view and take you to the
+  next/previous partition.
+* The DB view now defaults to not showing bar charts.
+
+Breaking changes:
+* Many of the lesser used column in the log format tables
+  (e.g. `log_tags`) have been moved to after the columns
+  defined by the format.  These columns are usually `NULL`
+  and are a distraction when previewing queries.
+
 ## lnav v0.12.0
 
 Features:

README.md

@@ -182,6 +182,7 @@ The following software packages are required to build lnav:
 - libcurl    - The cURL library for downloading files from URLs.  Version 7.23.0 or higher is required.
 - libarchive - The libarchive library for opening archive files, like zip/tgz.
 - wireshark  - The 'tshark' program is used to interpret pcap files.
+- cargo/rust - The Rust language is used to build the PRQL compiler.
 
 #### Build
 

TESTS_ENVIRONMENT.in

@@ -1,6 +1,7 @@
 #! /bin/bash
 
 unset NO_COLOR
+unset XDG_CONFIG_HOME
 
 top_srcdir="@abssrcdir@"
 export top_srcdir
@@ -37,9 +38,6 @@ export TSHARK_CMD
 LIBARCHIVE_LIBS="@LIBARCHIVE_LIBS@"
 export LIBARCHIVE_LIBS
 
-HOME="${top_builddir}/test"
-export HOME
-
 # The full path of the test case
 test_file=$1
 # The base name of the test case
@@ -47,6 +45,12 @@ test_file_base=`basename $1`
 # The current test number for shell based tests.
 test_num=0
 
+HOME="${top_builddir}/test/cfg/${test_file_base}"
+export HOME
+
+rm -rf "${top_builddir}/test/cfg/${test_file_base}"
+mkdir -p ${HOME}
+
 test_hash=""
 
 lnav_test="${top_builddir}/src/lnav-test"

cmake/Hunter/config.cmake

@@ -1,14 +0,0 @@
-
-hunter_config(
-    libpcre
-    VERSION 8.41
-    CMAKE_ARGS
-    EXTRA_FLAGS=--enable-unicode-properties --enable-jit --enable-utf
-)
-
-hunter_config(
-    readline
-        VERSION 6.3
-    CMAKE_ARGS
-    EXTRA_FLAGS=CFLAGS=-Wno-implicit-function-declaration
-)

cmake/HunterGate.cmake

@@ -1,528 +0,0 @@
-# Copyright (c) 2013-2019, Ruslan Baratov
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# * Redistributions of source code must retain the above copyright notice, this
-#   list of conditions and the following disclaimer.
-#
-# * Redistributions in binary form must reproduce the above copyright notice,
-#   this list of conditions and the following disclaimer in the documentation
-#   and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# This is a gate file to Hunter package manager.
-# Include this file using `include` command and add package you need, example:
-#
-#     cmake_minimum_required(VERSION 3.2)
-#
-#     include("cmake/HunterGate.cmake")
-#     HunterGate(
-#         URL "https://github.com/path/to/hunter/archive.tar.gz"
-#         SHA1 "798501e983f14b28b10cda16afa4de69eee1da1d"
-#     )
-#
-#     project(MyProject)
-#
-#     hunter_add_package(Foo)
-#     hunter_add_package(Boo COMPONENTS Bar Baz)
-#
-# Projects:
-#     * https://github.com/hunter-packages/gate/
-#     * https://github.com/ruslo/hunter
-
-option(HUNTER_ENABLED "Enable Hunter package manager support" ON)
-
-if(HUNTER_ENABLED)
-  if(CMAKE_VERSION VERSION_LESS "3.2")
-    message(
-        FATAL_ERROR
-        "At least CMake version 3.2 required for Hunter dependency management."
-        " Update CMake or set HUNTER_ENABLED to OFF."
-    )
-  endif()
-endif()
-
-include(CMakeParseArguments) # cmake_parse_arguments
-
-option(HUNTER_STATUS_PRINT "Print working status" ON)
-option(HUNTER_STATUS_DEBUG "Print a lot info" OFF)
-option(HUNTER_TLS_VERIFY "Enable/disable TLS certificate checking on downloads" ON)
-
-set(HUNTER_ERROR_PAGE "https://docs.hunter.sh/en/latest/reference/errors")
-
-function(hunter_gate_status_print)
-  if(HUNTER_STATUS_PRINT OR HUNTER_STATUS_DEBUG)
-    foreach(print_message ${ARGV})
-      message(STATUS "[hunter] ${print_message}")
-    endforeach()
-  endif()
-endfunction()
-
-function(hunter_gate_status_debug)
-  if(HUNTER_STATUS_DEBUG)
-    foreach(print_message ${ARGV})
-      string(TIMESTAMP timestamp)
-      message(STATUS "[hunter *** DEBUG *** ${timestamp}] ${print_message}")
-    endforeach()
-  endif()
-endfunction()
-
-function(hunter_gate_error_page error_page)
-  message("------------------------------ ERROR ------------------------------")
-  message("    ${HUNTER_ERROR_PAGE}/${error_page}.html")
-  message("-------------------------------------------------------------------")
-  message("")
-  message(FATAL_ERROR "")
-endfunction()
-
-function(hunter_gate_internal_error)
-  message("")
-  foreach(print_message ${ARGV})
-    message("[hunter ** INTERNAL **] ${print_message}")
-  endforeach()
-  message("[hunter ** INTERNAL **] [Directory:${CMAKE_CURRENT_LIST_DIR}]")
-  message("")
-  hunter_gate_error_page("error.internal")
-endfunction()
-
-function(hunter_gate_fatal_error)
-  cmake_parse_arguments(hunter "" "ERROR_PAGE" "" "${ARGV}")
-  if("${hunter_ERROR_PAGE}" STREQUAL "")
-    hunter_gate_internal_error("Expected ERROR_PAGE")
-  endif()
-  message("")
-  foreach(x ${hunter_UNPARSED_ARGUMENTS})
-    message("[hunter ** FATAL ERROR **] ${x}")
-  endforeach()
-  message("[hunter ** FATAL ERROR **] [Directory:${CMAKE_CURRENT_LIST_DIR}]")
-  message("")
-  hunter_gate_error_page("${hunter_ERROR_PAGE}")
-endfunction()
-
-function(hunter_gate_user_error)
-  hunter_gate_fatal_error(${ARGV} ERROR_PAGE "error.incorrect.input.data")
-endfunction()
-
-function(hunter_gate_self root version sha1 result)
-  string(COMPARE EQUAL "${root}" "" is_bad)
-  if(is_bad)
-    hunter_gate_internal_error("root is empty")
-  endif()
-
-  string(COMPARE EQUAL "${version}" "" is_bad)
-  if(is_bad)
-    hunter_gate_internal_error("version is empty")
-  endif()
-
-  string(COMPARE EQUAL "${sha1}" "" is_bad)
-  if(is_bad)
-    hunter_gate_internal_error("sha1 is empty")
-  endif()
-
-  string(SUBSTRING "${sha1}" 0 7 archive_id)
-
-  set(
-      hunter_self
-      "${root}/_Base/Download/Hunter/${version}/${archive_id}/Unpacked"
-  )
-
-  set("${result}" "${hunter_self}" PARENT_SCOPE)
-endfunction()
-
-# Set HUNTER_GATE_ROOT cmake variable to suitable value.
-function(hunter_gate_detect_root)
-  # Check CMake variable
-  string(COMPARE NOTEQUAL "${HUNTER_ROOT}" "" not_empty)
-  if(not_empty)
-    set(HUNTER_GATE_ROOT "${HUNTER_ROOT}" PARENT_SCOPE)
-    hunter_gate_status_debug("HUNTER_ROOT detected by cmake variable")
-    return()
-  endif()
-
-  # Check environment variable
-  string(COMPARE NOTEQUAL "$ENV{HUNTER_ROOT}" "" not_empty)
-  if(not_empty)
-    set(HUNTER_GATE_ROOT "$ENV{HUNTER_ROOT}" PARENT_SCOPE)
-    hunter_gate_status_debug("HUNTER_ROOT detected by environment variable")
-    return()
-  endif()
-
-  # Check HOME environment variable
-  string(COMPARE NOTEQUAL "$ENV{HOME}" "" result)
-  if(result)
-    set(HUNTER_GATE_ROOT "$ENV{HOME}/.hunter" PARENT_SCOPE)
-    hunter_gate_status_debug("HUNTER_ROOT set using HOME environment variable")
-    return()
-  endif()
-
-  # Check SYSTEMDRIVE and USERPROFILE environment variable (windows only)
-  if(WIN32)
-    string(COMPARE NOTEQUAL "$ENV{SYSTEMDRIVE}" "" result)
-    if(result)
-      set(HUNTER_GATE_ROOT "$ENV{SYSTEMDRIVE}/.hunter" PARENT_SCOPE)
-      hunter_gate_status_debug(
-          "HUNTER_ROOT set using SYSTEMDRIVE environment variable"
-      )
-      return()
-    endif()
-
-    string(COMPARE NOTEQUAL "$ENV{USERPROFILE}" "" result)
-    if(result)
-      set(HUNTER_GATE_ROOT "$ENV{USERPROFILE}/.hunter" PARENT_SCOPE)
-      hunter_gate_status_debug(
-          "HUNTER_ROOT set using USERPROFILE environment variable"
-      )
-      return()
-    endif()
-  endif()
-
-  hunter_gate_fatal_error(
-      "Can't detect HUNTER_ROOT"
-      ERROR_PAGE "error.detect.hunter.root"
-  )
-endfunction()
-
-function(hunter_gate_download dir)
-  string(
-      COMPARE
-      NOTEQUAL
-      "$ENV{HUNTER_DISABLE_AUTOINSTALL}"
-      ""
-      disable_autoinstall
-  )
-  if(disable_autoinstall AND NOT HUNTER_RUN_INSTALL)
-    hunter_gate_fatal_error(
-        "Hunter not found in '${dir}'"
-        "Set HUNTER_RUN_INSTALL=ON to auto-install it from '${HUNTER_GATE_URL}'"
-        "Settings:"
-        "  HUNTER_ROOT: ${HUNTER_GATE_ROOT}"
-        "  HUNTER_SHA1: ${HUNTER_GATE_SHA1}"
-        ERROR_PAGE "error.run.install"
-    )
-  endif()
-  string(COMPARE EQUAL "${dir}" "" is_bad)
-  if(is_bad)
-    hunter_gate_internal_error("Empty 'dir' argument")
-  endif()
-
-  string(COMPARE EQUAL "${HUNTER_GATE_SHA1}" "" is_bad)
-  if(is_bad)
-    hunter_gate_internal_error("HUNTER_GATE_SHA1 empty")
-  endif()
-
-  string(COMPARE EQUAL "${HUNTER_GATE_URL}" "" is_bad)
-  if(is_bad)
-    hunter_gate_internal_error("HUNTER_GATE_URL empty")
-  endif()
-
-  set(done_location "${dir}/DONE")
-  set(sha1_location "${dir}/SHA1")
-
-  set(build_dir "${dir}/Build")
-  set(cmakelists "${dir}/CMakeLists.txt")
-
-  hunter_gate_status_debug("Locking directory: ${dir}")
-  file(LOCK "${dir}" DIRECTORY GUARD FUNCTION)
-  hunter_gate_status_debug("Lock done")
-
-  if(EXISTS "${done_location}")
-    # while waiting for lock other instance can do all the job
-    hunter_gate_status_debug("File '${done_location}' found, skip install")
-    return()
-  endif()
-
-  file(REMOVE_RECURSE "${build_dir}")
-  file(REMOVE_RECURSE "${cmakelists}")
-
-  file(MAKE_DIRECTORY "${build_dir}") # check directory permissions
-
-  # Disabling languages speeds up a little bit, reduces noise in the output
-  # and avoids path too long windows error
-  file(
-      WRITE
-      "${cmakelists}"
-      "cmake_minimum_required(VERSION 3.2)\n"
-      "project(HunterDownload LANGUAGES NONE)\n"
-      "include(ExternalProject)\n"
-      "ExternalProject_Add(\n"
-      "    Hunter\n"
-      "    URL\n"
-      "    \"${HUNTER_GATE_URL}\"\n"
-      "    URL_HASH\n"
-      "    SHA1=${HUNTER_GATE_SHA1}\n"
-      "    DOWNLOAD_DIR\n"
-      "    \"${dir}\"\n"
-      "    TLS_VERIFY\n"
-      "    ${HUNTER_TLS_VERIFY}\n"
-      "    SOURCE_DIR\n"
-      "    \"${dir}/Unpacked\"\n"
-      "    CONFIGURE_COMMAND\n"
-      "    \"\"\n"
-      "    BUILD_COMMAND\n"
-      "    \"\"\n"
-      "    INSTALL_COMMAND\n"
-      "    \"\"\n"
-      ")\n"
-  )
-
-  if(HUNTER_STATUS_DEBUG)
-    set(logging_params "")
-  else()
-    set(logging_params OUTPUT_QUIET)
-  endif()
-
-  hunter_gate_status_debug("Run generate")
-
-  # Need to add toolchain file too.
-  # Otherwise on Visual Studio + MDD this will fail with error:
-  # "Could not find an appropriate version of the Windows 10 SDK installed on this machine"
-  if(EXISTS "${CMAKE_TOOLCHAIN_FILE}")
-    get_filename_component(absolute_CMAKE_TOOLCHAIN_FILE "${CMAKE_TOOLCHAIN_FILE}" ABSOLUTE)
-    set(toolchain_arg "-DCMAKE_TOOLCHAIN_FILE=${absolute_CMAKE_TOOLCHAIN_FILE}")
-  else()
-    # 'toolchain_arg' can't be empty
-    set(toolchain_arg "-DCMAKE_TOOLCHAIN_FILE=")
-  endif()
-
-  string(COMPARE EQUAL "${CMAKE_MAKE_PROGRAM}" "" no_make)
-  if(no_make)
-    set(make_arg "")
-  else()
-    # Test case: remove Ninja from PATH but set it via CMAKE_MAKE_PROGRAM
-    set(make_arg "-DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}")
-  endif()
-
-  execute_process(
-      COMMAND
-      "${CMAKE_COMMAND}"
-      "-H${dir}"
-      "-B${build_dir}"
-      "-G${CMAKE_GENERATOR}"
-      "${toolchain_arg}"
-      ${make_arg}
-      WORKING_DIRECTORY "${dir}"
-      RESULT_VARIABLE download_result
-      ${logging_params}
-  )
-
-  if(NOT download_result EQUAL 0)
-    hunter_gate_internal_error(
-        "Configure project failed."
-        "To reproduce the error run: ${CMAKE_COMMAND} -H${dir} -B${build_dir} -G${CMAKE_GENERATOR} ${toolchain_arg} ${make_arg}"
-        "In directory ${dir}"
-    )
-  endif()
-
-  hunter_gate_status_print(
-      "Initializing Hunter workspace (${HUNTER_GATE_SHA1})"
-      "  ${HUNTER_GATE_URL}"
-      "  -> ${dir}"
-  )
-  execute_process(
-      COMMAND "${CMAKE_COMMAND}" --build "${build_dir}"
-      WORKING_DIRECTORY "${dir}"
-      RESULT_VARIABLE download_result
-      ${logging_params}
-  )
-
-  if(NOT download_result EQUAL 0)
-    hunter_gate_internal_error("Build project failed")
-  endif()
-
-  file(REMOVE_RECURSE "${build_dir}")
-  file(REMOVE_RECURSE "${cmakelists}")
-
-  file(WRITE "${sha1_location}" "${HUNTER_GATE_SHA1}")
-  file(WRITE "${done_location}" "DONE")
-
-  hunter_gate_status_debug("Finished")
-endfunction()
-
-# Must be a macro so master file 'cmake/Hunter' can
-# apply all variables easily just by 'include' command
-# (otherwise PARENT_SCOPE magic needed)
-macro(HunterGate)
-  if(HUNTER_GATE_DONE)
-    # variable HUNTER_GATE_DONE set explicitly for external project
-    # (see `hunter_download`)
-    set_property(GLOBAL PROPERTY HUNTER_GATE_DONE YES)
-  endif()
-
-  # First HunterGate command will init Hunter, others will be ignored
-  get_property(_hunter_gate_done GLOBAL PROPERTY HUNTER_GATE_DONE SET)
-
-  if(NOT HUNTER_ENABLED)
-    # Empty function to avoid error "unknown function"
-    function(hunter_add_package)
-    endfunction()
-
-    set(
-        _hunter_gate_disabled_mode_dir
-        "${CMAKE_CURRENT_LIST_DIR}/cmake/Hunter/disabled-mode"
-    )
-    if(EXISTS "${_hunter_gate_disabled_mode_dir}")
-      hunter_gate_status_debug(
-          "Adding \"disabled-mode\" modules: ${_hunter_gate_disabled_mode_dir}"
-      )
-      list(APPEND CMAKE_PREFIX_PATH "${_hunter_gate_disabled_mode_dir}")
-    endif()
-  elseif(_hunter_gate_done)
-    hunter_gate_status_debug("Secondary HunterGate (use old settings)")
-    hunter_gate_self(
-        "${HUNTER_CACHED_ROOT}"
-        "${HUNTER_VERSION}"
-        "${HUNTER_SHA1}"
-        _hunter_self
-    )
-    include("${_hunter_self}/cmake/Hunter")
-  else()
-    set(HUNTER_GATE_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}")
-
-    string(COMPARE NOTEQUAL "${PROJECT_NAME}" "" _have_project_name)
-    if(_have_project_name)
-      hunter_gate_fatal_error(
-          "Please set HunterGate *before* 'project' command. "
-          "Detected project: ${PROJECT_NAME}"
-          ERROR_PAGE "error.huntergate.before.project"
-      )
-    endif()
-
-    cmake_parse_arguments(
-        HUNTER_GATE "LOCAL" "URL;SHA1;GLOBAL;FILEPATH" "" ${ARGV}
-    )
-
-    string(COMPARE EQUAL "${HUNTER_GATE_SHA1}" "" _empty_sha1)
-    string(COMPARE EQUAL "${HUNTER_GATE_URL}" "" _empty_url)
-    string(
-        COMPARE
-        NOTEQUAL
-        "${HUNTER_GATE_UNPARSED_ARGUMENTS}"
-        ""
-        _have_unparsed
-    )
-    string(COMPARE NOTEQUAL "${HUNTER_GATE_GLOBAL}" "" _have_global)
-    string(COMPARE NOTEQUAL "${HUNTER_GATE_FILEPATH}" "" _have_filepath)
-
-    if(_have_unparsed)
-      hunter_gate_user_error(
-          "HunterGate unparsed arguments: ${HUNTER_GATE_UNPARSED_ARGUMENTS}"
-      )
-    endif()
-    if(_empty_sha1)
-      hunter_gate_user_error("SHA1 suboption of HunterGate is mandatory")
-    endif()
-    if(_empty_url)
-      hunter_gate_user_error("URL suboption of HunterGate is mandatory")
-    endif()
-    if(_have_global)
-      if(HUNTER_GATE_LOCAL)
-        hunter_gate_user_error("Unexpected LOCAL (already has GLOBAL)")
-      endif()
-      if(_have_filepath)
-        hunter_gate_user_error("Unexpected FILEPATH (already has GLOBAL)")
-      endif()
-    endif()
-    if(HUNTER_GATE_LOCAL)
-      if(_have_global)
-        hunter_gate_user_error("Unexpected GLOBAL (already has LOCAL)")
-      endif()
-      if(_have_filepath)
-        hunter_gate_user_error("Unexpected FILEPATH (already has LOCAL)")
-      endif()
-    endif()
-    if(_have_filepath)
-      if(_have_global)
-        hunter_gate_user_error("Unexpected GLOBAL (already has FILEPATH)")
-      endif()
-      if(HUNTER_GATE_LOCAL)
-        hunter_gate_user_error("Unexpected LOCAL (already has FILEPATH)")
-      endif()
-    endif()
-
-    hunter_gate_detect_root() # set HUNTER_GATE_ROOT
-
-    # Beautify path, fix probable problems with windows path slashes
-    get_filename_component(
-        HUNTER_GATE_ROOT "${HUNTER_GATE_ROOT}" ABSOLUTE
-    )
-    hunter_gate_status_debug("HUNTER_ROOT: ${HUNTER_GATE_ROOT}")
-    if(NOT HUNTER_ALLOW_SPACES_IN_PATH)
-      string(FIND "${HUNTER_GATE_ROOT}" " " _contain_spaces)
-      if(NOT _contain_spaces EQUAL -1)
-        hunter_gate_fatal_error(
-            "HUNTER_ROOT (${HUNTER_GATE_ROOT}) contains spaces."
-            "Set HUNTER_ALLOW_SPACES_IN_PATH=ON to skip this error"
-            "(Use at your own risk!)"
-            ERROR_PAGE "error.spaces.in.hunter.root"
-        )
-      endif()
-    endif()
-
-    string(
-        REGEX
-        MATCH
-        "[0-9]+\\.[0-9]+\\.[0-9]+[-_a-z0-9]*"
-        HUNTER_GATE_VERSION
-        "${HUNTER_GATE_URL}"
-    )
-    string(COMPARE EQUAL "${HUNTER_GATE_VERSION}" "" _is_empty)
-    if(_is_empty)
-      set(HUNTER_GATE_VERSION "unknown")
-    endif()
-
-    hunter_gate_self(
-        "${HUNTER_GATE_ROOT}"
-        "${HUNTER_GATE_VERSION}"
-        "${HUNTER_GATE_SHA1}"
-        _hunter_self
-    )
-
-    set(_master_location "${_hunter_self}/cmake/Hunter")
-    get_filename_component(_archive_id_location "${_hunter_self}/.." ABSOLUTE)
-    set(_done_location "${_archive_id_location}/DONE")
-    set(_sha1_location "${_archive_id_location}/SHA1")
-
-    # Check Hunter already downloaded by HunterGate
-    if(NOT EXISTS "${_done_location}")
-      hunter_gate_download("${_archive_id_location}")
-    endif()
-
-    if(NOT EXISTS "${_done_location}")
-      hunter_gate_internal_error("hunter_gate_download failed")
-    endif()
-
-    if(NOT EXISTS "${_sha1_location}")
-      hunter_gate_internal_error("${_sha1_location} not found")
-    endif()
-    file(READ "${_sha1_location}" _sha1_value)
-    string(COMPARE EQUAL "${_sha1_value}" "${HUNTER_GATE_SHA1}" _is_equal)
-    if(NOT _is_equal)
-      hunter_gate_internal_error(
-          "Short SHA1 collision:"
-          "  ${_sha1_value} (from ${_sha1_location})"
-          "  ${HUNTER_GATE_SHA1} (HunterGate)"
-      )
-    endif()
-    if(NOT EXISTS "${_master_location}")
-      hunter_gate_user_error(
-          "Master file not found:"
-          "  ${_master_location}"
-          "try to update Hunter/HunterGate"
-      )
-    endif()
-    include("${_master_location}")
-    set_property(GLOBAL PROPERTY HUNTER_GATE_DONE YES)
-  endif()
-endmacro()

cmake/docs-ci.cmake

@@ -1,112 +0,0 @@
-cmake_minimum_required(VERSION 3.14)
-
-foreach(var IN ITEMS PROJECT_BINARY_DIR PROJECT_SOURCE_DIR)
-  if(NOT DEFINED "${var}")
-    message(FATAL_ERROR "${var} must be defined")
-  endif()
-endforeach()
-set(bin "${PROJECT_BINARY_DIR}")
-set(src "${PROJECT_SOURCE_DIR}")
-
-# ---- Dependencies ----
-
-set(mcss_SOURCE_DIR "${bin}/docs/.ci")
-if(NOT IS_DIRECTORY "${mcss_SOURCE_DIR}")
-  file(MAKE_DIRECTORY "${mcss_SOURCE_DIR}")
-  file(
-      DOWNLOAD
-      https://github.com/friendlyanon/m.css/releases/download/release-1/mcss.zip
-      "${mcss_SOURCE_DIR}/mcss.zip"
-      STATUS status
-      EXPECTED_MD5 00cd2757ebafb9bcba7f5d399b3bec7f
-  )
-  if(NOT status MATCHES "^0;")
-    message(FATAL_ERROR "Download failed with ${status}")
-  endif()
-  execute_process(
-      COMMAND "${CMAKE_COMMAND}" -E tar xf mcss.zip
-      WORKING_DIRECTORY "${mcss_SOURCE_DIR}"
-      RESULT_VARIABLE result
-  )
-  if(NOT result EQUAL "0")
-    message(FATAL_ERROR "Extraction failed with ${result}")
-  endif()
-  file(REMOVE "${mcss_SOURCE_DIR}/mcss.zip")
-endif()
-
-find_program(Python3_EXECUTABLE NAMES python3 python)
-if(NOT Python3_EXECUTABLE)
-  message(FATAL_ERROR "Python executable was not found")
-endif()
-
-# ---- Process project() call in CMakeLists.txt ----
-
-file(READ "${src}/CMakeLists.txt" content)
-
-string(FIND "${content}" "project(" index)
-if(index EQUAL "-1")
-  message(FATAL_ERROR "Could not find \"project(\"")
-endif()
-string(SUBSTRING "${content}" "${index}" -1 content)
-
-string(FIND "${content}" "\n)\n" index)
-if(index EQUAL "-1")
-  message(FATAL_ERROR "Could not find \"\\n)\\n\"")
-endif()
-string(SUBSTRING "${content}" 0 "${index}" content)
-
-file(WRITE "${bin}/docs-ci.project.cmake" "docs_${content}\n)\n")
-
-macro(list_pop_front list out)
-  list(GET "${list}" 0 "${out}")
-  list(REMOVE_AT "${list}" 0)
-endmacro()
-
-function(docs_project name)
-  cmake_parse_arguments(PARSE_ARGV 1 "" "" "VERSION;DESCRIPTION;HOMEPAGE_URL" LANGUAGES)
-  set(PROJECT_NAME "${name}" PARENT_SCOPE)
-  if(DEFINED _VERSION)
-    set(PROJECT_VERSION "${_VERSION}" PARENT_SCOPE)
-    string(REGEX MATCH "^[0-9]+(\\.[0-9]+)*" versions "${_VERSION}")
-    string(REPLACE . ";" versions "${versions}")
-    set(suffixes MAJOR MINOR PATCH TWEAK)
-    while(NOT versions STREQUAL "" AND NOT suffixes STREQUAL "")
-      list_pop_front(versions version)
-      list_pop_front(suffixes suffix)
-      set("PROJECT_VERSION_${suffix}" "${version}" PARENT_SCOPE)
-    endwhile()
-  endif()
-  if(DEFINED _DESCRIPTION)
-    set(PROJECT_DESCRIPTION "${_DESCRIPTION}" PARENT_SCOPE)
-  endif()
-  if(DEFINED _HOMEPAGE_URL)
-    set(PROJECT_HOMEPAGE_URL "${_HOMEPAGE_URL}" PARENT_SCOPE)
-  endif()
-endfunction()
-
-include("${bin}/docs-ci.project.cmake")
-
-# ---- Generate docs ----
-
-if(NOT DEFINED DOXYGEN_OUTPUT_DIRECTORY)
-  set(DOXYGEN_OUTPUT_DIRECTORY "${bin}/docs")
-endif()
-set(out "${DOXYGEN_OUTPUT_DIRECTORY}")
-
-foreach(file IN ITEMS Doxyfile conf.py)
-  configure_file("${src}/docs/${file}.in" "${bin}/docs/${file}" @ONLY)
-endforeach()
-
-set(mcss_script "${mcss_SOURCE_DIR}/documentation/doxygen.py")
-set(config "${bin}/docs/conf.py")
-
-file(REMOVE_RECURSE "${out}/html" "${out}/xml")
-
-execute_process(
-    COMMAND "${Python3_EXECUTABLE}" "${mcss_script}" "${config}"
-    WORKING_DIRECTORY "${bin}/docs"
-    RESULT_VARIABLE result
-)
-if(NOT result EQUAL "0")
-  message(FATAL_ERROR "m.css returned with ${result}")
-endif()

cmake/docs.cmake

@@ -1,46 +0,0 @@
-# ---- Dependencies ----
-
-set(extract_timestamps "")
-if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24")
-    set(extract_timestamps DOWNLOAD_EXTRACT_TIMESTAMP YES)
-endif ()
-
-include(FetchContent)
-FetchContent_Declare(
-        mcss URL
-        https://github.com/friendlyanon/m.css/releases/download/release-1/mcss.zip
-        URL_MD5 00cd2757ebafb9bcba7f5d399b3bec7f
-        SOURCE_DIR "${PROJECT_BINARY_DIR}/mcss"
-        UPDATE_DISCONNECTED YES
-        ${extract_timestamps}
-)
-FetchContent_MakeAvailable(mcss)
-
-find_package(Python3 3.6 REQUIRED)
-
-# ---- Declare documentation target ----
-
-set(
-        DOXYGEN_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/docs"
-        CACHE PATH "Path for the generated Doxygen documentation"
-)
-
-set(working_dir "${PROJECT_BINARY_DIR}/docs")
-
-foreach (file IN ITEMS Doxyfile conf.py)
-    configure_file("docs/${file}.in" "${working_dir}/${file}" @ONLY)
-endforeach ()
-
-set(mcss_script "${mcss_SOURCE_DIR}/documentation/doxygen.py")
-set(config "${working_dir}/conf.py")
-
-add_custom_target(
-        docs
-        COMMAND "${CMAKE_COMMAND}" -E remove_directory
-        "${DOXYGEN_OUTPUT_DIRECTORY}/html"
-        "${DOXYGEN_OUTPUT_DIRECTORY}/xml"
-        COMMAND "${Python3_EXECUTABLE}" "${mcss_script}" "${config}"
-        COMMENT "Building documentation using Doxygen and m.css"
-        WORKING_DIRECTORY "${working_dir}"
-        VERBATIM
-)

conanfile.py

@@ -1,58 +0,0 @@
-from conans import ConanFile
-from conan.tools.cmake import CMake, CMakeToolchain, CMakeDeps
-
-
-class LnavConan(ConanFile):
-    name = "lnav"
-    version = "0.12.0"
-    homepage = "https://lnav.org"
-    url = "https://github.com/tstack/lnav.git"
-    license = "BSD-2-Clause"
-    description = (
-        "The Log File Navigator, lnav for short, is an advanced "
-        "log file viewer for the small-scale"
-    )
-    settings = "os", "compiler", "build_type", "arch"
-    exports_sources = "*"
-    no_copy_source = True
-    requires = (
-        "bzip2/1.0.8",
-        "libarchive/3.6.0",
-        "libcurl/7.85.0",
-        "ncurses/6.3",
-        "pcre2/10.40",
-        "readline/8.1.2",
-        "sqlite3/3.38.0",
-        "zlib/1.2.12",
-    )
-    generators = ("virtualrunenv",)
-    default_options = {
-        "libarchive:with_bzip2": True,
-        "libarchive:with_lz4": True,
-        "libarchive:with_lzo": True,
-        "libarchive:with_lzma": True,
-        "libarchive:with_zstd": True,
-        "pcre2:support_jit": True,
-        "pcre2:build_pcre2_8": True,
-        "sqlite3:enable_json1": True,
-        "sqlite3:enable_soundex": True,
-        "readline:with_library": "curses",
-    }
-
-    def generate(self):
-        CMakeToolchain(self).generate()
-        CMakeDeps(self).generate()
-
-    def build(self):
-        cmake = CMake(self)
-        cmake.configure()
-        if self.settings.os == "Macos" and self.settings.arch == "armv8":
-            cmake.definitions["CMAKE_SYSTEM_PROCESSOR"] = "arm64"
-        cmake.build()
-
-    def package(self):
-        cmake = CMake(self)
-        cmake.install()
-
-    def deploy(self):
-        self.copy("*", dst="bin", src="bin")

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT([lnav],[0.12.0],[lnav@googlegroups.com],[lnav],[http://lnav.org])
+AC_INIT([lnav],[0.12.3],[lnav@googlegroups.com],[lnav],[http://lnav.org])
 AC_CONFIG_SRCDIR([src/lnav.cc])
 AC_CONFIG_MACRO_DIR([m4])
 AM_INIT_AUTOMAKE([foreign subdir-objects])
@@ -17,7 +17,7 @@ CXX="$PTHREAD_CXX"
 CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS"
 
 AC_LANG(C++)
-AX_CXX_COMPILE_STDCXX_14([noext], [mandatory])
+AX_CXX_COMPILE_STDCXX_17([noext], [mandatory])
 
 dnl abssrcdir is the absolute path to the source base (regardless of where
 dnl you are building it)
@@ -50,11 +50,13 @@ AM_PROG_AR
 AC_PROG_LN_S
 AC_PROG_MAKE_SET
 
+AC_PATH_PROG(CARGO_CMD, [cargo])
 AC_PATH_PROG(BZIP2_CMD, [bzip2])
 AC_PATH_PROG(RE2C_CMD, [re2c])
 AM_CONDITIONAL(HAVE_RE2C, test x"$RE2C_CMD" != x"")
 AC_PATH_PROG(XZ_CMD, [xz])
 AC_PATH_PROG(TSHARK_CMD, [tshark])
+AC_PATH_PROG(CHECK_JSONSCHEMA, [check-jsonschema])
 
 AC_CHECK_SIZEOF(off_t)
 AC_CHECK_SIZEOF(size_t)
@@ -265,7 +267,7 @@ AC_SUBST(STATIC_LDFLAGS)
 
 AS_CASE(["$host_os"],
     [darwin*],
-    [],
+    [LDFLAGS="$LDFLAGS -framework CoreFoundation"],
     [
     curses_lib=$(echo $CURSES_LIB | sed -e 's/-l//')
     AS_IF([test $? -eq 0],
@@ -305,9 +307,11 @@ AS_IF([test $? -eq 0],
          [VCS package string])],
     AC_DEFINE_UNQUOTED([VCS_PACKAGE_STRING], ["$PACKAGE_STRING"], [VCS package string]))
 
+AM_CONDITIONAL(HAVE_CARGO, test x"$CARGO_CMD" != x"")
 AM_CONDITIONAL(USE_INCLUDED_YAJL, test $HAVE_LOCAL_YAJL -eq 0)
 AM_CONDITIONAL(HAVE_LIBCURL, test x"$LIBCURL" != x"")
 AM_CONDITIONAL([CROSS_COMPILING], [ test x"$cross_compiling" != x"no" ])
+AM_CONDITIONAL(HAVE_CHECK_JSONSCHEMA, test x"$CHECK_JSONSCHEMA" != x"")
 
 AS_VAR_SET(USER_CXXFLAGS, ["$CXXFLAGS"])
 AC_SUBST(USER_CXXFLAGS)

crashd/Dockerfile

@@ -0,0 +1,19 @@
+# syntax=docker/dockerfile:1
+
+ARG PYTHON_VERSION=3.12.3
+
+FROM python:${PYTHON_VERSION}-slim
+
+LABEL fly_launch_runtime="flask"
+
+WORKDIR /code
+
+RUN apt-get update && apt-get install -y --no-install-recommends gcc build-essential
+COPY requirements.txt requirements.txt
+RUN pip3 install -r requirements.txt
+
+COPY . .
+
+EXPOSE 8080
+
+CMD [ "python3", "-m" , "flask", "run", "--host=0.0.0.0", "--port=8080"]

crashd/Procfile

@@ -0,0 +1 @@
+web: gunicorn app:app

crashd/app.py

@@ -0,0 +1,148 @@
+from werkzeug.middleware.proxy_fix import ProxyFix
+from flask import Flask, request, send_file, after_this_request
+import os
+import glob
+import uuid
+from datetime import datetime
+import zipfile
+import spookyhash
+
+app = Flask(__name__)
+app.wsgi_app = ProxyFix(
+    app.wsgi_app, x_for=2, x_proto=2, x_host=2, x_prefix=2
+)
+
+ROOT_PAGE = """
+<html>
+<head>
+<title>lnav crash upload site</title>
+</head>
+
+<body>
+You can help improve <b>lnav</b> by uploading your crash logs by running:
+<pre>
+lnav -m crash upload
+</pre>
+</body>
+</html>
+"""
+
+
+# Function to check free space on filesystem
+def check_free_space():
+    statvfs = os.statvfs('/logs')
+    # Calculate free space in MB
+    free_space_mb = (statvfs.f_bsize * statvfs.f_bavail) / (1024 * 1024)
+    return free_space_mb
+
+
+@app.route('/crash', methods=['POST'])
+def crash_handler():
+    # Check free space on filesystem
+    free_space_mb = check_free_space()
+    if free_space_mb < 100:
+        return 'Insufficient free space on the filesystem!\n', 500
+
+    # Retrieve the secret value from the environment variable
+    lnav_secret_env = os.environ.get('LNAV_SECRET')
+
+    # Check if header 'lnav-secret' exists and has the correct value
+    if request.headers.get('lnav-secret') == lnav_secret_env:
+        # Check if content length is provided
+        if 'Content-Length' not in request.headers:
+            return 'Content length header is missing!', 400
+
+        # Get the content length
+        content_length = int(request.headers['Content-Length'])
+
+        # Check if content length is zero
+        if content_length == 0:
+            return 'Empty request body!', 400
+
+        # Check if content length exceeds 10MB
+        if content_length > 10 * 1024 * 1024:  # 10MB limit
+            return 'Content size exceeds the limit of 10MB!', 413
+
+        # Get the content from the request body
+        content = request.data
+
+        nonce = request.headers.get('X-lnav-nonce', '')
+        crash_hash = request.headers.get('X-lnav-hash', '')
+        if not crash_hash.startswith("0000"):
+            return "Invalid proof of work hash", 401
+
+        sh = spookyhash.Hash128()
+        sh.update(nonce.encode('utf-8'))
+        sh.update(content)
+        verify_hash = sh.hexdigest()
+        if verify_hash != crash_hash:
+            return "Proof of work hash does not match", 401
+
+        # Generate a unique ID
+        unique_id = str(uuid.uuid4())
+
+        # Get the current time
+        current_time = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
+
+        # Construct the file name with current time and unique ID
+        file_name = f'crash_log_{current_time}_{unique_id}.txt'
+        full_path = os.path.join("/logs", file_name)
+
+        # Save the content to the file in the current directory
+        with open(full_path, 'wb') as f:
+            f.write(content)
+
+        return 'Data saved successfully!', 200
+    else:
+        return 'Unauthorized access!', 401
+
+
+@app.route('/download_crashes', methods=['GET'])
+def download_crashes():
+    # Retrieve the secret value for downloading from the environment variable
+    lnav_download_secret_env = os.environ.get('LNAV_DOWNLOAD_SECRET')
+
+    # Check if header 'lnav-secret' exists and has the correct value for downloading
+    if request.headers.get('lnav-secret') == lnav_download_secret_env:
+        # Get all the files in the current directory
+        crash_files = glob.glob("/logs/crash_log_*")
+
+        # Generate a unique ID for the zip file
+        zip_id = str(uuid.uuid4())
+
+        # Construct the zip file name
+        zip_file_name = f'crash_archive_{zip_id}.zip'
+
+        # Create a new zip file
+        with zipfile.ZipFile(zip_file_name, 'w') as zipf:
+            # Add each crash file to the zip file
+            for crash_file in crash_files:
+                zipf.write(crash_file)
+
+        # Delete the crash files
+        for crash_file in crash_files:
+            os.remove(crash_file)
+
+        @after_this_request
+        def remove_zip(response):
+            try:
+                os.remove(zip_file_name)
+            except Exception as e:
+                app.logger.error(f"Error removing zip file: {e}")
+            return response
+
+        # Send the zip file as a response
+        return send_file(zip_file_name, as_attachment=True, download_name=zip_file_name)
+
+    else:
+        return 'Unauthorized access!\n', 401
+
+
+@app.route('/', methods=['GET'])
+def root_page():
+    return ROOT_PAGE, 200
+
+
+if __name__ == '__main__':
+    # Run the Flask app
+    app.run()

crashd/fly.toml

@@ -0,0 +1,26 @@
+# fly.toml app configuration file generated for crashd on 2024-05-24T15:13:43-07:00
+#
+# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
+#
+
+app = 'crashd'
+primary_region = 'sea'
+
+[build]
+
+[mounts]
+  source = "crash_logs"
+  destination = "/logs"
+
+[http_service]
+  internal_port = 8080
+  force_https = true
+  auto_stop_machines = true
+  auto_start_machines = true
+  min_machines_running = 0
+  processes = ['app']
+
+[[vm]]
+  memory = '1gb'
+  cpu_kind = 'shared'
+  cpus = 1

crashd/requirements.txt

@@ -0,0 +1,10 @@
+blinker==1.8.2
+click==8.1.7
+Flask==3.0.3
+gunicorn==22.0.0
+itsdangerous==2.2.0
+Jinja2==3.1.4
+MarkupSafe==2.1.5
+packaging==24.0
+spookyhash==2.1.0
+Werkzeug==3.0.3

docs/02_downloads.md

@@ -11,7 +11,7 @@ The following options are available for installing **lnav**:
 ## Linux
 
 <!-- markdown-link-check-disable-next-line -->
-Download a [statically linked 64-bit binary](https://github.com/tstack/lnav/releases/download/v{{site.version}}/lnav-{{site.version}}-x86_64-linux-musl.zip).
+Download a [statically linked 64-bit binary](https://github.com/tstack/lnav/releases/download/v{{site.version}}/lnav-{{site.version}}-linux-musl-x86_64.zip).
 
 Install from the [Snap Store](https://snapcraft.io/lnav):
 

docs/Gemfile.lock

@@ -14,7 +14,7 @@ GEM
       execjs
     coffee-script-source (1.11.1)
     colorator (1.1.0)
-    commonmarker (0.23.9)
+    commonmarker (0.23.10)
     concurrent-ruby (1.2.2)
     dnsruby (1.70.0)
       simpleidn (~> 0.2.1)
@@ -211,9 +211,11 @@ GEM
       jekyll-feed (~> 0.9)
       jekyll-seo-tag (~> 2.1)
     minitest (5.18.1)
-    nokogiri (1.13.10-arm64-darwin)
+    nokogiri (1.16.5-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.13.10-x86_64-linux)
+    nokogiri (1.16.5-x86_64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.16.5-x86_64-linux)
       racc (~> 1.4)
     octokit (4.25.1)
       faraday (>= 1, < 3)
@@ -221,11 +223,12 @@ GEM
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     public_suffix (4.0.7)
-    racc (1.7.1)
+    racc (1.8.0)
     rb-fsevent (0.11.2)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
-    rexml (3.2.5)
+    rexml (3.2.8)
+      strscan (>= 3.0.9)
     rouge (3.26.0)
     ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
@@ -240,6 +243,7 @@ GEM
       faraday (>= 0.17.3, < 3)
     simpleidn (0.2.1)
       unf (~> 0.1.4)
+    strscan (3.1.0)
     terminal-table (1.8.0)
       unicode-display_width (~> 1.1, >= 1.1.1)
     typhoeus (1.4.0)

docs/_config.yml

@@ -19,7 +19,7 @@
 # in the templates via {{ site.myvariable }}.
 
 title: The Logfile Navigator
-version: 0.11.2
+version: 0.12.2
 email: support@lnav.org
 description: >- # this means to ignore newlines until "baseurl:"
   The Logfile Navigator, lnav for short, is an advanced log file viewer
@@ -37,6 +37,9 @@ plugins:
 
 show_excerpts: true
 
+kramdown:
+  syntax_highlighter: rouge
+
 exclude:
   - source
   - tutorials

docs/_posts/2024-01-09-hammered.md

@@ -1,7 +1,9 @@
 ---
 layout: post
-title: Hammered Challenge
-excerpt: Use lnav for the cyberdefenders Hammered challenge
+title: Using lnav to solve the CyberDefenders Hammered Challenge
+excerpt: >-
+  A walkthrough that uses lnav's analysis functionality
+  to answer questions about a collection of logs
 ---
 
 I recently stumbled on this nice [review of lnav](https://lopes.id/2023-lnav-test/)

docs/_posts/2024-03-29-prql-support.md

@@ -0,0 +1,51 @@
+---
+layout: post
+title: Support for the PRQL in the database query prompt
+excerpt: >-
+  PRQL is a database query language that is pipeline-oriented
+  and easier to use interactively
+---
+
+The v0.12.1 release of lnav includes support for
+[PRQL](https://prql-lang.org).  PRQL is a database query language
+that has a pipeline-oriented syntax.  The main advantage of PRQL,
+in the context of lnav, is that it is easier to work with
+interactively compared to SQL.  For example, lnav can provide
+previews of different stages of the pipeline and provide more
+accurate tab-completions for the columns in the result set.  I'm
+hoping that the ease-of-use will make doing log analysis in lnav
+much easier.
+
+You can execute a PRQL query using the existing database prompt
+(press `;`).  A query is interpreted as PRQL if it starts with
+the [`from`](https://prql-lang.org/book/reference/data/from.html)
+keyword.  After `from`, the database table should be provided.
+The table for the focused log message will be suggested by default.
+You can accept the suggestion by pressing TAB.  To add a new stage
+to the pipeline, enter a pipe symbol (`|`), followed by a
+[PRQL transform](https://prql-lang.org/book/reference/stdlib/transforms/index.html)
+and its arguments.  In addition to the standard set of transforms,
+lnav provides some convenience transforms in the `stats` and `utils`
+namespaces.  For example, `stats.count_by` can be passed one or more
+column names to group by and count, with the result sorted by most
+to least.
+
+As you enter a query, lnav will update various panels on the display
+to show help, preview data, and errors.  The following is a
+screenshot of lnav viewing a web access log with a query in progress:
+
+![Screenshot of PRQL in action](/assets/images/lnav-prql-preview.png)
+
+The top half is the usual log message view.  Below that is the online
+help panel showing the documentation for the `stats.count_by` PRQL
+function.  lnav will show the help for what is currently under the
+cursor.  The next panel shows the preview data for the pipeline stage
+that precedes the stage where the cursor is.  In this case, the
+results of `from access_log`, which is the contents of the access
+log table.  The second preview window shows the result of the
+pipeline stage where the cursor is located.
+
+There is still a lot of work to be done on the integration and PRQL
+itself, but I'm very hopeful this will work out well in the long
+term.  Many thanks to the PRQL team for starting the project and
+keeping it going, it's not easy competing with SQL.

docs/schemas/config-v1.schema.json

@@ -140,7 +140,7 @@
                                     "title": "/tuning/remote/ssh/options",
                                     "type": "object",
                                     "patternProperties": {
-                                        "(\\w+)": {
+                                        "^(\\w+)$": {
                                             "title": "/tuning/remote/ssh/options/<option_name>",
                                             "description": "Set an option to be passed to the SSH command",
                                             "type": "string"
@@ -153,7 +153,7 @@
                                     "title": "/tuning/remote/ssh/config",
                                     "type": "object",
                                     "patternProperties": {
-                                        "(\\w+)": {
+                                        "^(\\w+)$": {
                                             "title": "/tuning/remote/ssh/config/<config_name>",
                                             "description": "Set an SSH configuration value",
                                             "type": "string"
@@ -177,14 +177,14 @@
                             "title": "/tuning/clipboard/impls",
                             "type": "object",
                             "patternProperties": {
-                                "([\\w\\-]+)": {
+                                "^([\\w\\-]+)$": {
                                     "description": "Clipboard implementation",
                                     "title": "/tuning/clipboard/impls/<clipboard_impl_name>",
                                     "type": "object",
                                     "properties": {
                                         "test": {
                                             "title": "/tuning/clipboard/impls/<clipboard_impl_name>/test",
-                                            "description": "The command that checks",
+                                            "description": "The command that checks if a clipboard command is available",
                                             "type": "string",
                                             "examples": [
                                                 "command -v pbcopy"
@@ -209,12 +209,52 @@
                     },
                     "additionalProperties": false
                 },
+                "external-opener": {
+                    "description": "Settings related to opening external files/URLs",
+                    "title": "/tuning/external-opener",
+                    "type": "object",
+                    "properties": {
+                        "impls": {
+                            "description": "External opener implementations",
+                            "title": "/tuning/external-opener/impls",
+                            "type": "object",
+                            "patternProperties": {
+                                "^([\\w\\-]+)$": {
+                                    "description": "External opener implementation",
+                                    "title": "/tuning/external-opener/impls/<opener_impl_name>",
+                                    "type": "object",
+                                    "properties": {
+                                        "test": {
+                                            "title": "/tuning/external-opener/impls/<opener_impl_name>/test",
+                                            "description": "The command that checks if an external opener is available",
+                                            "type": "string",
+                                            "examples": [
+                                                "command -v open"
+                                            ]
+                                        },
+                                        "command": {
+                                            "title": "/tuning/external-opener/impls/<opener_impl_name>/command",
+                                            "description": "The command used to open a file or URL",
+                                            "type": "string",
+                                            "examples": [
+                                                "open"
+                                            ]
+                                        }
+                                    },
+                                    "additionalProperties": false
+                                }
+                            },
+                            "additionalProperties": false
+                        }
+                    },
+                    "additionalProperties": false
+                },
                 "url-scheme": {
                     "description": "Settings related to custom URL handling",
                     "title": "/tuning/url-scheme",
                     "type": "object",
                     "patternProperties": {
-                        "([a-z][\\w\\-\\+\\.]+)": {
+                        "^([a-z][\\w\\-\\+\\.]+)$": {
                             "description": "Definition of a custom URL scheme",
                             "title": "/tuning/url-scheme/<url_scheme>",
                             "type": "object",
@@ -272,7 +312,7 @@
                     "title": "/ui/theme-defs",
                     "type": "object",
                     "patternProperties": {
-                        "([\\w\\-]+)": {
+                        "^([\\w\\-]+)$": {
                             "description": "Theme definitions",
                             "title": "/ui/theme-defs/<theme_name>",
                             "type": "object",
@@ -282,7 +322,7 @@
                                     "title": "/ui/theme-defs/<theme_name>/vars",
                                     "type": "object",
                                     "patternProperties": {
-                                        "(\\w+)": {
+                                        "^(\\w+)$": {
                                             "title": "/ui/theme-defs/<theme_name>/vars/<var_name>",
                                             "description": "A theme variable definition",
                                             "type": "string"
@@ -305,6 +345,11 @@
                                             "title": "/ui/theme-defs/<theme_name>/styles/text",
                                             "$ref": "#/definitions/style"
                                         },
+                                        "selected-text": {
+                                            "description": "Styling for text selected in a view",
+                                            "title": "/ui/theme-defs/<theme_name>/styles/selected-text",
+                                            "$ref": "#/definitions/style"
+                                        },
                                         "alt-text": {
                                             "description": "Styling for plain text when alternating",
                                             "title": "/ui/theme-defs/<theme_name>/styles/alt-text",
@@ -680,6 +725,11 @@
                                             "description": "Styling for hotkey highlights of status bars",
                                             "title": "/ui/theme-defs/<theme_name>/status-styles/hotkey",
                                             "$ref": "#/definitions/style"
+                                        },
+                                        "suggestion": {
+                                            "description": "Styling for suggested values",
+                                            "title": "/ui/theme-defs/<theme_name>/status-styles/suggestion",
+                                            "$ref": "#/definitions/style"
                                         }
                                     },
                                     "additionalProperties": false
@@ -689,7 +739,7 @@
                                     "title": "/ui/theme-defs/<theme_name>/log-level-styles",
                                     "type": "object",
                                     "patternProperties": {
-                                        "(trace|debug5|debug4|debug3|debug2|debug|info|stats|notice|warning|error|critical|fatal|invalid)": {
+                                        "^(trace|debug5|debug4|debug3|debug2|debug|info|stats|notice|warning|error|critical|fatal|invalid)$": {
                                             "title": "/ui/theme-defs/<theme_name>/log-level-styles/<level>",
                                             "$ref": "#/definitions/style"
                                         }
@@ -701,7 +751,7 @@
                                     "title": "/ui/theme-defs/<theme_name>/highlights",
                                     "type": "object",
                                     "patternProperties": {
-                                        "([\\w\\-]+)": {
+                                        "^([\\w\\-]+)$": {
                                             "title": "/ui/theme-defs/<theme_name>/highlights/<highlight_name>",
                                             "type": "object",
                                             "properties": {
@@ -727,6 +777,27 @@
                     },
                     "additionalProperties": false
                 },
+                "mouse": {
+                    "description": "Mouse-related settings",
+                    "title": "/ui/mouse",
+                    "type": "object",
+                    "properties": {
+                        "mode": {
+                            "title": "/ui/mouse/mode",
+                            "description": "Overall control for mouse support",
+                            "type": "string",
+                            "enum": [
+                                "disabled",
+                                "enabled"
+                            ],
+                            "examples": [
+                                "enabled",
+                                "disabled"
+                            ]
+                        }
+                    },
+                    "additionalProperties": false
+                },
                 "movement": {
                     "description": "Log file cursor movement mode settings",
                     "title": "/ui/movement",
@@ -753,21 +824,26 @@
                     "title": "/ui/keymap-defs",
                     "type": "object",
                     "patternProperties": {
-                        "([\\w\\-]+)": {
+                        "^([\\w\\-]+)$": {
                             "description": "The keymap definitions",
                             "title": "/ui/keymap-defs/<keymap_name>",
                             "type": "object",
                             "patternProperties": {
-                                "((?:x[0-9a-f]{2})+)": {
+                                "^((?:x[0-9a-f]{2}|f[0-9]{1,2})+)$": {
                                     "description": "Map of key codes to commands to execute.  The field names are the keys to be mapped using as a hexadecimal representation of the UTF-8 encoding.  Each byte of the UTF-8 should start with an 'x' followed by the hexadecimal representation of the byte.",
                                     "title": "/ui/keymap-defs/<keymap_name>/<key_seq>",
                                     "type": "object",
                                     "properties": {
+                                        "id": {
+                                            "title": "/ui/keymap-defs/<keymap_name>/<key_seq>/id",
+                                            "description": "The identifier that can be used to refer to this key",
+                                            "type": "string"
+                                        },
                                         "command": {
                                             "title": "/ui/keymap-defs/<keymap_name>/<key_seq>/command",
                                             "description": "The command to execute for the given key sequence.  Use a script to execute more complicated operations.",
                                             "type": "string",
-                                            "pattern": "^[:|;].*",
+                                            "pattern": "^$|^[:|;].*",
                                             "examples": [
                                                 ":goto next hour"
                                             ]
@@ -812,7 +888,7 @@
                     "title": "/log/watch-expressions",
                     "type": "object",
                     "patternProperties": {
-                        "([\\w\\.\\-]+)": {
+                        "^([\\w\\.\\-]+)$": {
                             "description": "A log message watch expression",
                             "title": "/log/watch-expressions/<watch_name>",
                             "type": "object",
@@ -837,7 +913,7 @@
                     "title": "/log/annotations",
                     "type": "object",
                     "patternProperties": {
-                        "([\\w\\.\\-]+)": {
+                        "^([\\w\\.\\-]+)$": {
                             "title": "/log/annotations/<annotation_name>",
                             "type": "object",
                             "properties": {
@@ -863,6 +939,37 @@
                         }
                     },
                     "additionalProperties": false
+                },
+                "demux": {
+                    "description": "Demultiplexer definitions",
+                    "title": "/log/demux",
+                    "type": "object",
+                    "patternProperties": {
+                        "^([\\w\\-\\.]+)$": {
+                            "description": "The definition of a demultiplexer",
+                            "title": "/log/demux/<name>",
+                            "type": "object",
+                            "properties": {
+                                "enabled": {
+                                    "title": "/log/demux/<name>/enabled",
+                                    "description": "Indicates whether this demuxer will be used at the demuxing stage",
+                                    "type": "boolean"
+                                },
+                                "pattern": {
+                                    "title": "/log/demux/<name>/pattern",
+                                    "description": "A regular expression to match a line in a multiplexed file",
+                                    "type": "string"
+                                },
+                                "control-pattern": {
+                                    "title": "/log/demux/<name>/control-pattern",
+                                    "description": "A regular expression to match a control line in a multiplexed file",
+                                    "type": "string"
+                                }
+                            },
+                            "additionalProperties": false
+                        }
+                    },
+                    "additionalProperties": false
                 }
             },
             "additionalProperties": false
@@ -872,7 +979,7 @@
             "title": "/global",
             "type": "object",
             "patternProperties": {
-                "(\\w+)": {
+                "^(\\w+)$": {
                     "title": "/global/<var_name>",
                     "description": "A global variable definition.  Global variables can be referenced in scripts, SQL statements, or commands.",
                     "type": "string"

docs/schemas/event-log-msg-detected-v1.schema.json

@@ -41,7 +41,7 @@
             "title": "/values",
             "type": "object",
             "patternProperties": {
-                "([\\w\\-]+)": {
+                "^([\\w\\-]+)$": {
                     "title": "/values/<name>",
                     "type": [
                         "boolean",

docs/schemas/format-v1.schema.json

@@ -10,7 +10,7 @@
         }
     },
     "patternProperties": {
-        "(\\w+)": {
+        "^(\\w+)$": {
             "description": "The definition of a log file format.",
             "title": "/<format_name>",
             "type": "object",
@@ -20,7 +20,7 @@
                     "title": "/<format_name>/regex",
                     "type": "object",
                     "patternProperties": {
-                        "([^/]+)": {
+                        "^(.+)$": {
                             "description": "The set of patterns used to match log messages",
                             "title": "/<format_name>/regex/<pattern_name>",
                             "type": "object",
@@ -95,7 +95,7 @@
                                     "title": "/<format_name>/converter/header/expr",
                                     "type": "object",
                                     "patternProperties": {
-                                        "(\\w+)": {
+                                        "^(\\w+)$": {
                                             "title": "/<format_name>/converter/header/expr/<header_expr_name>",
                                             "description": "SQLite expression",
                                             "type": "string"
@@ -214,7 +214,7 @@
                             "title": "/<format_name>/opid/description",
                             "type": "object",
                             "patternProperties": {
-                                "([\\w\\.\\-]+)": {
+                                "^([\\w\\.\\-]+)$": {
                                     "description": "A type of description for this operation",
                                     "title": "/<format_name>/opid/description/<opid_descriptor>",
                                     "type": "object",
@@ -266,7 +266,7 @@
                             "title": "/<format_name>/opid/sub-description",
                             "type": "object",
                             "patternProperties": {
-                                "([\\w\\.\\-]+)": {
+                                "^([\\w\\.\\-]+)$": {
                                     "description": "A type of description for this sub-operation",
                                     "title": "/<format_name>/opid/sub-description/<subid_descriptor>",
                                     "type": "object",
@@ -326,7 +326,7 @@
                     "title": "/<format_name>/level",
                     "type": "object",
                     "patternProperties": {
-                        "(trace|debug[2345]?|info|stats|notice|warning|error|critical|fatal)": {
+                        "^(trace|debug[2345]?|info|stats|notice|warning|error|critical|fatal)$": {
                             "title": "/<format_name>/level/<level>",
                             "description": "The regular expression used to match the log text for this level.  For JSON logs with numeric levels, this should be the number for the corresponding level.",
                             "type": [
@@ -342,7 +342,7 @@
                     "title": "/<format_name>/value",
                     "type": "object",
                     "patternProperties": {
-                        "([^/]+)": {
+                        "^(.+)$": {
                             "description": "The set of values captured by the log message patterns",
                             "title": "/<format_name>/value/<value_name>",
                             "type": "object",
@@ -382,7 +382,7 @@
                                             "title": "/<format_name>/value/<value_name>/unit/scaling-factor",
                                             "type": "object",
                                             "patternProperties": {
-                                                "([^/]+)": {
+                                                "^(.+)$": {
                                                     "title": "/<format_name>/value/<value_name>/unit/scaling-factor/<scale>",
                                                     "type": "object",
                                                     "properties": {
@@ -455,7 +455,7 @@
                     "title": "/<format_name>/tags",
                     "type": "object",
                     "patternProperties": {
-                        "([\\w:;\\._\\-]+)": {
+                        "^([\\w:;\\._\\-]+)$": {
                             "description": "The name of the tag to apply",
                             "title": "/<format_name>/tags/<tag_name>",
                             "type": "object",
@@ -518,11 +518,79 @@
                     },
                     "additionalProperties": false
                 },
+                "partitions": {
+                    "description": "The partitions to automatically apply to log messages",
+                    "title": "/<format_name>/partitions",
+                    "type": "object",
+                    "patternProperties": {
+                        "^([\\w:;\\._\\-]+)$": {
+                            "description": "The type of partition to apply",
+                            "title": "/<format_name>/partitions/<partition_type>",
+                            "type": "object",
+                            "properties": {
+                                "paths": {
+                                    "description": "Restrict partitioning to the given paths",
+                                    "title": "/<format_name>/partitions/<partition_type>/paths",
+                                    "type": "array",
+                                    "items": {
+                                        "type": "object",
+                                        "properties": {
+                                            "glob": {
+                                                "title": "/<format_name>/partitions/<partition_type>/paths/glob",
+                                                "description": "The glob to match against file paths",
+                                                "type": "string",
+                                                "examples": [
+                                                    "*/system.log*"
+                                                ]
+                                            }
+                                        },
+                                        "additionalProperties": false
+                                    }
+                                },
+                                "pattern": {
+                                    "title": "/<format_name>/partitions/<partition_type>/pattern",
+                                    "description": "The regular expression to match against the body of the log message",
+                                    "type": "string",
+                                    "examples": [
+                                        "\\w+ is down"
+                                    ]
+                                },
+                                "description": {
+                                    "title": "/<format_name>/partitions/<partition_type>/description",
+                                    "description": "A description of this partition",
+                                    "type": "string"
+                                },
+                                "level": {
+                                    "title": "/<format_name>/partitions/<partition_type>/level",
+                                    "description": "Constrain hits to log messages with this level",
+                                    "type": "string",
+                                    "enum": [
+                                        "trace",
+                                        "debug5",
+                                        "debug4",
+                                        "debug3",
+                                        "debug2",
+                                        "debug",
+                                        "info",
+                                        "stats",
+                                        "notice",
+                                        "warning",
+                                        "error",
+                                        "critical",
+                                        "fatal"
+                                    ]
+                                }
+                            },
+                            "additionalProperties": false
+                        }
+                    },
+                    "additionalProperties": false
+                },
                 "action": {
                     "title": "/<format_name>/action",
                     "type": "object",
                     "patternProperties": {
-                        "(\\w+)": {
+                        "^(\\w+)$": {
                             "title": "/<format_name>/action/<action_name>",
                             "type": [
                                 "string",
@@ -650,7 +718,8 @@
                                 "enum": [
                                     "abbrev",
                                     "truncate",
-                                    "dot-dot"
+                                    "dot-dot",
+                                    "last-word"
                                 ]
                             },
                             "text-transform": {
@@ -683,7 +752,7 @@
                     "title": "/<format_name>/search-table",
                     "type": "object",
                     "patternProperties": {
-                        "(\\w+)": {
+                        "^(\\w+)$": {
                             "description": "The set of search tables to be automatically defined",
                             "title": "/<format_name>/search-table/<table_name>",
                             "type": "object",
@@ -729,7 +798,7 @@
                     "title": "/<format_name>/highlights",
                     "type": "object",
                     "patternProperties": {
-                        "([^/]+)": {
+                        "^(.+)$": {
                             "description": "The definition of a highlight",
                             "title": "/<format_name>/highlights/<highlight_name>",
                             "type": "object",

docs/source/commands.rst

@@ -68,9 +68,13 @@ an error message in the status bar, like so:
 .. note:: The following commands can be disabled by setting the ``LNAVSECURE``
    environment variable before executing the **lnav** binary:
 
+   - :code:`:cd`
+   - :code:`:export-session-to`
    - :code:`:open`
    - :code:`:pipe-to`
    - :code:`:pipe-line-to`
+   - :code:`:redirecto-to`
+   - :code:`:sh`
    - :code:`:write-*-to`
 
    This makes it easier to run **lnav** in restricted environments without the

docs/source/conf.py

@@ -258,7 +258,7 @@ copyright = u'2023, Tim Stack'
 # The short X.Y version.
 version = '0.12'
 # The full version, including alpha/beta/rc tags.
-release = '0.12.0'
+release = '0.12.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/source/config.rst

@@ -158,15 +158,15 @@ definition, see one of the definitions built into **lnav**, like
 Reference
 ^^^^^^^^^
 
-.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/vars
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/^([\w\-]+)$/properties/vars
 
-.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/styles
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/^([\w\-]+)$/properties/styles
 
-.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/syntax-styles
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/^([\w\-]+)$/properties/syntax-styles
 
-.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/status-styles
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/^([\w\-]+)$/properties/status-styles
 
-.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/([\w\-]+)/properties/log-level-styles
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/theme-defs/patternProperties/^([\w\-]+)$/properties/log-level-styles
 
 .. _theme_style:
 
@@ -194,10 +194,11 @@ bottom-right section of the UI will be updated with the help text.
 Key Sequence Encoding
 ^^^^^^^^^^^^^^^^^^^^^
 
-Key presses are converted into a hex-encoded string that is used to lookup an
-entry in the keymap.  Each byte of the keypress value is formatted as an
+Key presses are converted into a string that is used to lookup an
+entry in the keymap.  Function keys are encoded as an :code:`f` followed by
+the key number.  Other keys are encoded as UTF-8 bytes and formatted as an
 :code:`x` followed by the hex-encoding in lowercase.  For example, the encoding
-for the £ key would be :code:`xc2xa3`.  To make it easier to discover the
+for the :code:`£` key would be :code:`xc2xa3`.  To make it easier to discover the
 encoding for unassigned keys, **lnav** will print in the command prompt the
 :code:`:config` command and
 `JSON-Pointer <https://tools.ietf.org/html/rfc6901>`_ for assigning a command
@@ -217,7 +218,7 @@ to the key.
 Reference
 ^^^^^^^^^
 
-.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/keymap-defs/patternProperties/([\w\-]+)
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/ui/properties/keymap-defs/patternProperties/^([\w\-]+)$
 
 
 Log Handling
@@ -306,11 +307,43 @@ standard input.  The handler should then generate the annotation
 content on the standard output.  The output is treated as Markdown,
 so the content can be styled as desired.
 
+Demultiplexing (v0.12.3+)
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Files that are a mix of content from different sources, like
+the output of :code:`docker compose logs`, can be automatically
+demultiplexed so that *lnav* can process them correctly.  Each
+line of the input file must have a unique identifier that can
+be used to determine which service the line belongs to.  The
+lines are then distributed to separate files based on the
+identifier.  A demultiplexer is a regular expression that
+extracts the identifier, the log message, and an optional
+timestamp.
+
+Demultiplexers are defined in the main configuration under
+the :code:`/log/demux` path.  The pattern for the demuxer
+has the following known capture names:
+
+:mux_id: (required) Captures the unique identifier.
+
+:body: (required) Captures the body of the log message
+  that should be written to the file.
+
+:timestamp: (optional) The timestamp for the log message.
+  If this is available and the log message does not have
+  it's own timestamp, this will be used instead.
+
+If there are additional captures, they will be included
+in the file metadata that can be accessed by the
+:code:`lnav_file_demux_metadata` view of the
+:code:`lnav_file_metadata` table.
+
 Reference
 ^^^^^^^^^
 
-.. jsonschema:: ../schemas/config-v1.schema.json#/properties/log/properties/watch-expressions/patternProperties/([\w\.\-]+)
-.. jsonschema:: ../schemas/config-v1.schema.json#/properties/log/properties/annotations/patternProperties/([\w\.\-]+)
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/log/properties/watch-expressions/patternProperties/^([\w\.\-]+)$
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/log/properties/annotations/patternProperties/^([\w\.\-]+)$
+.. jsonschema:: ../schemas/config-v1.schema.json#/properties/log/properties/demux/patternProperties/^([\w\-\.]+)$
 
 
 .. _tuning:

docs/source/cookbook.rst

@@ -101,3 +101,12 @@ can use the following commands to generate customized output for a report:
    :language: custsqlite
    :caption: report-demo.lnav
    :linenos:
+
+
+Creating a Runbook
+^^^^^^^^^^^^^^^^^^
+
+If you have a collection of lnav scripts, you can create a markdown file
+that links to the scripts and use it as a runbook.  Clicking on a link to
+a :file:`.lnav` file will open an "Actions" menu with an "Execute" menu
+item.  Clicking on "Execute" will launch the script.

docs/source/formats.rst

@@ -227,6 +227,9 @@ object with the following fields:
       :truncate: Truncates any text past the maximum width.
       :dot-dot: Cuts out the middle of the text and replaces it with two
         dots (i.e. '..').
+      :last-word: Removes all but the last word in text with dot, dash,
+        forward-slash, or colon separators. For example, "com.example.foo"
+        would be shortened to "foo".
 
     (v0.8.2+)
   :timestamp-format: The timestamp format to use when displaying the time
@@ -367,6 +370,19 @@ object with the following fields:
 
     :glob: A glob pattern to check against the log files read by lnav.
 
+:partitions: This object contains a description of partitions that should
+  automatically be created in the log view.
+
+  :pattern: The regular expression evaluated over a line in the log file as
+    it is read in.  If there is a match, the log message the line is a part
+    of will be used as the start of the partition.  The name of the
+    partition will be taken from any captures in the regex.
+  :paths: This array contains objects that define restrictions on the file
+    paths in which partitions will be created.  The objects in this array
+    can contain:
+
+    :glob: A glob pattern to check against the log files read by lnav.
+
 .. _format_sample:
 
 :sample: A list of objects that contain sample log messages.  All formats

docs/source/intro.rst

@@ -113,16 +113,33 @@ To create or customize a theme, consult the :ref:`themes` section.
 Cursor Mode (v0.11.2+)
 ^^^^^^^^^^^^^^^^^^^^^^
 
-The default mode for scrolling in **lnav** is to move the contents of the
-main view when the arrow keys are pressed.  Any interactions, such as
-jumping to a search hit, are then focused on the top line in the view.
-Alternatively, you can enable "cursor" mode where there is a cursor line
-in the view that is moved by the arrow keys and other interactions.  You
-can enable cursor mode with the following command:
+The default mode for scrolling in **lnav** is "cursor" mode where
+there is a cursor line in the view that is moved by the arrow keys
+and other interactions.  Any interactions, such as jumping to a
+search hit, are then focused on that line.
+
+Alternatively, you can enable "top" mode where the contents of the
+main view are moved when the arrow keys are pressed.  Any
+interactions, such as jumping to a search hit, are then focused
+on the top line in the view.  You can change to "top" mode with
+the following command:
 
 .. code-block:: lnav
 
-   :config /ui/movement/mode cursor
+   :config /ui/movement/mode top
+
+
+Mouse Support (v0.12.2+)
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Mouse support can be enabled temporarily by pressing :kbd:`F2`
+and can be set as the default by executing the following command:
+
+.. code-block:: lnav
+
+   :config /ui/mouse/mode enabled
+
+See :ref:`ui_mouse` for more details.
 
 Log Formats
 ^^^^^^^^^^^
@@ -167,6 +184,10 @@ Check the `downloads page <http://lnav.org/downloads>`_ to see if there are
 packages for your operating system.  To compile from source, use the following
 commands:
 
+.. note::
+
+   Run :code:`./autogen.sh` if compiling from the git repository.
+
 .. prompt:: bash
 
    ./configure

docs/source/sqlext.rst

@@ -47,6 +47,14 @@ and maximum number of bytes returned by the server, grouped by IP address:
 
     ;SELECT c_ip, avg(sc_bytes), max(sc_bytes) FROM access_log GROUP BY c_ip
 
+.. note::
+
+   For reference, the PRQL query would look like this:
+
+   .. code-block:: elm
+
+       from access_log | stats.by c_ip {average sc_bytes, max sc_bytes}
+
 After pressing :kbd:`Enter`, SQLite will execute the query using **lnav**'s
 virtual table implementation to extract the data directly from the log files.
 Once the query has finished, the main window will switch to the DB view to
@@ -71,6 +79,39 @@ The DB view has the following display features:
   the `jget`_ function.
 
 
+PRQL Support (v0.12.1+)
+-----------------------
+
+`PRQL <https://prql-lang.org>`_ is an alternative database query language
+that compiles to SQLite.  You can enter PRQL in the database query prompt
+and lnav will switch accordingly.  A major advantage of using PRQL is that
+lnav can show previews of the results of the pipeline stages and provide
+better tab completion options.
+
+A PRQL query starts with the :code:`from` keyword that specifies the table
+to use as a data source.  The next stage of a pipeline is started by
+entering a pipe symbol (:code:`|`) followed by a
+`PRQL transform <https://prql-lang.org/book/reference/stdlib/transforms/index.html>`_.
+As you build the query in the prompt, lnav will display any relevant
+help and preview for the current and previous stages of the pipeline.
+
+The following is a screenshot of lnav viewing a web access log with a
+query in progress:
+
+.. figure:: ../assets/images/lnav-prql-preview.png
+   :align: center
+
+   Screenshot of a PRQL query in progress
+
+The top half of the window is the usual log message view.  Below that is
+the online help panel showing the documentation for the :code:`stats.count_by`
+PRQL function.  lnav will show the help for what is currently under the
+cursor.  The next panel shows the preview data for the pipeline stage
+that precedes the stage where the cursor is.  In this case, the
+results of :code:`from access_log`, which is the contents of the access
+log table.  The second preview window shows the result of the
+pipeline stage where the cursor is located.
+
 Log Tables
 ----------
 

docs/source/ui.rst

@@ -26,11 +26,11 @@ times in the views.
 
 **lnav** provides many operations to work with the log/text data in the
 main view.  For example, you can add comments and tags to log messages.
-By default, the top line is used as the reference point to edit the
+The highlighted cursor line is used as the reference point to edit the
 comment or tags.  Alternatively, you can press :kbd:`Ctrl` + :kbd:`x`
-to switch to "cursor" mode where the "focused" line is highlighted and
-most operations now work with that line.  When in "cursor" mode, the
-:kbd:`↑` and :kbd:`↓` keys now move the focused line instead of scrolling
+to switch to "top" mode where the "focused" line is the top line in the
+view and most operations now work with that line.  When in "cursor" mode,
+the :kbd:`↑` and :kbd:`↓` keys now move the focused line instead of scrolling
 the view.  Jumping to bookmarks, like errors, will also move the focused
 line instead of moving the next error to the top of the view.
 
@@ -104,14 +104,19 @@ To hide the panels again, press :kbd:`q`.
    Screenshot of the files panel showing the loaded files.
 
 The Files panel is open initially to display progress in loading files.
-The following information can be displayed for each file:
+The following information is displayed for each file:
+
+* the "unique" portion of the path relative to the other files and
+* the amount of data that has been indexed.
+
+To the right of the file list is a panel that shows details for each
+file, such as:
 
-* the "unique" portion of the path relative to the other files;
-* the amount of data that has been indexed;
 * the date range of log messages contained in the file;
 * the errors that were encountered while trying to index the file;
 * the notes recorded for files where some automatic action was taken,
-  like hiding the file if it was seen as a duplicate of another file.
+  like hiding the file if it was seen as a duplicate of another file;
+* the details of the demultiplexing and log format matching process.
 
 .. figure:: lnav-filters-panel.png
    :align: center
@@ -305,19 +310,19 @@ can also press :kbd:`Shift` + :kbd:`i` to toggle the histogram view
 while synchronizing the top time.  While in the histogram view,
 pressing :kbd:`z` / :kbd:`Shift` + :kbd:`z` will zoom in/out.
 
-GANTT
-^^^^^
+TIMELINE
+^^^^^^^^
 
 .. note:: This feature is available in v0.12.0+.
 
-.. figure:: lnav-gantt-1.png
+.. figure:: lnav-timeline-1.png
    :align: center
 
-   Screenshot of the Gantt chart view when viewing logs from the
+   Screenshot of the timeline view when viewing logs from the
    VMWare Update Manager.  Most rows show API requests as they
    are received and processed.
 
-The Gantt Chart view visualizes operations over time.  The operations
+The timeline view [#]_ visualizes operations over time.  The operations
 are identified by the "opid" field defined in the log format.  In the
 view, there is a header that shows the overall time span, the
 narrowed time span around the focused line, and the column headers.
@@ -346,7 +351,7 @@ The following hotkeys can be useful in this view:
   :code:`opid/subid` property, this will toggle an overlay panel
   that displays the sub-operation descriptions.
 
-  .. figure:: lnav-gantt-2.png
+  .. figure:: lnav-timeline-2.png
      :align: center
 
      Screenshot showing the same log as above after pressing
@@ -354,12 +359,14 @@ The following hotkeys can be useful in this view:
      sub-operations performed while processing the main operation.
 
 * :kbd:`Shift` + :kbd:`q` -- Return to the previous view and change
-  its focused line to match the time that was focused in the gantt
+  its focused line to match the time that was focused in the timeline
   view.
-* :kbd:`Shift` + :kbd:`a` -- After leaving the gantt view, pressing
-  these keys will return to the Gantt view while keeping the focused
+* :kbd:`Shift` + :kbd:`a` -- After leaving the timeline view, pressing
+  these keys will return to the timeline view while keeping the focused
   time in sync.
 
+.. [#] Formerly called the "Gantt Chart" view.
+
 PRETTY
 ^^^^^^
 
@@ -416,3 +423,74 @@ range of values.  The panel at the bottom of the view shows the data points
 themselves from the original source, the log file or the SQL query results.
 You can press :kbd:`TAB` to focus on the details panel so you can scroll
 around and get a closer look at the values.
+
+.. _ui_mouse:
+
+Mouse Support (v0.12.2+)
+------------------------
+
+With mouse support enabled, either through the `/ui/mouse/mode`
+configuration option or by pressing :kbd:`F2`, many of the UI
+elements will respond to mouse inputs:
+
+* clicking on the main view will move the cursor to the given
+  row and dragging will scroll the view as needed;
+* :kbd:`Shift` (or :kbd:`CTRL`) clicking/dragging in the main
+  view will highlight lines and then toggle their bookmark
+  status on release;
+* double-clicking in the main view will select the underlying
+  text and drag-selecting within a line will select the given
+  text;
+* when double-clicking text: if the mouse pointer is inside
+  a quoted string, the contents of the string will be selected;
+  if the mouse pointer is on the quote, the quote will be included
+  in the selection; if the mouse pointer is over a bracket
+  (e.g. [],{},()) where the matching bracket is on the same line,
+  the selection will span from one bracket to the other;
+* when text is selected, a menu will pop up that can be used
+  to filter based on the current text, search for it, or copy
+  it to the clipboard;
+* right-clicking the start of a log message in the main view
+  will open the parser details overlay;
+* the parser details now displays a diamond next to fields to
+  indicate whether they are shown/hidden and this can be
+  clicked to toggle the state;
+* the parser details will show a bar chart icon for fields with
+  values which, when clicked, will open either the spectrogram
+  view for the given field or open the DB query prompt with a
+  PRQL query to generate a histogram of the field values;
+* clicking in the scroll area will move the view by a page,
+  double-clicking will move the view to that area, and
+  dragging the scrollbar will move the view to the given spot;
+* clicking on the breadcrumb bar will select a crumb and
+  selecting a possibility from the popup will move to that
+  location in the view;
+* clicking on portions of the bottom status bar will trigger
+  a relevant action (e.g. clicking the line number will open
+  the command prompt with :code:`:goto <current-line>`);
+* clicking on the configuration panel tabs (i.e. Files/Filters)
+  will open the selected panel and clicking parts of the
+  display in there will perform the relevant action (e.g.
+  clicking the diamond will enable/disable the file/filter);
+* clicking in a prompt will move the cursor to the location;
+* clicking on a column in the spectrogram view will select it.
+
+.. note::
+
+   A downside of enabling mouse support is that normal text
+   selection and copy will no longer work.  While lnav has
+   some support for selection in the main view, there are
+   still likely to be cases where that is insufficient.
+   In those cases, you can press :kbd:`F2` to quickly
+   switch back-and-forth.  Or, some terminals have support
+   for switching while a modifier is pressed:
+
+   .. list-table::
+      :header-rows: 1
+
+      * - Key
+        - Terminal
+      * - :kbd:`Option`
+        - iTerm, Hyper
+      * - :kbd:`Fn`
+        - Terminal.app

docs/source/usage.rst

@@ -265,10 +265,41 @@ columns will be included in the table.
 Taking Notes
 ------------
 
-A few of the columns in the log tables can be updated on a row-by-row basis to
-allow you to take notes.  The majority of the columns in a log table are
-read-only since they are backed by the log files themselves.  However, the
-following columns can be changed by an :code:`UPDATE` statement:
+As you are looking through logs, you might find that you want to leave some
+notes of your findings.  **lnav** can help here by saving information in
+the session without needing to modify the actual log files.  Thus, when
+you re-open the files in lnav, the notes will be restored.  The following
+types of information can be saved:
+
+:tags: Log messages can be tagged with the :ref:`:tag<tag>` command as a
+  simple way to leave a descriptive mark.  The tags attached to a
+  message will be shown underneath the message.  You can press
+  :kbd:`u` and :kbd:`Shift` + :kbd:`u` to jump to the next/previous
+  marked line.  A regular search will also match tags.
+
+:comments: Free-form text can be attached to a log message with the
+  :ref:`:comment<comment>` command.  The comment will be shown
+  underneath the message. If the text contains markdown syntax,
+  it will be rendered to the best of the terminal's ability.
+  You can press :kbd:`u` and :kbd:`Shift` + :kbd:`u` to jump to the
+  next/previous marked line.  A regular search will also match the
+  comment text.
+
+:partitions: The log view can be partitioned to provide some context
+  about where you are in a collection of logs.  For example, in logs
+  for a test run, partitions could be created with the name for each
+  test.  The current partition is shown in the breadcrumb bar and
+  prefixed by the "⊑" symbol.  You can select the partition breadcrumb
+  to jump to another partition.  Pressing :kbd:`{` and :kbd:`}` will
+  jump to the next/previous partition.
+
+Accessing notes through the SQLite interface
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The note taking functionality in lnav can also be accessed through the
+log tables exposed through SQLite.  The majority of the columns in a log
+table are read-only since they are backed by the log files themselves.
+However, the following columns can be changed by an :code:`UPDATE` statement:
 
 * **log_part** - The "partition" the log message belongs to.  This column can
   also be changed by the :ref:`:partition-name<partition_name>` command.

example-scripts/report-demo.lnav

@@ -17,7 +17,7 @@
 ;SELECT printf('\n%d total requests', count(1)) AS msg FROM access_log
 :echo $msg
 
-;WITH top_paths AS (
+;WITH top_paths AS MATERIALIZED (
         SELECT
             cs_uri_stem,
             count(1) AS total_hits,
@@ -28,7 +28,7 @@
           GROUP BY cs_uri_stem
           ORDER BY total_hits DESC
           LIMIT 50),
-      weekly_hits_with_gaps AS (
+      weekly_hits_with_gaps AS MATERIALIZED (
         SELECT timeslice(log_time_msecs, '1w') AS week,
                cs_uri_stem,
                count(1) AS weekly_hits

lnav.1

@@ -1,4 +1,3 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.43.3.
 .\" Some roff macros, for reference:
 .\" .nh        disable hyphenation
 .\" .hy        enable hyphenation
@@ -19,21 +18,27 @@
 .ft R
 .fi
 ..
-.TH LNAV "1" "August 2022"
+.TH LNAV "1" "April 2024"
 .SH NAME
-lnav \- ncurses-based log file viewer
+lnav \- log file viewer for the terminal
 .SH SYNOPSIS
 .B lnav 
-[\-hVsar] [logfile1 logfile2 ...]
+[options] [logfile1 logfile2 ...]
 .SH DESCRIPTION
-The log file navigator, lnav, is an enhanced log file viewer that
-takes advantage of any semantic information that can be gleaned from
-the files being viewed, such as timestamps and log levels.  Using this
-extra semantic information, lnav can do things like interleaving
-messages from different files, generate histograms of messages over
-time, and providing hotkeys for navigating through the file.  It is
-hoped that these features will allow the user to quickly and
-efficiently zero in on problems.
+The Logfile Navigator, lnav, is a log file viewer for the terminal.
+Given a set of files/directories, lnav will:
+.TP
+- decompress as needed;
+.TP
+- detect their format;
+.TP
+- merge the files together by time into a single view;
+.TP
+- tail the files, follow renames, find new files in directories;
+.TP
+- build an index of errors and warnings;
+.TP
+- pretty-print JSON-lines.
 .SH KEY BINDINGS
 .TP
 ?
@@ -49,36 +54,15 @@ Print help and exit
 \fB\-H\fR
 Display the internal help text.
 .TP
-\fB\-n\fR
-Run without the curses UI.  (headless mode)
-.TP
-\fB\-c\fR cmd
-Execute a command after the files have been loaded.
-.TP
-\fB\-f\fR path
-Execute the commands in the given file.
-.TP
 \fB\-I\fR path
 Add the given configuration directory to the search path.
 .TP
-\fB\-n\fR
-Do not open the default syslog file if no files are given.
-.TP
-\fB\-q\fR
-Quiet mode.  Do not print the log messages after executing all of the commands.
-.TP
-\fB\-i\fR
-Install the given format files in the $HOME/.lnav/formats/installed directory
-and exit.
+\fB\-W\fR
+Print warnings related to lnav's configuration.
 .TP
 \fB\-u\fR
 Update formats installed from git repositories.
 .TP
-\fB\-C\fR
-Check the configuration and exit.  The log format files will be loaded and
-checked.  Any files given on the command-line will be loaded checked to make
-sure they match a log format.
-.TP
 \fB\-d\fR file
 Write debug messages to the given file.
 .TP
@@ -91,15 +75,47 @@ Recursively load files from the given directories.
 \fB\-R\fR
 Load older rotated log files as well.
 .TP
+\fB\-c\fR cmd
+Execute a command after the files have been loaded.
+.TP
+\fB\-f\fR path
+Execute the commands in the given file.
+.TP
+\fB\-e\fR cmd
+Execute a shell command-line.
+.TP
 \fB\-t\fR
 Prepend timestamps to the lines of data being read in
 on the standard input.
+.TP
+\fB\-n\fR
+Run without the curses UI.  (headless mode)
+.TP
+\fB\-N\fR
+Do not open the default syslog file if no files are given.
+.TP
+\fB\-q\fR
+Quiet mode.  Do not print the log messages after executing all of the commands.
 .SS "Optional arguments:"
 .TP
 logfile1
 The log files or directories to view.  If a
 directory is given, all of the files in the
 directory will be loaded.
+.SH MANAGEMENT-MODE OPTIONS
+.TP
+\fB\-i\fR
+Install the given format files in the $HOME/.lnav/formats/installed directory
+and exit.
+.TP
+\fB\-m\fR
+Switch to the management command-line mode.  This mode is
+used to work with lnav's configuration.
+.TP
+\fB\-C\fR
+Check the configuration and exit.  The log format files will be loaded and
+checked.  Any files given on the command-line will be loaded checked to make
+sure they match a log format.
 .SH EXAMPLES
 To load and follow the syslog file:
 .PP

overlay-ports/ncurses/portfile.cmake

@@ -0,0 +1,70 @@
+vcpkg_download_distfile(
+        ARCHIVE_PATH
+        URLS
+        "https://invisible-mirror.net/archives/ncurses/ncurses-${VERSION}.tar.gz"
+        "ftp://ftp.invisible-island.net/ncurses/ncurses-${VERSION}.tar.gz"
+        "https://ftp.gnu.org/gnu/ncurses/ncurses-${VERSION}.tar.gz"
+        FILENAME "ncurses-${VERSION}.tgz"
+        SHA512 1c2efff87a82a57e57b0c60023c87bae93f6718114c8f9dc010d4c21119a2f7576d0225dab5f0a227c2cfc6fb6bdbd62728e407f35fce5bf351bb50cf9e0fd34
+)
+
+vcpkg_extract_source_archive(
+        SOURCE_PATH
+        ARCHIVE "${ARCHIVE_PATH}"
+)
+
+vcpkg_list(SET OPTIONS)
+
+if (VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic")
+    list(APPEND OPTIONS
+            --with-cxx-shared
+            --with-shared    # "lib model"
+            --without-normal # "lib model"
+    )
+endif ()
+
+if (NOT VCPKG_TARGET_IS_MINGW)
+    list(APPEND OPTIONS
+            --enable-mixed-case
+    )
+endif ()
+
+if (VCPKG_TARGET_IS_MINGW)
+    list(APPEND OPTIONS
+            --disable-home-terminfo
+            --enable-term-driver
+            --disable-termcap
+    )
+endif ()
+
+vcpkg_configure_make(
+        SOURCE_PATH "${SOURCE_PATH}"
+        DETERMINE_BUILD_TRIPLET
+        NO_ADDITIONAL_PATHS
+        OPTIONS
+        ${OPTIONS}
+        --disable-db-install
+        --enable-pc-files
+        --enable-widec
+        --enable-sigwinch
+        --enable-termcap
+        --enable-ext-colors
+        --without-ada
+        --without-debug # "lib model"
+        --without-manpages
+        --without-progs
+        --without-tack
+        --without-tests
+        --with-pkg-config-libdir=libdir
+)
+vcpkg_install_make()
+
+vcpkg_fixup_pkgconfig()
+
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin")
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin")
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include")
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share")
+
+file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}")
+vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/COPYING")

overlay-ports/ncurses/usage

@@ -0,0 +1,10 @@
+The package ncurses is compatible with built-in CMake variables:
+
+    set(CURSES_NEED_NCURSES TRUE)
+    find_package(Curses REQUIRED)
+    target_include_directories(main PRIVATE ${CURSES_INCLUDE_DIRS})
+    target_compile_options(main PRIVATE ${CURSES_CFLAGS})
+    target_link_libraries(main PRIVATE ${CURSES_LIBRARIES})
+
+
+Blah

overlay-ports/ncurses/vcpkg.json

@@ -0,0 +1,9 @@
+{
+    "name": "ncurses",
+    "version": "6.4",
+    "port-version": 2,
+    "description": "Free software emulation of curses in System V Release 4.0, and more",
+    "homepage": "https://invisible-island.net/ncurses/announce.html",
+    "license": "MIT",
+    "supports": "!windows | mingw"
+}

overlay-ports/readline-osx/portfile.cmake

@@ -0,0 +1 @@
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)

overlay-ports/readline-osx/vcpkg.json

@@ -0,0 +1,9 @@
+{
+  "name": "readline-osx",
+  "version-date": "2020-01-04",
+  "description": "empty package, linking to readline-unix",
+  "supports": "osx",
+  "dependencies": [
+    "readline-unix"
+  ]
+}

overlay-ports/readline-unix/portfile.cmake

@@ -0,0 +1,34 @@
+set(filename readline-${VERSION}.tar.gz)
+vcpkg_download_distfile(
+    ARCHIVE
+    URLS
+        "https://ftpmirror.gnu.org/gnu/readline/${filename}"
+        "https://ftp.gnu.org/gnu/readline/${filename}"
+    FILENAME "${filename}"
+    SHA512 0a451d459146bfdeecc9cdd94bda6a6416d3e93abd80885a40b334312f16eb890f8618a27ca26868cebbddf1224983e631b1cbc002c1a4d1cd0d65fba9fea49a
+)
+
+vcpkg_extract_source_archive(SOURCE_PATH
+    ARCHIVE "${ARCHIVE}"
+    PATCHES
+        8.2p1.diff
+)
+
+vcpkg_configure_make(
+    SOURCE_PATH "${SOURCE_PATH}"
+    DETERMINE_BUILD_TRIPLET
+    OPTIONS
+        --with-curses=yes
+        --disable-install-examples
+)
+
+vcpkg_install_make()
+
+file(REMOVE_RECURSE
+    "${CURRENT_PACKAGES_DIR}/debug/share"
+    "${CURRENT_PACKAGES_DIR}/tools"
+)
+
+vcpkg_fixup_pkgconfig()
+
+file(INSTALL "${SOURCE_PATH}/COPYING" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright)

overlay-ports/readline-unix/vcpkg.json

@@ -0,0 +1,12 @@
+{
+  "name": "readline-unix",
+  "version": "8.2",
+  "port-version": 1,
+  "description": "The GNU Readline library provides a set of functions for use by applications that allow users to edit command lines as they are typed in.",
+  "homepage": "https://tiswww.case.edu/php/chet/readline/rltop.html",
+  "license": "GPL-3.0-or-later",
+  "supports": "!windows",
+  "dependencies": [
+    "ncurses"
+  ]
+}

overlay-ports/readline/portfile.cmake

@@ -0,0 +1,6 @@
+if(VCPKG_CMAKE_SYSTEM_NAME STREQUAL "WindowsStore")
+    message(FATAL_ERROR "No implementation of readline is currently available for UWP targets")
+endif()
+
+set(VCPKG_POLICY_EMPTY_PACKAGE enabled)
+FILE(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}")

overlay-ports/readline/usage

@@ -0,0 +1,10 @@
+the package readline can be used under windows via:
+
+    find_package(unofficial-readline-win32 CONFIG REQUIRED)
+    target_link_libraries(main PRIVATE unofficial::readline-win32::readline)
+
+The package readline can be imported via the CMake FindPkgConfig module:
+    
+    find_package(PkgConfig REQUIRED)
+    pkg_check_modules(readline REQUIRED IMPORTED_TARGET readline)
+    target_link_libraries(main PRIVATE PkgConfig::readline)

overlay-ports/readline/vcpkg.json

@@ -0,0 +1,17 @@
+{
+  "name": "readline",
+  "version": "0",
+  "port-version": 5,
+  "description": "GNU readline and history libraries",
+  "supports": "!uwp",
+  "dependencies": [
+    {
+      "name": "readline-unix",
+      "platform": "!windows"
+    },
+    {
+      "name": "readline-win32",
+      "platform": "windows"
+    }
+  ]
+}

release/Makefile

@@ -1,5 +1,5 @@
 
-VERSION=0.12.0
+VERSION=0.12.3
 
 VERSION_TAG=v$(VERSION)
 
@@ -95,7 +95,7 @@ release: osx-package musl-package release-NEWS.md
 
 push:
 	env LANG=UTF-8 package_cloud push tstack/lnav/ubuntu/lucid outbox/lnav*.deb
-	env LANG=UTF-8 package_cloud push tstack/lnav/el/5 outbox/lnav-0.12.0-1.x86_64.rpm
+	env LANG=UTF-8 package_cloud push tstack/lnav/el/5 outbox/lnav-0.12.1-1.x86_64.rpm
 
 clean:
 	cd vagrant-static && vagrant destroy -f

release/lnav.spec.in

@@ -25,11 +25,12 @@ URL:            https://lnav.org
 # Source:       https://github.com/tstack/%{name}/archive/v%{version}.tar.gz#/%{name}-%{version}.tar.gz
 Source:         /github/home/rpmbuild/SOURCES/%{name}-%{version}.tar.gz
 # Source1:      lnav.desktop
-BuildRequires:  gcc-toolset-12
-BuildRequires:  gcc-toolset-12-annobin-plugin-gcc
-BuildRequires:  gcc-toolset-12-annobin-annocheck
+BuildRequires:  gcc-toolset-13
+BuildRequires:  gcc-toolset-13-annobin-plugin-gcc
+BuildRequires:  gcc-toolset-13-annobin-annocheck
 BuildRequires:  autoconf
 BuildRequires:  automake
+BuildRequires:  cargo
 BuildRequires:  libarchive-devel
 BuildRequires:  libcurl-devel
 BuildRequires:  ncurses-devel
@@ -38,6 +39,7 @@ BuildRequires:  openssh
 BuildRequires:  bzip2-devel
 BuildRequires:  pcre2-devel
 BuildRequires:  readline-devel
+BuildRequires:  rust
 BuildRequires:  zlib-devel
 %if 0%{?suse_version}
 BuildRequires:  sqlite3-devel >= 3.9.0
@@ -62,8 +64,7 @@ quickly and efficiently focus on problems.
 %autosetup -p1
 
 %build
-source /opt/rh/gcc-toolset-12/enable
-(cd /opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12/plugin/ && ln -s annobin.so gcc-annobin.so)
+source /opt/rh/gcc-toolset-13/enable
 autoreconf -fiv
 %configure \
      --disable-silent-rules \

snapcraft.yaml

@@ -66,6 +66,7 @@ parts:
       "$SNAPCRAFT_STAGE"/scriptlets/selective-checkout --debug --force-snapshot
     build-packages:
       - build-essential
+      - cargo
       - libarchive-dev
       - libcurl4-gnutls-dev
       - libpcre2-dev
@@ -75,6 +76,7 @@ parts:
       - zlib1g-dev
       - libbz2-dev
       - libgpm-dev
+      - rustc
     stage-packages:
       - zlib1g
       - git-core

src/CMakeLists.txt

@@ -45,16 +45,23 @@ set(TIME_FORMATS
         "%Y-%m-%d %H:%M:%S:%L"
         "%Y-%m-%d %H:%M:%S"
         "%Y-%m-%d %H:%M"
+        "%Y-%m-%dT%H:%M:%S %p %Z"
+        "%Y-%m-%dT%H:%M:%S.%N%z"
+        "%y-%m-%dT%H:%M:%S.%N%z"
         "%Y-%m-%dT%H:%M:%S.%f%z"
         "%y-%m-%dT%H:%M:%S.%f%z"
         "%Y-%m-%dT%H:%M:%S.%L%z"
         "%y-%m-%dT%H:%M:%S.%L%z"
         "%Y-%m-%dT%H:%M:%S%z"
-        "%Y-%m-%dT%H:%M:%S"
         "%Y-%m-%dT%H:%M:%S%z"
+        "%Y-%m-%dT%H:%M:%S"
         "%Y-%m-%dT%H:%M"
         "%Y/%m/%d %H:%M:%S %z"
         "%Y/%m/%d %H:%M:%S%z"
+        "%Y/%m/%d %H:%M:%S.%f%z"
+        "%Y/%m/%d %H:%M:%S.%f%Z"
+        "%Y/%m/%d %H:%M:%S.%f %z"
+        "%Y/%m/%d %H:%M:%S.%f %Z"
         "%Y/%m/%d %H:%M:%S.%f"
         "%Y/%m/%d %H:%M:%S.%L"
         "%Y/%m/%d %H:%M:%S"
@@ -64,6 +71,7 @@ set(TIME_FORMATS
         "%Y %b %d %H:%M:%S"
         "%a %b %d %H:%M:%S %Y"
         "%a %b %d %H:%M:%S.%f %Y"
+        "%a %b %d %H:%M:%S:%f %Y"
         "%a %b %d %H:%M:%S %Z %Y"
         "%a %b %d %I:%M:%S %p %Z %Y"
         "%a %b %d %H:%M:%S "
@@ -79,6 +87,7 @@ set(TIME_FORMATS
         "%d %b %Y %H:%M:%S.%L"
         "%d %b %Y %H:%M:%S,%L"
         "%d %b %Y %H:%M"
+        "%b-%d %H:%M:%S"
         "%b %d %H:%M:%S"
         "%b %d %k:%M:%S"
         "%b %d %l:%M:%S"
@@ -110,6 +119,7 @@ set(TIME_FORMATS
         "%Y/%m/%d"
         "%Y/%m"
         "%s.%f"
+
 )
 
 set(GEN_SRCS "")
@@ -118,7 +128,7 @@ add_custom_command(OUTPUT time_fmts.cc COMMAND ptimec ${TIME_FORMATS} >
         time_fmts.cc)
 
 add_library(lnavdt STATIC config.h.in ptimec.hh ptimec_rt.cc time_fmts.cc)
-target_include_directories(lnavdt PUBLIC . ${CMAKE_CURRENT_BINARY_DIR})
+target_include_directories(lnavdt PUBLIC . ${CMAKE_CURRENT_BINARY_DIR} third-party/date/include)
 
 function(bin2c)
     cmake_parse_arguments(BIN2C_ "" "VARNAME" "" ${ARGN})
@@ -144,8 +154,11 @@ set(FORMAT_FILES
         formats/access_log.json
         formats/alb_log.json
         formats/block_log.json
+        formats/bunyan_log.json
         formats/candlepin_log.json
         formats/choose_repo_log.json
+        formats/cloudflare_log.json
+        formats/cloudvm_ram_log.json
         formats/cups_log.json
         formats/dpkg_log.json
         formats/elb_log.json
@@ -153,18 +166,22 @@ set(FORMAT_FILES
         formats/error_log.json
         formats/esx_syslog_log.json
         formats/fsck_hfs_log.json
+        formats/github_events_log.json
         formats/glog_log.json
         formats/haproxy_log.json
         formats/java_log.json
         formats/journald_json_log.json
         formats/katello_log.json
+        formats/lnav_debug_log.json
+        formats/nextcloud_log.json
+        formats/nextflow_log.json
         formats/openam_log.json
         formats/openamdb_log.json
         formats/openstack_log.json
         formats/page_log.json
-        formats/papertrail_log.json
         formats/pcap_log.json
         formats/procstate_log.json
+        formats/redis_log.json
         formats/snaplogic_log.json
         formats/sssd_log.json
         formats/strace_log.json
@@ -173,13 +190,17 @@ set(FORMAT_FILES
         formats/s3_log.json
         formats/tcf_log.json
         formats/tcsh_history.json
+        formats/unifi_log.json
         formats/uwsgi_log.json
         formats/vdsm_log.json
         formats/vmk_log.json
         formats/vmw_log.json
         formats/vmw_vc_svc_log.json
         formats/vmw_py_log.json
-        formats/xmlrpc_log.json)
+        formats/vpostgres_log.json
+        formats/xmlrpc_log.json
+        formats/zookeeper_log.json
+)
 
 set(FORMAT_FILE_PATHS ${FORMAT_FILES})
 
@@ -191,6 +212,20 @@ add_custom_command(
         DEPENDS bin2c ${FORMAT_FILES})
 list(APPEND GEN_SRCS default-formats.h default-formats.cc)
 
+set(PRQL_FILES
+        prql/stats.prql
+        prql/utils.prql)
+
+set(PRQL_FILE_PATHS ${PRQL_FILES})
+
+list(TRANSFORM PRQL_FILE_PATHS PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
+
+add_custom_command(
+        OUTPUT prql-modules.h prql-modules.cc
+        COMMAND bin2c -n lnav_prql_modules prql-modules ${PRQL_FILE_PATHS}
+        DEPENDS bin2c ${PRQL_FILES})
+list(APPEND GEN_SRCS prql-modules.h prql-modules.cc)
+
 set(CONFIG_FILES
         root-config.json
         keymaps/de-keymap.json
@@ -219,6 +254,7 @@ list(APPEND GEN_SRCS default-config.h default-config.cc)
 set(BUILTIN_LNAV_SCRIPTS
         scripts/dhclient-summary.lnav
         scripts/docker-url-handler.lnav
+        scripts/journald-url-handler.lnav
         scripts/lnav-pop-view.lnav
         scripts/partition-by-boot.lnav
         scripts/piper-url-handler.lnav
@@ -274,6 +310,7 @@ add_library(
         third-party/date/src/tz.cpp
 )
 target_include_directories(datepp PUBLIC third-party/date/include)
+target_link_libraries(datepp ${lnav_LIBS})
 
 add_library(
         cppscnlib STATIC
@@ -385,12 +422,14 @@ add_library(
         collation-functions.cc
         column_namer.cc
         command_executor.cc
+        crashd.client.cc
         curl_looper.cc
         db_sub_source.cc
         dump_internals.cc
         elem_to_json.cc
         environ_vtab.cc
         extension-functions.cc
+        external_opener.cc
         field_overlay_source.cc
         file_collection.cc
         file_converter_manager.cc
@@ -404,7 +443,6 @@ add_library(
         fs-extension-functions.cc
         fstat_vtab.cc
         fts_fuzzy_match.cc
-        gantt_source.cc
         help_text.cc
         help_text_formatter.cc
         highlighter.cc
@@ -466,9 +504,12 @@ add_library(
         styling.cc
         text_anonymizer.cc
         text_format.cc
+        text_link_handler.cc
+        text_overlay_menu.cc
         textfile_highlighters.cc
         textfile_sub_source.cc
         textview_curses.cc
+        timeline_source.cc
         top_status_source.cc
         time-extension-functions.cc
         timer.cc
@@ -502,10 +543,13 @@ add_library(
         byte_array.hh
         command_executor.hh
         column_namer.hh
+        crashd.client.hh
         curl_looper.hh
         doc_status_source.hh
         dump_internals.hh
         elem_to_json.hh
+        external_opener.hh
+        external_opener.cfg.hh
         field_overlay_source.hh
         file_collection.hh
         file_converter_manager.hh
@@ -517,8 +561,6 @@ add_library(
         filter_sub_source.hh
         fstat_vtab.hh
         fts_fuzzy_match.hh
-        gantt_source.hh
-        gantt_status_source.hh
         grep_highlighter.hh
         hasher.hh
         help_text.hh
@@ -553,7 +595,6 @@ add_library(
         logfile_stats.hh
         md2attr_line.hh
         md4cpp.hh
-        optional.hpp
         file_converter_manager.hh
         plain_text_source.hh
         pretty_printer.hh
@@ -592,11 +633,15 @@ add_library(
         termios_guard.hh
         text_anonymizer.hh
         text_format.hh
+        text_link_handler.hh
+        text_overlay_menu.hh
         textfile_highlighters.hh
         textfile_sub_source.hh
         textview_curses.hh
         textview_curses_fwd.hh
         time_T.hh
+        timeline_source.hh
+        timeline_status_source.hh
         timer.hh
         top_status_source.hh
         url_handler.cfg.hh
@@ -615,12 +660,6 @@ add_library(
         mapbox/variant.hpp
         mapbox/variant_io.hpp
         mapbox/variant_visitor.hpp
-        ghc/filesystem.hpp
-        ghc/fs_fwd.hpp
-        ghc/fs_impl.hpp
-        ghc/fs_std.hpp
-        ghc/fs_std_fwd.hpp
-        ghc/fs_std_impl.hpp
         ww898/cp_utf8.hpp
         log_level_re.cc
 
@@ -659,6 +698,9 @@ add_library(
         third-party/md4c/md4c.h
 
         third-party/robin_hood/robin_hood.h
+
+        third-party/prqlc-c/prqlc.cxx.hh
+        third-party/prqlc-c/prqlc.cxx.cc
 )
 
 set(lnav_SRCS lnav.cc)
@@ -667,8 +709,11 @@ target_include_directories(diag PUBLIC . fmtlib ${CMAKE_CURRENT_BINARY_DIR}
         third-party
         third-party/base64/include
         third-party/date/include
+        third-party/prqlc-c
         third-party/rapidyaml
+        ${CURSES_INCLUDE_DIRS}
 )
+target_compile_options(diag PRIVATE ${CURSES_CFLAGS})
 
 target_link_libraries(
         diag

src/Makefile.am

@@ -43,6 +43,7 @@ default-formats.cc: $(BIN2C_PATH) $(FORMAT_FILES)
 	$(BIN2C_V)$(BIN2C_PATH) -n lnav_format_json default-formats $(FORMAT_FILES)
 
 include keymaps/keymaps.am
+include prql/prql.am
 include themes/themes.am
 
 CONFIG_FILES = \
@@ -54,6 +55,9 @@ CONFIG_FILES = \
 default-config.cc: $(BIN2C_PATH) $(CONFIG_FILES)
 	$(BIN2C_V)$(BIN2C_PATH) -n lnav_config_json default-config $(CONFIG_FILES)
 
+prql-modules.cc: $(BIN2C_PATH) $(PRQL_FILES)
+	$(BIN2C_V)$(BIN2C_PATH) -n lnav_prql_modules prql-modules $(PRQL_FILES)
+
 include scripts/scripts.am
 
 builtin-scripts.cc: $(BIN2C_PATH) $(BUILTIN_LNAVSCRIPTS)
@@ -74,6 +78,9 @@ builtin-sh-scripts.cc: $(BIN2C_PATH) $(BUILTIN_SHSCRIPTS)
 %-sql.cc: $(srcdir)/%.sql $(BIN2C_PATH)
 	$(BIN2C_V)$(BIN2C_PATH) $(*)-sql $<
 
+%-prql.cc: $(srcdir)/%.prql $(BIN2C_PATH)
+	$(BIN2C_V)$(BIN2C_PATH) $(*)-prql $<
+
 %-lnav.cc: $(srcdir)/%.lnav $(BIN2C_PATH)
 	$(BIN2C_V)$(BIN2C_PATH) $(*)-lnav $<
 
@@ -104,6 +111,7 @@ LNAV_BUILT_FILES = \
     words-json.cc \
     help-md.cc \
     init-sql.cc \
+    prql-modules.cc \
     time_fmts.cc \
     xml-entities-json.cc \
     xterm-palette-json.cc
@@ -114,6 +122,22 @@ AM_LIBS = $(CODE_COVERAGE_LIBS)
 AM_CFLAGS = $(CODE_COVERAGE_CFLAGS)
 AM_CXXFLAGS = $(CODE_COVERAGE_CXXFLAGS) $(USER_CXXFLAGS)
 
+if HAVE_CARGO
+RUST_DEPS_CPPFLAGS = -DHAVE_RUST_DEPS=1
+PRQLC_DIR = third-party/prqlc-c/target
+RUST_DEPS_LIBS = $(PRQLC_DIR)/release/libprqlc_c.a
+
+$(RUST_DEPS_LIBS): $(srcdir)/third-party/prqlc-c/src/lib.rs $(srcdir)/third-party/prqlc-c/Cargo.toml
+	mkdir -p $(PRQLC_DIR)
+	env CARGO_TARGET_DIR=third-party/prqlc-c/target $(CARGO_CMD) build --manifest-path \
+	    $(srcdir)/third-party/prqlc-c/Cargo.toml --package prqlc-c --release
+
+else
+RUST_DEPS =
+RUST_DEPS_CPPFLAGS =
+RUST_DEPS_LIBS =
+endif
+
 AM_LDFLAGS = \
 	$(STATIC_LDFLAGS) \
 	$(LIBARCHIVE_LDFLAGS) \
@@ -128,6 +152,7 @@ AM_CPPFLAGS = \
 	-I$(srcdir)/third-party \
 	-I$(srcdir)/third-party/base64/include \
 	-I$(srcdir)/third-party/date/include \
+	-I$(srcdir)/third-party/prqlc-c \
 	-I$(srcdir)/third-party/rapidyaml \
 	-I$(top_srcdir)/src/third-party/scnlib/include \
 	-Wall \
@@ -136,7 +161,8 @@ AM_CPPFLAGS = \
 	$(READLINE_CFLAGS) \
 	$(SQLITE3_CFLAGS) \
 	$(PCRE_CFLAGS) \
-	$(LIBCURL_CPPFLAGS)
+	$(LIBCURL_CPPFLAGS) \
+	$(RUST_DEPS_CPPFLAGS)
 
 LDADD = \
 	libdiag.a \
@@ -154,6 +180,7 @@ LDADD = \
 	yajl/libyajl.a \
 	yajlpp/libyajlpp.a \
 	third-party/base64/lib/libbase64.a \
+    $(RUST_DEPS_LIBS) \
 	$(READLINE_LIBS) \
 	$(CURSES_LIB) \
 	$(SQLITE3_LIBS) \
@@ -175,12 +202,16 @@ dist_noinst_DATA = \
 	$(BUILTIN_SHSCRIPTS) \
 	$(CONFIG_FILES) \
 	$(FORMAT_FILES) \
+	$(PRQL_FILES) \
 	words.json \
 	xml-entities.json \
 	xterm-palette.json
 
 noinst_HEADERS = \
     third-party/md4c/md4c.h \
+    third-party/prqlc-c/prqlc.h \
+    third-party/prqlc-c/prqlc.hpp \
+    third-party/prqlc-c/prqlc.cxx.hh \
     third-party/rapidyaml/ryml_all.hpp \
 	all_logs_vtab.hh \
 	archive_manager.hh \
@@ -196,6 +227,7 @@ noinst_HEADERS = \
 	byte_array.hh \
 	column_namer.hh \
 	command_executor.hh \
+	crashd.client.hh \
 	curl_looper.hh \
 	data_scanner.hh \
 	data_scanner_re.re \
@@ -206,6 +238,8 @@ noinst_HEADERS = \
 	dump_internals.hh \
 	elem_to_json.hh \
 	environ_vtab.hh \
+	external_opener.hh \
+	external_opener.cfg.hh \
 	field_overlay_source.hh \
 	file_collection.hh \
 	file_converter_manager.hh \
@@ -218,8 +252,6 @@ noinst_HEADERS = \
 	filter_sub_source.hh \
 	fstat_vtab.hh \
 	fts_fuzzy_match.hh \
-	gantt_source.hh \
-	gantt_status_source.hh \
 	grep_highlighter.hh \
 	grep_proc.hh \
 	hasher.hh \
@@ -271,7 +303,6 @@ noinst_HEADERS = \
 	mapbox/variant_visitor.hpp \
 	md2attr_line.hh \
 	md4cpp.hh \
-	optional.hpp \
 	piper.looper.hh \
 	piper.looper.cfg.hh \
 	plain_text_source.hh \
@@ -318,11 +349,15 @@ noinst_HEADERS = \
 	term_extra.hh \
 	text_anonymizer.hh \
 	text_format.hh \
+	text_link_handler.hh \
+	text_overlay_menu.hh \
 	textfile_highlighters.hh \
 	textfile_sub_source.hh \
 	textview_curses.hh \
 	textview_curses_fwd.hh \
 	time_T.hh \
+	timeline_source.hh \
+	timeline_status_source.hh \
 	timer.hh \
 	top_status_source.hh \
 	top_status_source.cfg.hh \
@@ -345,12 +380,6 @@ noinst_HEADERS = \
 	xpath_vtab.hh \
 	xterm_mouse.hh \
 	spookyhash/SpookyV2.h \
-	ghc/filesystem.hpp \
-	ghc/fs_fwd.hpp \
-	ghc/fs_impl.hpp \
-	ghc/fs_std.hpp \
-	ghc/fs_std_fwd.hpp \
-	ghc/fs_std_impl.hpp \
     ww898/cp_utf8.hpp
 
 nodist_libdiag_a_SOURCES = \
@@ -379,6 +408,7 @@ THIRD_PARTY_SRCS = \
     third-party/doctest-root/doctest/doctest.h \
     third-party/intervaltree/IntervalTree.h \
     third-party/md4c/md4c.c \
+    third-party/prqlc-c/prqlc.cxx.cc \
     third-party/robin_hood/robin_hood.h \
 	third-party/sqlite/ext/dbdump.c \
 	third-party/sqlite/ext/series.c
@@ -397,6 +427,7 @@ libdiag_a_SOURCES = \
 	collation-functions.cc \
 	column_namer.cc \
 	command_executor.cc \
+	crashd.client.cc \
 	curl_looper.cc \
 	db_sub_source.cc \
 	document.sections.cc \
@@ -404,6 +435,7 @@ libdiag_a_SOURCES = \
 	elem_to_json.cc \
 	environ_vtab.cc \
 	extension-functions.cc \
+	external_opener.cc \
 	field_overlay_source.cc \
 	file_collection.cc \
 	file_converter_manager.cc \
@@ -416,7 +448,6 @@ libdiag_a_SOURCES = \
 	fstat_vtab.cc \
     fs-extension-functions.cc \
     fts_fuzzy_match.cc \
-    gantt_source.cc \
 	grep_proc.cc \
 	help_text.cc \
 	help_text_formatter.cc \
@@ -474,6 +505,8 @@ libdiag_a_SOURCES = \
 	styling.cc \
 	text_anonymizer.cc \
 	text_format.cc \
+	text_link_handler.cc \
+	text_overlay_menu.cc \
 	textfile_sub_source.cc \
 	timer.cc \
 	sql_commands.cc \
@@ -483,6 +516,7 @@ libdiag_a_SOURCES = \
 	textfile_highlighters.cc \
 	textview_curses.cc \
 	time-extension-functions.cc \
+	timeline_source.cc \
 	top_status_source.cc \
 	unique_path.cc \
 	view_curses.cc \
@@ -528,6 +562,10 @@ RE2C_FILES = data_scanner_re.cc log_level_re.cc
 endif
 
 EXTRA_DIST = \
+    third-party/prqlc-c/Cargo.lock \
+    third-party/prqlc-c/Cargo.toml \
+    third-party/prqlc-c/cbindgen.toml \
+    third-party/prqlc-c/src/lib.rs \
 	ptimec.c
 
 CLEANFILES = \
@@ -547,11 +585,18 @@ DISTCLEANFILES = \
     words-json.h \
     help-md.h \
     init-sql.h \
+    prql-modules.h \
     time_fmts.h \
     xml-entities-json.h \
     xterm-palette-json.h \
 	$(RE2C_FILES)
 
+if HAVE_CARGO
+clean-local:
+	env CARGO_TARGET_DIR=third-party/prqlc-c/target $(CARGO_CMD) clean --manifest-path \
+    	    $(srcdir)/third-party/prqlc-c/Cargo.toml
+endif
+
 distclean-local:
 	$(RM_V)rm -rf *.dSYM
 
@@ -560,13 +605,19 @@ uncrusty:
 		$(HEADERS))
 
 if !CROSS_COMPILING
-all-local: $(LNAV_BUILT_FILES) lnav
+all-local: $(LNAV_BUILT_FILES) lnav $(RUST_DEPS)
 	if test -w $(srcdir)/internals; then \
 		env DUMP_INTERNALS_DIR=$(srcdir)/internals DUMP_CRASH=1 ./lnav Makefile; \
 		mv $(srcdir)/internals/*.schema.json $(top_srcdir)/docs/schemas; \
 	fi
 else
-all-local: $(LNAV_BUILT_FILES)
+all-local: $(LNAV_BUILT_FILES) $(RUST_DEPS)
+endif
+
+check-local:
+if HAVE_CHECK_JSONSCHEMA
+	$(CHECK_JSONSCHEMA) --schemafile $(top_srcdir)/docs/schemas/format-v1.schema.json $(FORMAT_FILES)
+	$(CHECK_JSONSCHEMA) --schemafile $(top_srcdir)/docs/schemas/config-v1.schema.json $(CONFIG_FILES)
 endif
 
 install-exec-hook:

src/all_logs_vtab.cc

@@ -83,7 +83,7 @@ all_logs_vtab::extract(logfile* lf,
 
     this->vi_attrs.clear();
     sub_values.lvv_sbr = line.clone();
-    format->annotate(line_number, this->vi_attrs, sub_values, false);
+    format->annotate(lf, line_number, this->vi_attrs, sub_values, false);
 
     auto body = find_string_attr_range(this->vi_attrs, &SA_BODY);
     if (body.lr_start == -1) {
@@ -111,6 +111,7 @@ all_logs_vtab::extract(logfile* lf,
         this->alv_values_meta,
         json_string(gen).to_string_fragment().to_string());
     values.lvv_opid_value = std::move(sub_values.lvv_opid_value);
+    values.lvv_opid_provenance = sub_values.lvv_opid_provenance;
 }
 
 bool

src/archive_manager.cc

@@ -52,7 +52,7 @@
 #include "fmt/format.h"
 #include "hasher.hh"
 
-namespace fs = ghc::filesystem;
+namespace fs = std::filesystem;
 
 namespace archive_manager {
 
@@ -74,10 +74,13 @@ enable_desired_archive_formats(archive* arc)
 }
 #endif
 
-bool
-is_archive(const fs::path& filename)
+Result<describe_result, std::string>
+describe(const fs::path& filename)
 {
 #if HAVE_ARCHIVE_H
+    static const auto RAW_FORMAT_NAME = string_fragment::from_const("raw");
+    static const auto GZ_FILTER_NAME = string_fragment::from_const("gzip");
+
     auto_mem<archive> arc(archive_read_free);
 
     arc = archive_read_new();
@@ -96,39 +99,56 @@ is_archive(const fs::path& filename)
         if (archive_read_next_header(arc, &entry) == ARCHIVE_OK) {
             log_debug("read next done %s", filename.c_str());
 
-            static const auto RAW_FORMAT_NAME = string_fragment("raw");
-            static const auto GZ_FILTER_NAME = string_fragment("gzip");
-
             format_name = archive_format_name(arc);
-
             if (RAW_FORMAT_NAME == format_name) {
                 auto filter_count = archive_filter_count(arc);
 
                 if (filter_count == 1) {
-                    return false;
+                    return Ok(describe_result{unknown_file{}});
                 }
 
                 const auto* first_filter_name = archive_filter_name(arc, 0);
                 if (filter_count == 2 && GZ_FILTER_NAME == first_filter_name) {
-                    return false;
+                    return Ok(describe_result{unknown_file{}});
                 }
             }
             log_info(
                 "detected archive: %s -- %s", filename.c_str(), format_name);
-            return true;
+            auto ai = archive_info{
+                format_name,
+            };
+
+            do {
+                ai.ai_entries.emplace_back(archive_info::entry{
+                    archive_entry_pathname_utf8(entry),
+                    archive_entry_strmode(entry),
+                    archive_entry_mtime(entry),
+                    archive_entry_size_is_set(entry)
+                        ? std::make_optional(archive_entry_size(entry))
+                        : std::nullopt,
+                });
+            } while (archive_read_next_header(arc, &entry) == ARCHIVE_OK);
+
+            return Ok(describe_result{ai});
         }
 
-        log_info("archive read header failed: %s -- %s",
-                 filename.c_str(),
-                 archive_error_string(arc));
+        const auto* errstr = archive_error_string(arc);
+        log_info(
+            "archive read header failed: %s -- %s", filename.c_str(), errstr);
+        return Err(
+            fmt::format(FMT_STRING("unable to read archive header: {} -- {}"),
+                        filename,
+                        errstr ? errstr : "not an archive"));
     } else {
-        log_info("archive open failed: %s -- %s",
-                 filename.c_str(),
-                 archive_error_string(arc));
+        const auto* errstr = archive_error_string(arc);
+        log_info("archive open failed: %s -- %s", filename.c_str(), errstr);
+        return Err(fmt::format(FMT_STRING("unable to open file: {} -- {}"),
+                               filename,
+                               errstr ? errstr : "unknown"));
     }
 #endif
 
-    return false;
+    return Ok(describe_result{unknown_file{}});
 }
 
 static fs::path
@@ -249,7 +269,8 @@ extract(const std::string& filename, const extract_cb& cb)
             }
         }
         if (file_count > 0) {
-            fs::last_write_time(done_path, std::chrono::system_clock::now());
+            auto now = fs::file_time_type::clock::now();
+            fs::last_write_time(done_path, now);
             log_info("%s: archive has already been extracted!",
                      done_path.c_str());
             return Ok();
@@ -379,7 +400,7 @@ void
 cleanup_cache()
 {
     (void) std::async(std::launch::async, []() {
-        auto now = std::chrono::system_clock::now();
+        auto now = std::filesystem::file_time_type::clock::now();
         auto cache_path = archive_cache_path();
         const auto& cfg = injector::get<const config&>();
         std::vector<fs::path> to_remove;

src/archive_manager.hh

@@ -37,28 +37,45 @@
 #include <string>
 #include <utility>
 
+#include "base/file_range.hh"
 #include "base/result.h"
-#include "ghc/filesystem.hpp"
+#include <filesystem>
+#include "mapbox/variant.hpp"
 
 namespace archive_manager {
 
 struct extract_progress {
-    extract_progress(ghc::filesystem::path path, ssize_t total)
+    extract_progress(std::filesystem::path path, ssize_t total)
         : ep_path(std::move(path)), ep_total_size(total)
     {
     }
 
-    const ghc::filesystem::path ep_path;
+    const std::filesystem::path ep_path;
     const ssize_t ep_total_size;
     std::atomic<size_t> ep_out_size{0};
 };
 
 using extract_cb
-    = std::function<extract_progress*(const ghc::filesystem::path&, ssize_t)>;
+    = std::function<extract_progress*(const std::filesystem::path&, ssize_t)>;
 
-bool is_archive(const ghc::filesystem::path& filename);
+struct archive_info {
+    struct entry {
+        std::filesystem::path e_name;
+        const char* e_mode;
+        time_t e_mtime;
+        std::optional<file_ssize_t> e_size;
+    };
+    const char* ai_format_name;
+    std::vector<entry> ai_entries;
+};
+struct unknown_file {};
+
+using describe_result = mapbox::util::variant<archive_info, unknown_file>;
+
+Result<describe_result, std::string> describe(
+    const std::filesystem::path& filename);
 
-ghc::filesystem::path filename_to_tmp_path(const std::string& filename);
+std::filesystem::path filename_to_tmp_path(const std::string& filename);
 
 using walk_result_t = Result<void, std::string>;
 
@@ -73,8 +90,8 @@ using walk_result_t = Result<void, std::string>;
 walk_result_t walk_archive_files(
     const std::string& filename,
     const extract_cb& cb,
-    const std::function<void(const ghc::filesystem::path&,
-                             const ghc::filesystem::directory_entry&)>&);
+    const std::function<void(const std::filesystem::path&,
+                             const std::filesystem::directory_entry&)>&);
 
 void cleanup_cache();
 

src/base/CMakeLists.txt

@@ -52,6 +52,7 @@ add_library(
         is_utf8.hh
         isc.hh
         itertools.hh
+        itertools.enumerate.hh
         keycodes.hh
         line_range.hh
         lnav.console.hh
@@ -69,14 +70,24 @@ add_library(
         strnatcmp.h
         time_util.hh
         types.hh
+        wcwidth9.h
 
         ../third-party/xxHash/xxhash.h
         ../third-party/xxHash/xxhash.c
 )
 
 target_include_directories(base PUBLIC . .. ../third-party
+        ../third-party/date/include
+        ${CURSES_INCLUDE_DIRS}
         ${CMAKE_CURRENT_BINARY_DIR}/..)
-target_link_libraries(base cppfmt cppscnlib pcrepp ncurses::libcurses pthread lnavdt datepp)
+target_link_libraries(base
+        cppfmt
+        cppscnlib
+        pcrepp
+        ${CURSES_LIBRARIES}
+        pthread
+        lnavdt
+        datepp)
 
 add_executable(
         test_base

src/base/Makefile.am

@@ -48,6 +48,7 @@ noinst_HEADERS = \
     is_utf8.hh \
     isc.hh \
     itertools.hh \
+    itertools.enumerate.hh \
     keycodes.hh \
     line_range.hh \
     lnav_log.hh \
@@ -68,7 +69,8 @@ noinst_HEADERS = \
     string_util.hh \
     strnatcmp.h \
     time_util.hh \
-    types.hh
+    types.hh \
+    wcwidth9.h
 
 libbase_a_SOURCES = \
     ansi_scrubber.cc \

src/base/ansi_scrubber.cc

@@ -56,7 +56,7 @@ erase_ansi_escapes(string_fragment input)
     static thread_local auto md = lnav::pcre2pp::match_data::unitialized();
 
     const auto& regex = ansi_regex();
-    nonstd::optional<int> move_start;
+    std::optional<int> move_start;
     size_t fill_index = 0;
 
     auto matcher = regex.capture_from(input).into(md);
@@ -125,12 +125,15 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
     static const auto semi_pred = string_fragment::tag1{';'};
 
     const auto& regex = ansi_regex();
-    int64_t origin_offset = 0;
-    int last_origin_offset_end = 0;
-    nonstd::optional<std::string> href;
+    std::optional<std::string> href;
     size_t href_start = 0;
+    string_attrs_t tmp_sa;
+    size_t cp_dst = std::string::npos;
+    size_t cp_start = std::string::npos;
+    int last_origin_end = 0;
+    int erased = 0;
 
-    replace(str.begin(), str.end(), '\0', ' ');
+    std::replace(str.begin(), str.end(), '\0', ' ');
     auto matcher = regex.capture_from(str).into(md);
     while (true) {
         auto match_res = matcher.matches(PCRE2_NO_UTF_CHECK);
@@ -146,13 +149,22 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
         const auto sf = md[0].value();
         auto bs_index_res = sf.codepoint_to_byte_index(1);
 
+        if (cp_dst != std::string::npos) {
+            auto cp_len = sf.sf_begin - cp_start;
+            memmove(&str[cp_dst], &str[cp_start], cp_len);
+            cp_dst += cp_len;
+        } else {
+            cp_dst = sf.sf_begin;
+        }
+
         if (sf.length() >= 3 && bs_index_res.isOk()
             && sf[bs_index_res.unwrap()] == '\b')
         {
-            ssize_t fill_index = sf.sf_begin;
+            ssize_t fill_index = cp_dst;
             line_range bold_range;
             line_range ul_range;
             auto sub_sf = sf;
+            auto mid_sf = string_fragment();
 
             while (!sub_sf.empty()) {
                 auto lhs_opt = sub_sf.consume_codepoint();
@@ -170,14 +182,13 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
                     return;
                 }
                 auto rhs_pair = rhs_opt.value();
-                sub_sf = rhs_pair.second;
 
                 if (lhs_pair.first == '_' || rhs_pair.first == '_') {
                     if (sa != nullptr && bold_range.is_valid()) {
                         shift_string_attrs(
                             *sa, bold_range.lr_start, -bold_range.length() * 2);
-                        sa->emplace_back(bold_range,
-                                         VC_STYLE.value(text_attrs{A_BOLD}));
+                        tmp_sa.emplace_back(bold_range,
+                                            VC_STYLE.value(text_attrs{A_BOLD}));
                         bold_range.clear();
                     }
                     if (ul_range.is_valid()) {
@@ -191,11 +202,13 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
                     ww898::utf::utf8::write(cp, [&str, &fill_index](auto ch) {
                         str[fill_index++] = ch;
                     });
-                } else {
+                } else if (lhs_pair.first == rhs_pair.first
+                           && !fmt::v10::detail::needs_escape(lhs_pair.first))
+                {
                     if (sa != nullptr && ul_range.is_valid()) {
                         shift_string_attrs(
                             *sa, ul_range.lr_start, -ul_range.length() * 2);
-                        sa->emplace_back(
+                        tmp_sa.emplace_back(
                             ul_range, VC_STYLE.value(text_attrs{A_UNDERLINE}));
                         ul_range.clear();
                     }
@@ -214,43 +227,47 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
                         log_error("invalid UTF-8 at %d", sf.sf_begin);
                         return;
                     }
+                } else {
+                    mid_sf = mid_pair.second;
+                    break;
                 }
+                sub_sf = rhs_pair.second;
             }
 
-            auto output_size = fill_index - sf.sf_begin;
-            auto erased_size = sf.length() - output_size;
-
+            auto output_size = fill_index - cp_dst;
             if (sa != nullptr && ul_range.is_valid()) {
                 shift_string_attrs(
                     *sa, ul_range.lr_start, -ul_range.length() * 2);
-                sa->emplace_back(ul_range,
-                                 VC_STYLE.value(text_attrs{A_UNDERLINE}));
+                tmp_sa.emplace_back(ul_range,
+                                    VC_STYLE.value(text_attrs{A_UNDERLINE}));
                 ul_range.clear();
             }
             if (sa != nullptr && bold_range.is_valid()) {
                 shift_string_attrs(
                     *sa, bold_range.lr_start, -bold_range.length() * 2);
-                sa->emplace_back(bold_range,
-                                 VC_STYLE.value(text_attrs{A_BOLD}));
+                tmp_sa.emplace_back(bold_range,
+                                    VC_STYLE.value(text_attrs{A_BOLD}));
                 bold_range.clear();
             }
-            if (sa != nullptr) {
-                sa->emplace_back(line_range{last_origin_offset_end,
-                                            sf.sf_begin + (int) output_size},
-                                 SA_ORIGIN_OFFSET.value(origin_offset));
+            if (sa != nullptr && output_size > 0 && cp_dst > 0) {
+                tmp_sa.emplace_back(
+                    line_range{
+                        (int) last_origin_end,
+                        (int) cp_dst + (int) output_size,
+                    },
+                    SA_ORIGIN_OFFSET.value(erased));
             }
-
-            str.erase(str.begin() + fill_index, str.begin() + sf.sf_end);
-            last_origin_offset_end = sf.sf_begin + output_size;
-            origin_offset += erased_size;
-            matcher.reload_input(str, last_origin_offset_end);
+            last_origin_end = cp_dst + output_size;
+            cp_dst = fill_index;
+            cp_start = sub_sf.sf_begin;
+            erased += sf.length() - output_size;
             continue;
         }
 
         struct line_range lr;
         text_attrs attrs;
         bool has_attrs = false;
-        nonstd::optional<role_t> role;
+        std::optional<role_t> role;
 
         if (md[3]) {
             auto osc_id = scn::scan_value<int32_t>(md[3]->to_string_view());
@@ -265,16 +282,18 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
 
                             if (href) {
                                 if (sa != nullptr) {
-                                    sa->emplace_back(
-                                        line_range{(int) href_start,
-                                                   (int) str.size()},
+                                    tmp_sa.emplace_back(
+                                        line_range{
+                                            (int) href_start,
+                                            (int) cp_dst,
+                                        },
                                         VC_HYPERLINK.value(href.value()));
                                 }
-                                href = nonstd::nullopt;
+                                href = std::nullopt;
                             }
                             if (!uri.empty()) {
                                 href = uri.to_string();
-                                href_start = sf.sf_begin;
+                                href_start = cp_dst;
                             }
                         }
                         break;
@@ -323,7 +342,10 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
                                         semi_pred);
                                 auto color_index = scn::scan_value<short>(
                                     color_index_pair.first.to_string_view());
-                                if (!color_index.has_value()) {
+                                if (!color_index.has_value()
+                                    || color_index.value() < 0
+                                    || color_index.value() > 255)
+                                {
                                     break;
                                 }
                                 if (ansi_code == 38) {
@@ -357,6 +379,7 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
                     has_attrs = true;
                     break;
 
+#if 0
                 case 'C': {
                     auto spaces_res
                         = scn::scan_value<unsigned int>(seq.to_string_view());
@@ -384,6 +407,7 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
                     }
                     break;
                 }
+#endif
 
                 case 'O': {
                     auto role_res = scn::scan_value<int>(seq.to_string_view());
@@ -402,40 +426,54 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
             }
         }
         if (md[1] || md[3] || md[5]) {
-            str.erase(str.begin() + sf.sf_begin, str.begin() + sf.sf_end);
             if (sa != nullptr) {
                 shift_string_attrs(*sa, sf.sf_begin, -sf.length());
 
                 if (has_attrs) {
-                    for (auto rit = sa->rbegin(); rit != sa->rend(); rit++) {
+                    for (auto rit = tmp_sa.rbegin(); rit != tmp_sa.rend();
+                         rit++)
+                    {
                         if (rit->sa_range.lr_end != -1) {
                             continue;
                         }
-                        rit->sa_range.lr_end = sf.sf_begin;
+                        rit->sa_range.lr_end = cp_dst;
                     }
-                    lr.lr_start = sf.sf_begin;
+                    lr.lr_start = cp_dst;
                     lr.lr_end = -1;
                     if (!attrs.empty()) {
-                        sa->emplace_back(lr, VC_STYLE.value(attrs));
+                        tmp_sa.emplace_back(lr, VC_STYLE.value(attrs));
                     }
-                    role | [&lr, &sa](role_t r) {
-                        sa->emplace_back(lr, VC_ROLE.value(r));
+                    role | [&lr, &tmp_sa](role_t r) {
+                        tmp_sa.emplace_back(lr, VC_ROLE.value(r));
                     };
                 }
-                sa->emplace_back(
-                    line_range{last_origin_offset_end, sf.sf_begin},
-                    SA_ORIGIN_OFFSET.value(origin_offset));
-                last_origin_offset_end = sf.sf_begin;
-                origin_offset += sf.length();
+                if (cp_dst > 0) {
+                    tmp_sa.emplace_back(
+                        line_range{
+                            (int) last_origin_end,
+                            (int) cp_dst,
+                        },
+                        SA_ORIGIN_OFFSET.value(erased));
+                }
+                last_origin_end = cp_dst;
             }
-
-            matcher.reload_input(str, sf.sf_begin);
+            erased += sf.length();
         }
+        cp_start = sf.sf_end;
     }
 
-    if (sa != nullptr && last_origin_offset_end > 0) {
-        sa->emplace_back(line_range{last_origin_offset_end, (int) str.size()},
-                         SA_ORIGIN_OFFSET.value(origin_offset));
+    if (cp_dst != std::string::npos) {
+        auto cp_len = str.size() - cp_start;
+        memmove(&str[cp_dst], &str[cp_start], cp_len);
+        cp_dst += cp_len;
+        str.resize(cp_dst);
+    }
+    if (sa != nullptr && last_origin_end > 0 && last_origin_end != str.size()) {
+        tmp_sa.emplace_back(line_range{(int) last_origin_end, (int) str.size()},
+                            SA_ORIGIN_OFFSET.value(erased));
+    }
+    if (sa != nullptr) {
+        sa->insert(sa->end(), tmp_sa.begin(), tmp_sa.end());
     }
 }
 

src/base/attr_line.builder.cc

@@ -28,3 +28,57 @@
  */
 
 #include "attr_line.builder.hh"
+
+attr_line_builder&
+attr_line_builder::append_as_hexdump(const string_fragment& sf)
+{
+    auto byte_off = size_t{0};
+    for (auto ch : sf) {
+        if (byte_off == 8) {
+            this->append(" ");
+        }
+        std::optional<role_t> ro;
+        if (ch == '\0') {
+            ro = role_t::VCR_NULL;
+        } else if (isspace(ch) || iscntrl(ch)) {
+            ro = role_t::VCR_ASCII_CTRL;
+        } else if (!isprint(ch)) {
+            ro = role_t::VCR_NON_ASCII;
+        }
+        auto ag = ro.has_value() ? this->with_attr(VC_ROLE.value(ro.value()))
+                                 : this->with_default();
+        this->appendf(FMT_STRING(" {:0>2x}"), ch);
+        byte_off += 1;
+    }
+    for (; byte_off < 16; byte_off++) {
+        if (byte_off == 8) {
+            this->append(" ");
+        }
+        this->append("   ");
+    }
+    this->append("  ");
+    byte_off = 0;
+    for (auto ch : sf) {
+        if (byte_off == 8) {
+            this->append(" ");
+        }
+        if (ch == '\0') {
+            auto ag = this->with_attr(VC_ROLE.value(role_t::VCR_NULL));
+            this->append("\u22c4");
+        } else if (isspace(ch)) {
+            auto ag = this->with_attr(VC_ROLE.value(role_t::VCR_ASCII_CTRL));
+            this->append("_");
+        } else if (iscntrl(ch)) {
+            auto ag = this->with_attr(VC_ROLE.value(role_t::VCR_ASCII_CTRL));
+            this->append("\u2022");
+        } else if (isprint(ch)) {
+            this->alb_line.get_string().push_back(ch);
+        } else {
+            auto ag = this->with_attr(VC_ROLE.value(role_t::VCR_NON_ASCII));
+            this->append("\u00d7");
+        }
+        byte_off += 1;
+    }
+
+    return *this;
+}

src/base/attr_line.builder.hh

@@ -41,7 +41,7 @@ public:
     class attr_guard {
     public:
         explicit attr_guard(attr_line_t& al)
-            : ag_line(al), ag_start(nonstd::nullopt)
+            : ag_line(al), ag_start(std::nullopt)
         {
         }
 
@@ -59,7 +59,7 @@ public:
             : ag_line(other.ag_line), ag_start(std::move(other.ag_start)),
               ag_attr(std::move(other.ag_attr))
         {
-            other.ag_start = nonstd::nullopt;
+            other.ag_start = std::nullopt;
         }
 
         ~attr_guard()
@@ -76,7 +76,7 @@ public:
 
     private:
         attr_line_t& ag_line;
-        nonstd::optional<int> ag_start;
+        std::optional<int> ag_start;
         string_attr_pair ag_attr;
     };
 
@@ -127,6 +127,8 @@ public:
         return *this;
     }
 
+    attr_line_builder& append_as_hexdump(const string_fragment& sf);
+
 private:
     attr_line_t& alb_line;
 };

src/base/attr_line.cc

@@ -356,6 +356,19 @@ attr_line_t::insert(size_t index,
     return *this;
 }
 
+attr_line_t&
+attr_line_t::wrap_with(text_wrap_settings* tws)
+{
+    attr_line_t tmp;
+
+    tmp.al_string = std::move(this->al_string);
+    tmp.al_attrs = std::move(this->al_attrs);
+
+    this->append(tmp, tws);
+
+    return *this;
+}
+
 attr_line_t
 attr_line_t::subline(size_t start, size_t len) const
 {
@@ -457,7 +470,7 @@ attr_line_t::apply_hide()
 }
 
 attr_line_t&
-attr_line_t::rtrim()
+attr_line_t::rtrim(std::optional<const char*> chars)
 {
     auto index = this->al_string.length();
 
@@ -468,7 +481,12 @@ attr_line_t::rtrim()
         {
             break;
         }
-        if (!isspace(this->al_string[index - 1])) {
+        if (chars
+            && strchr(chars.value(), this->al_string[index - 1]) == nullptr)
+        {
+            break;
+        }
+        if (!chars && !isspace(this->al_string[index - 1])) {
             break;
         }
     }
@@ -490,7 +508,9 @@ attr_line_t::erase(size_t pos, size_t len)
 
     this->al_string.erase(pos, len);
 
-    shift_string_attrs(this->al_attrs, pos, -((int32_t) len));
+    shift_string_attrs(this->al_attrs,
+                       line_range{(int) pos, (int) (pos + len)},
+                       -((int32_t) len));
     auto new_end = std::remove_if(
         this->al_attrs.begin(), this->al_attrs.end(), [](const auto& attr) {
             return attr.sa_range.empty();
@@ -503,7 +523,7 @@ attr_line_t::erase(size_t pos, size_t len)
 attr_line_t&
 attr_line_t::pad_to(ssize_t size)
 {
-    const auto curr_len = this->utf8_length_or_length();
+    const auto curr_len = this->column_width();
 
     if (curr_len < size) {
         this->append((size - curr_len), ' ');
@@ -541,12 +561,18 @@ line_range::shift_range(const line_range& cover, int32_t amount)
         if (this->lr_end != -1) {
             this->lr_end = std::max(0, this->lr_end + amount);
         }
-    } else if (this->lr_end != -1) {
-        if (cover.lr_start < this->lr_end) {
-            if (amount < 0 && amount < (cover.lr_start - this->lr_end)) {
-                this->lr_end = cover.lr_start;
-            } else {
-                this->lr_end = std::max(this->lr_start, this->lr_end + amount);
+    } else {
+        if (amount < 0 && cover.contains(*this)) {
+            this->lr_start = cover.lr_start;
+        }
+        if (this->lr_end != -1) {
+            if (cover.lr_start < this->lr_end) {
+                if (amount < 0 && amount < (cover.lr_start - this->lr_end)) {
+                    this->lr_end = cover.lr_start;
+                } else {
+                    this->lr_end
+                        = std::max(this->lr_start, this->lr_end + amount);
+                }
             }
         }
     }
@@ -677,9 +703,8 @@ find_string_attr(const string_attrs_t& sa,
                  const string_attr_type_base* type,
                  int start)
 {
-    string_attrs_t::const_iterator iter;
-
-    for (iter = sa.begin(); iter != sa.end(); ++iter) {
+    auto iter = sa.begin();
+    for (; iter != sa.end(); ++iter) {
         if (iter->sa_type == type && iter->sa_range.lr_start >= start) {
             break;
         }
@@ -688,7 +713,7 @@ find_string_attr(const string_attrs_t& sa,
     return iter;
 }
 
-nonstd::optional<const string_attr*>
+std::optional<const string_attr*>
 get_string_attr(const string_attrs_t& sa,
                 const string_attr_type_base* type,
                 int start)
@@ -696,8 +721,8 @@ get_string_attr(const string_attrs_t& sa,
     auto iter = find_string_attr(sa, type, start);
 
     if (iter == sa.end()) {
-        return nonstd::nullopt;
+        return std::nullopt;
     }
 
-    return nonstd::make_optional(&(*iter));
+    return std::make_optional(&(*iter));
 }

src/base/attr_line.hh

@@ -33,6 +33,7 @@
 #define attr_line_hh
 
 #include <new>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -97,11 +98,11 @@ using string_attrs_t = std::vector<string_attr>;
 string_attrs_t::const_iterator find_string_attr(
     const string_attrs_t& sa, const string_attr_type_base* type, int start = 0);
 
-nonstd::optional<const string_attr*> get_string_attr(
+std::optional<const string_attr*> get_string_attr(
     const string_attrs_t& sa, const string_attr_type_base* type, int start = 0);
 
 template<typename T>
-inline nonstd::optional<string_attr_wrapper<T>>
+inline std::optional<string_attr_wrapper<T>>
 get_string_attr(const string_attrs_t& sa,
                 const string_attr_type<T>& type,
                 int start = 0)
@@ -109,10 +110,10 @@ get_string_attr(const string_attrs_t& sa,
     auto iter = find_string_attr(sa, &type, start);
 
     if (iter == sa.end()) {
-        return nonstd::nullopt;
+        return std::nullopt;
     }
 
-    return nonstd::make_optional(string_attr_wrapper<T>(&(*iter)));
+    return std::make_optional(string_attr_wrapper<T>(&(*iter)));
 }
 
 template<typename T>
@@ -221,6 +222,13 @@ public:
         return retval.with_ansi_string("%s", str);
     }
 
+    static inline attr_line_t from_ansi_str(const std::string& str)
+    {
+        attr_line_t retval;
+
+        return retval.with_ansi_string(str);
+    }
+
     /** @return The string itself. */
     std::string& get_string() { return this->al_string; }
 
@@ -293,7 +301,7 @@ public:
 
         size_t start_len = this->al_string.length();
 
-        this->al_string.append(std::move(value.first));
+        this->append(std::move(value.first));
 
         line_range lr{(int) start_len, (int) this->al_string.length()};
 
@@ -313,15 +321,6 @@ public:
         return *this;
     }
 
-    attr_line_t& append_quoted(const attr_line_t& al)
-    {
-        this->al_string.append("\u201c");
-        this->append(al);
-        this->al_string.append("\u201d");
-
-        return *this;
-    }
-
     template<typename S>
     attr_line_t& append_quoted(S s)
     {
@@ -344,13 +343,25 @@ public:
         return *this;
     }
 
-    template<typename S>
-    attr_line_t& append(S str)
+    attr_line_t& append(const std::string& str)
     {
         this->al_string.append(str);
         return *this;
     }
 
+    attr_line_t& append(const char* str)
+    {
+        this->al_string.append(str);
+        return *this;
+    }
+
+    template<typename V>
+    attr_line_t& append(const V& v)
+    {
+        this->al_string.append(fmt::to_string(v));
+        return *this;
+    }
+
     template<typename... Args>
     attr_line_t& appendf(fmt::format_string<Args...> fstr, Args&&... args)
     {
@@ -472,7 +483,7 @@ public:
 
     attr_line_t& erase(size_t pos, size_t len = std::string::npos);
 
-    attr_line_t& rtrim();
+    attr_line_t& rtrim(std::optional<const char*> chars = std::nullopt);
 
     attr_line_t& erase_utf8_chars(size_t start)
     {
@@ -510,6 +521,11 @@ public:
         return utf8_string_length(this->al_string).unwrapOr(this->length());
     }
 
+    size_t column_width() const
+    {
+        return string_fragment::from_str(this->al_string).column_width();
+    }
+
     std::string get_substring(const line_range& lr) const
     {
         if (!lr.is_valid()) {
@@ -564,8 +580,13 @@ public:
 
     size_t nearest_text(size_t x) const;
 
+    attr_line_t& wrap_with(text_wrap_settings* tws);
+
     void apply_hide();
 
+    attr_line_t move() & { return std::move(*this); }
+    attr_line_t move() && { return std::move(*this); }
+
     std::string al_string;
     string_attrs_t al_attrs;
 };

src/base/auto_fd.cc

@@ -79,12 +79,18 @@ auto_fd::openpt(int flags)
     return Ok(auto_fd{rc});
 }
 
-auto_fd::auto_fd(int fd) : af_fd(fd)
+auto_fd::
+auto_fd(int fd)
+    : af_fd(fd)
 {
     require(fd >= -1);
 }
 
-auto_fd::auto_fd(auto_fd&& af) noexcept : af_fd(af.release()) {}
+auto_fd::
+auto_fd(auto_fd&& af) noexcept
+    : af_fd(af.release())
+{
+}
 
 auto_fd
 auto_fd::dup() const
@@ -98,11 +104,18 @@ auto_fd::dup() const
     return auto_fd{new_fd};
 }
 
-auto_fd::~auto_fd()
+auto_fd::~
+auto_fd()
 {
     this->reset();
 }
 
+void
+auto_fd::copy_to(int fd) const
+{
+    dup2(this->get(), fd);
+}
+
 void
 auto_fd::reset(int fd)
 {
@@ -184,7 +197,8 @@ auto_pipe::for_child_fd(int child_fd)
     return Ok(std::move(retval));
 }
 
-auto_pipe::auto_pipe(int child_fd, int child_flags)
+auto_pipe::
+auto_pipe(int child_fd, int child_flags)
     : ap_child_flags(child_flags), ap_child_fd(child_fd)
 {
     switch (child_fd) {

src/base/auto_fd.hh

@@ -146,6 +146,8 @@ public:
         return retval;
     }
 
+    void copy_to(int fd) const;
+
     /**
      * @return The file descriptor.
      */

src/base/auto_mem.hh

@@ -223,7 +223,7 @@ public:
         this->ab_capacity = 0;
     }
 
-    auto_buffer& operator=(auto_buffer&) = delete;
+    auto_buffer& operator=(const auto_buffer&) = delete;
 
     auto_buffer& operator=(auto_buffer&& other) noexcept
     {

src/base/color_spaces.cc

@@ -163,3 +163,14 @@ lab_color::operator!=(const lab_color& rhs) const
 {
     return !(rhs == *this);
 }
+
+bool
+lab_color::sufficient_contrast(const lab_color& other) const
+{
+    if (std::abs(this->lc_l - other.lc_l) > 15) {
+        return true;
+    }
+
+    return (std::signbit(this->lc_a) != std::signbit(other.lc_a)
+            || std::signbit(this->lc_b) != std::signbit(other.lc_b));
+}

src/base/color_spaces.hh

@@ -65,6 +65,8 @@ struct lab_color {
 
     double deltaE(const lab_color& other) const;
 
+    bool sufficient_contrast(const lab_color& other) const;
+
     lab_color& operator=(const lab_color& other)
     {
         this->lc_l = other.lc_l;

src/base/date_time_scanner.cc

@@ -47,8 +47,14 @@ date_time_scanner::ftime(char* dst,
 {
     off_t off = 0;
 
-    if (time_fmt == nullptr) {
-        PTIMEC_FORMATS[this->dts_fmt_lock].pf_ffunc(dst, off, len, tm);
+    if (time_fmt == nullptr || this->dts_fmt_lock == -1
+        || (tm.et_flags & ETF_MACHINE_ORIENTED))
+    {
+        auto index
+            = this->dts_fmt_lock != -1 && !(tm.et_flags & ETF_MACHINE_ORIENTED)
+            ? this->dts_fmt_lock
+            : PTIMEC_DEFAULT_FMT_INDEX;
+        PTIMEC_FORMATS[index].pf_ffunc(dst, off, len, tm);
         if (tm.et_flags & ETF_SUB_NOT_IN_FORMAT) {
             if (tm.et_flags & ETF_MILLIS_SET) {
                 dst[off++] = '.';
@@ -61,6 +67,9 @@ date_time_scanner::ftime(char* dst,
                 ftime_N(dst, off, len, tm);
             }
         }
+        if (index == PTIMEC_DEFAULT_FMT_INDEX && tm.et_flags & ETF_ZONE_SET) {
+            ftime_z(dst, off, len, tm);
+        }
         dst[off] = '\0';
     } else {
         off = ftime_fmt(dst, len, time_fmt[this->dts_fmt_lock], tm);
@@ -186,7 +195,6 @@ date_time_scanner::scan(const char* time_dest,
                 {
                     const auto sec_diff = tm_out->et_tm.tm_sec - last_tm.tm_sec;
 
-                    // log_debug("diff %d", sec_diff);
                     tv_out = this->dts_last_tv;
                     tv_out.tv_sec += sec_diff;
                     tm_out->et_tm.tm_wday = last_tm.tm_wday;
@@ -222,20 +230,43 @@ date_time_scanner::scan(const char* time_dest,
                 if (convert_local
                     && (this->dts_local_time
                         || tm_out->et_flags & ETF_EPOCH_TIME
-                        || (tm_out->et_flags & ETF_ZONE_SET
+                        || ((tm_out->et_flags & ETF_ZONE_SET
+                             || this->dts_default_zone != nullptr)
                             && this->dts_zoned_to_local)))
                 {
                     time_t gmt = tm_out->to_timeval().tv_sec;
 
+                    if (!(tm_out->et_flags & ETF_ZONE_SET)
+                        && !(tm_out->et_flags & ETF_EPOCH_TIME)
+                        && this->dts_default_zone != nullptr)
+                    {
+                        date::local_seconds stime;
+                        stime += std::chrono::seconds{gmt};
+                        auto ztime
+                            = date::make_zoned(this->dts_default_zone, stime);
+                        gmt = std::chrono::duration_cast<std::chrono::seconds>(
+                                  ztime.get_sys_time().time_since_epoch())
+                                  .count();
+                    }
                     this->to_localtime(gmt, *tm_out);
-#ifdef HAVE_STRUCT_TM_TM_ZONE
-                    tm_out->et_tm.tm_zone = nullptr;
-#endif
-                    tm_out->et_tm.tm_isdst = 0;
                 }
+                const auto& last_tm = this->dts_last_tm.et_tm;
+                if (last_tm.tm_year == tm_out->et_tm.tm_year
+                    && last_tm.tm_mon == tm_out->et_tm.tm_mon
+                    && last_tm.tm_mday == tm_out->et_tm.tm_mday
+                    && last_tm.tm_hour == tm_out->et_tm.tm_hour
+                    && last_tm.tm_min == tm_out->et_tm.tm_min)
+                {
+                    const auto sec_diff = tm_out->et_tm.tm_sec - last_tm.tm_sec;
 
-                tv_out = tm_out->to_timeval();
-                secs2wday(tv_out, &tm_out->et_tm);
+                    tv_out = this->dts_last_tv;
+                    tv_out.tv_sec += sec_diff;
+                    tm_out->et_tm.tm_wday = last_tm.tm_wday;
+                } else {
+                    tv_out = tm_out->to_timeval();
+                    secs2wday(tv_out, &tm_out->et_tm);
+                }
+                tv_out.tv_usec = tm_out->et_nsec / 1000;
 
                 this->dts_fmt_lock = curr_time_fmt;
                 this->dts_fmt_len = retval - time_dest;

src/base/file_range.hh

@@ -43,6 +43,18 @@ public:
     struct metadata {
         bool m_valid_utf{true};
         bool m_has_ansi{false};
+
+        metadata& operator|=(const metadata& meta)
+        {
+            if (!meta.m_valid_utf) {
+                this->m_valid_utf = false;
+            }
+            if (meta.m_has_ansi) {
+                this->m_has_ansi = true;
+            }
+
+            return *this;
+        }
     };
 
     file_off_t fr_offset{0};

src/base/fs_util.cc

@@ -27,20 +27,54 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <fstream>
+
 #include "fs_util.hh"
 
 #include <stdlib.h>
+#include <sys/param.h>
 
 #include "config.h"
 #include "fmt/format.h"
 #include "itertools.hh"
+#include "lnav_log.hh"
 #include "opt_util.hh"
 
 namespace lnav {
 namespace filesystem {
 
-Result<ghc::filesystem::path, std::string>
-realpath(const ghc::filesystem::path& path)
+std::string
+escape_path(const std::filesystem::path& p)
+{
+    auto p_str = p.string();
+    std::string retval;
+
+    for (const auto ch : p_str) {
+        switch (ch) {
+            case ' ':
+            case '$':
+            case '\\':
+            case ';':
+            case '&':
+            case '<':
+            case '>':
+            case '\'':
+            case '"':
+            case '*':
+            case '[':
+            case ']':
+            case '?':
+                retval.push_back('\\');
+                break;
+        }
+        retval.push_back(ch);
+    }
+
+    return retval;
+}
+
+Result<std::filesystem::path, std::string>
+realpath(const std::filesystem::path& path)
 {
     char resolved[PATH_MAX];
     auto rc = ::realpath(path.c_str(), resolved);
@@ -49,11 +83,11 @@ realpath(const ghc::filesystem::path& path)
         return Err(std::string(strerror(errno)));
     }
 
-    return Ok(ghc::filesystem::path(resolved));
+    return Ok(std::filesystem::path(resolved));
 }
 
 Result<auto_fd, std::string>
-create_file(const ghc::filesystem::path& path, int flags, mode_t mode)
+create_file(const std::filesystem::path& path, int flags, mode_t mode)
 {
     auto fd = openp(path, flags | O_CREAT, mode);
 
@@ -67,7 +101,7 @@ create_file(const ghc::filesystem::path& path, int flags, mode_t mode)
 }
 
 Result<auto_fd, std::string>
-open_file(const ghc::filesystem::path& path, int flags)
+open_file(const std::filesystem::path& path, int flags)
 {
     auto fd = openp(path, flags);
 
@@ -80,8 +114,8 @@ open_file(const ghc::filesystem::path& path, int flags)
     return Ok(auto_fd(fd));
 }
 
-Result<std::pair<ghc::filesystem::path, auto_fd>, std::string>
-open_temp_file(const ghc::filesystem::path& pattern)
+Result<std::pair<std::filesystem::path, auto_fd>, std::string>
+open_temp_file(const std::filesystem::path& pattern)
 {
     auto pattern_str = pattern.string();
     char pattern_copy[pattern_str.size() + 1];
@@ -95,14 +129,14 @@ open_temp_file(const ghc::filesystem::path& pattern)
                         strerror(errno)));
     }
 
-    return Ok(std::make_pair(ghc::filesystem::path(pattern_copy), auto_fd(fd)));
+    return Ok(std::make_pair(std::filesystem::path(pattern_copy), auto_fd(fd)));
 }
 
 Result<std::string, std::string>
-read_file(const ghc::filesystem::path& path)
+read_file(const std::filesystem::path& path)
 {
     try {
-        ghc::filesystem::ifstream file_stream(path);
+        std::ifstream file_stream(path);
 
         if (!file_stream) {
             return Err(std::string(strerror(errno)));
@@ -117,9 +151,12 @@ read_file(const ghc::filesystem::path& path)
     }
 }
 
-Result<void, std::string>
-write_file(const ghc::filesystem::path& path, const string_fragment& content)
+Result<write_file_result, std::string>
+write_file(const std::filesystem::path& path,
+           const string_fragment& content,
+           std::set<write_file_options> options)
 {
+    write_file_result retval;
     auto tmp_pattern = path;
     tmp_pattern += ".XXXXXX";
 
@@ -138,8 +175,26 @@ write_file(const ghc::filesystem::path& path, const string_fragment& content)
                                bytes_written,
                                content.length()));
     }
+
     std::error_code ec;
-    ghc::filesystem::rename(tmp_pair.first, path, ec);
+    if (options.count(write_file_options::backup_existing)) {
+        if (std::filesystem::exists(path, ec)) {
+            auto backup_path = path;
+
+            backup_path += ".bak";
+            std::filesystem::rename(path, backup_path, ec);
+            if (ec) {
+                return Err(
+                    fmt::format(FMT_STRING("unable to backup file {}: {}"),
+                                path.string(),
+                                ec.message()));
+            }
+
+            retval.wfr_backup_path = backup_path;
+        }
+    }
+
+    std::filesystem::rename(tmp_pair.first, path, ec);
     if (ec) {
         return Err(
             fmt::format(FMT_STRING("unable to move temporary file {}: {}"),
@@ -147,11 +202,11 @@ write_file(const ghc::filesystem::path& path, const string_fragment& content)
                         ec.message()));
     }
 
-    return Ok();
+    return Ok(retval);
 }
 
 std::string
-build_path(const std::vector<ghc::filesystem::path>& paths)
+build_path(const std::vector<std::filesystem::path>& paths)
 {
     return paths
         | lnav::itertools::map([](const auto& path) { return path.string(); })
@@ -168,7 +223,7 @@ build_path(const std::vector<ghc::filesystem::path>& paths)
 }
 
 Result<struct stat, std::string>
-stat_file(const ghc::filesystem::path& path)
+stat_file(const std::filesystem::path& path)
 {
     struct stat retval;
 
@@ -181,7 +236,8 @@ stat_file(const ghc::filesystem::path& path)
                            strerror(errno)));
 }
 
-file_lock::file_lock(const ghc::filesystem::path& archive_path)
+file_lock::
+file_lock(const std::filesystem::path& archive_path)
 {
     auto lock_path = archive_path;
 
@@ -196,3 +252,20 @@ file_lock::file_lock(const ghc::filesystem::path& archive_path)
 
 }  // namespace filesystem
 }  // namespace lnav
+
+namespace fmt {
+
+auto
+formatter<std::filesystem::path>::format(const std::filesystem::path& p,
+                                         format_context& ctx)
+    -> decltype(ctx.out()) const
+{
+    auto esc_res = fmt::v10::detail::find_escape(&(*p.native().begin()),
+                                                 &(*p.native().end()));
+    if (esc_res.end == nullptr) {
+        return formatter<string_view>::format(p.native(), ctx);
+    }
+
+    return format_to(ctx.out(), FMT_STRING("{:?}"), p.native());
+}
+}  // namespace fmt

src/base/fs_util.hh

@@ -30,11 +30,17 @@
 #ifndef lnav_fs_util_hh
 #define lnav_fs_util_hh
 
+#include <filesystem>
+#include <optional>
+#include <set>
 #include <string>
+#include <utility>
 #include <vector>
 
+#include <sys/stat.h>
+#include <unistd.h>
+
 #include "auto_fd.hh"
-#include "ghc/filesystem.hpp"
 #include "intern_string.hh"
 #include "result.h"
 
@@ -49,45 +55,57 @@ is_glob(const std::string& fn)
             || fn.find('[') != std::string::npos);
 }
 
+std::string escape_path(const std::filesystem::path& p);
+
 inline int
-statp(const ghc::filesystem::path& path, struct stat* buf)
+statp(const std::filesystem::path& path, struct stat* buf)
 {
     return stat(path.c_str(), buf);
 }
 
 inline int
-openp(const ghc::filesystem::path& path, int flags)
+openp(const std::filesystem::path& path, int flags)
 {
     return open(path.c_str(), flags);
 }
 
 inline int
-openp(const ghc::filesystem::path& path, int flags, mode_t mode)
+openp(const std::filesystem::path& path, int flags, mode_t mode)
 {
     return open(path.c_str(), flags, mode);
 }
 
-Result<ghc::filesystem::path, std::string> realpath(
-    const ghc::filesystem::path& path);
+Result<std::filesystem::path, std::string> realpath(
+    const std::filesystem::path& path);
 
-Result<auto_fd, std::string> create_file(const ghc::filesystem::path& path,
+Result<auto_fd, std::string> create_file(const std::filesystem::path& path,
                                          int flags,
                                          mode_t mode);
 
-Result<auto_fd, std::string> open_file(const ghc::filesystem::path& path,
+Result<auto_fd, std::string> open_file(const std::filesystem::path& path,
                                        int flags);
 
-Result<struct stat, std::string> stat_file(const ghc::filesystem::path& path);
+Result<struct stat, std::string> stat_file(const std::filesystem::path& path);
+
+Result<std::pair<std::filesystem::path, auto_fd>, std::string> open_temp_file(
+    const std::filesystem::path& pattern);
+
+Result<std::string, std::string> read_file(const std::filesystem::path& path);
 
-Result<std::pair<ghc::filesystem::path, auto_fd>, std::string> open_temp_file(
-    const ghc::filesystem::path& pattern);
+enum class write_file_options {
+    backup_existing,
+};
 
-Result<std::string, std::string> read_file(const ghc::filesystem::path& path);
+struct write_file_result {
+    std::optional<std::filesystem::path> wfr_backup_path;
+};
 
-Result<void, std::string> write_file(const ghc::filesystem::path& path,
-                                     const string_fragment& content);
+Result<write_file_result, std::string> write_file(
+    const std::filesystem::path& path,
+    const string_fragment& content,
+    std::set<write_file_options> options = {});
 
-std::string build_path(const std::vector<ghc::filesystem::path>& paths);
+std::string build_path(const std::vector<std::filesystem::path>& paths);
 
 class file_lock {
 public:
@@ -122,7 +140,7 @@ public:
 
     void unlock() const { lockf(this->lh_fd, F_ULOCK, 0); }
 
-    explicit file_lock(const ghc::filesystem::path& archive_path);
+    explicit file_lock(const std::filesystem::path& archive_path);
 
     auto_fd lh_fd;
 };
@@ -130,4 +148,12 @@ public:
 }  // namespace filesystem
 }  // namespace lnav
 
+namespace fmt {
+template<>
+struct formatter<std::filesystem::path> : formatter<string_view> {
+    auto format(const std::filesystem::path& p, format_context& ctx)
+        -> decltype(ctx.out()) const;
+};
+}  // namespace fmt
+
 #endif

src/base/fs_util.tests.cc

@@ -27,6 +27,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <filesystem>
 #include <iostream>
 
 #include "base/fs_util.hh"

src/base/future_util.hh

@@ -74,7 +74,7 @@ public:
     explicit future_queue(
         std::function<processor_result_t(std::future<T>&)> processor,
         size_t max_queue_size = 8)
-        : fq_processor(processor), fq_max_queue_size(max_queue_size)
+        : fq_processor(std::move(processor)), fq_max_queue_size(max_queue_size)
     {
     }
 

src/base/humanize.cc

@@ -77,7 +77,7 @@ file_size(file_ssize_t value, alignment align)
 }
 
 const std::string&
-sparkline(double value, nonstd::optional<double> upper_opt)
+sparkline(double value, std::optional<double> upper_opt)
 {
     static const std::string ZERO = " ";
     static const std::string BARS[] = {

src/base/humanize.hh

@@ -51,7 +51,7 @@ enum class alignment {
  */
 std::string file_size(file_ssize_t value, alignment align);
 
-const std::string& sparkline(double value, nonstd::optional<double> upper);
+const std::string& sparkline(double value, std::optional<double> upper);
 
 }  // namespace humanize
 

src/base/humanize.network.cc

@@ -30,13 +30,14 @@
 #include "humanize.network.hh"
 
 #include "config.h"
+#include "itertools.hh"
 #include "pcrepp/pcre2pp.hh"
 
 namespace humanize {
 namespace network {
 namespace path {
 
-nonstd::optional<::network::path>
+std::optional<::network::path>
 from_str(string_fragment sf)
 {
     static const auto REMOTE_PATTERN = lnav::pcre2pp::code::from_const(
@@ -52,11 +53,11 @@ from_str(string_fragment sf)
                          .ignore_error();
 
     if (!match_res) {
-        return nonstd::nullopt;
+        return std::nullopt;
     }
 
-    const auto username = REMOTE_MATCH_DATA["username"].map(
-        [](auto sf) { return sf.to_string(); });
+    const auto username = REMOTE_MATCH_DATA["username"]
+        | lnav::itertools::map([](auto sf) { return sf.to_string(); });
     const auto ipv6 = REMOTE_MATCH_DATA["ipv6"];
     const auto hostname = REMOTE_MATCH_DATA["hostname"];
     const auto locality_hostname = ipv6 ? ipv6.value() : hostname.value();
@@ -66,7 +67,7 @@ from_str(string_fragment sf)
         path = string_fragment::from_const(".");
     }
     return ::network::path{
-        {username, locality_hostname.to_string(), nonstd::nullopt},
+        {username, locality_hostname.to_string(), std::nullopt},
         path.to_string(),
     };
 }

src/base/humanize.network.hh

@@ -35,7 +35,6 @@
 #include "fmt/format.h"
 #include "intern_string.hh"
 #include "network.tcp.hh"
-#include "optional.hpp"
 
 namespace fmt {
 
@@ -100,7 +99,7 @@ namespace humanize {
 namespace network {
 namespace path {
 
-nonstd::optional<::network::path> from_str(string_fragment sf);
+std::optional<::network::path> from_str(string_fragment sf);
 
 }  // namespace path
 }  // namespace network

src/base/humanize.time.cc

@@ -57,8 +57,9 @@ point::as_time_ago() const
         current_time.tv_sec = convert_log_time_to_local(current_time.tv_sec);
     }
 
-    auto delta
-        = std::chrono::seconds(current_time.tv_sec - this->p_past_point.tv_sec);
+    auto curr_secs = std::chrono::seconds(current_time.tv_sec);
+    auto past_secs = std::chrono::seconds(this->p_past_point.tv_sec);
+    auto delta = curr_secs - past_secs;
     if (delta < 0s) {
         return "in the future";
     }
@@ -113,8 +114,8 @@ point::as_precise_time_ago() const
             return fmt::format(FMT_STRING("{:2} seconds ago"), diff.tv_sec);
         }
 
-        time_t seconds = diff.tv_sec % 60;
-        time_t minutes = diff.tv_sec / 60;
+        lnav::time64_t seconds = diff.tv_sec % 60;
+        lnav::time64_t minutes = diff.tv_sec / 60;
 
         return fmt::format(FMT_STRING("{:2} minute{} and {:2} second{} ago"),
                            minutes,

src/base/humanize.time.hh

@@ -30,12 +30,11 @@
 #ifndef lnav_humanize_time_hh
 #define lnav_humanize_time_hh
 
+#include <optional>
 #include <string>
 
 #include <sys/time.h>
 
-#include "optional.hpp"
-
 namespace humanize {
 namespace time {
 
@@ -66,7 +65,7 @@ private:
     }
 
     struct timeval p_past_point;
-    nonstd::optional<struct timeval> p_recent_point;
+    std::optional<struct timeval> p_recent_point;
     bool p_convert_to_local{false};
 };
 

src/base/injector.hh

@@ -32,6 +32,7 @@
 #ifndef lnav_injector_hh
 #define lnav_injector_hh
 
+#include <functional>
 #include <map>
 #include <memory>
 #include <type_traits>

src/base/intern_string.cc

@@ -36,6 +36,7 @@
 #include <string.h>
 
 #include "config.h"
+#include "fmt/ostream.h"
 #include "pcrepp/pcre2pp.hh"
 #include "ww898/cp_utf8.hpp"
 #include "xxHash/xxhash.h"
@@ -179,11 +180,35 @@ string_fragment::trim() const
     return this->trim(" \t\r\n");
 }
 
-nonstd::optional<string_fragment>
+string_fragment
+string_fragment::rtrim(const char* tokens) const
+{
+    string_fragment retval = *this;
+
+    while (retval.sf_begin < retval.sf_end) {
+        bool found = false;
+
+        for (int lpc = 0; tokens[lpc] != '\0'; lpc++) {
+            if (retval.sf_string[retval.sf_end - 1] == tokens[lpc]) {
+                found = true;
+                break;
+            }
+        }
+        if (!found) {
+            break;
+        }
+
+        retval.sf_end -= 1;
+    }
+
+    return retval;
+}
+
+std::optional<string_fragment>
 string_fragment::consume_n(int amount) const
 {
     if (amount > this->length()) {
-        return nonstd::nullopt;
+        return std::nullopt;
     }
 
     return string_fragment{
@@ -197,7 +222,7 @@ string_fragment::split_result
 string_fragment::split_n(int amount) const
 {
     if (amount > this->length()) {
-        return nonstd::nullopt;
+        return std::nullopt;
     }
 
     return std::make_pair(
@@ -358,12 +383,12 @@ uint32_t
 string_fragment::front_codepoint() const
 {
     size_t index = 0;
-    try {
-        return ww898::utf::utf8::read(
-            [this, &index]() { return this->data()[index++]; });
-    } catch (const std::runtime_error& e) {
+    auto read_res = ww898::utf::utf8::read(
+        [this, &index]() { return this->data()[index++]; });
+    if (read_res.isErr()) {
         return this->data()[0];
     }
+    return read_res.unwrap();
 }
 
 Result<ssize_t, const char*>
@@ -386,3 +411,92 @@ string_fragment::codepoint_to_byte_index(ssize_t cp_index) const
 
     return Ok(retval);
 }
+
+string_fragment
+string_fragment::sub_cell_range(int cell_start, int cell_end) const
+{
+    int byte_index = this->sf_begin;
+    std::optional<int> byte_start;
+    std::optional<int> byte_end;
+    int cell_index = 0;
+
+    while (byte_index < this->sf_end) {
+        if (cell_start == cell_index) {
+            byte_start = byte_index;
+        }
+        if (!byte_end && cell_index >= cell_end) {
+            byte_end = byte_index;
+            break;
+        }
+        auto read_res = ww898::utf::utf8::read(
+            [this, &byte_index]() { return this->sf_string[byte_index++]; });
+        if (read_res.isErr()) {
+            byte_index += 1;
+        } else {
+            auto ch = read_res.unwrap();
+
+            switch (ch) {
+                case '\t':
+                    do {
+                        cell_index += 1;
+                    } while (cell_index % 8);
+                    break;
+                default: {
+                    auto wcw_res = wcwidth(read_res.unwrap());
+                    if (wcw_res < 0) {
+                        wcw_res = 1;
+                    }
+                    cell_index += wcw_res;
+                    break;
+                }
+            }
+        }
+    }
+    if (cell_start == cell_index) {
+        byte_start = byte_index;
+    }
+    if (!byte_end) {
+        byte_end = byte_index;
+    }
+
+    if (byte_start && byte_end) {
+        return this->sub_range(byte_start.value(), byte_end.value());
+    }
+
+    return string_fragment{};
+}
+
+size_t
+string_fragment::column_width() const
+{
+    auto index = this->sf_begin;
+    size_t retval = 0;
+
+    while (index < this->sf_end) {
+        auto read_res = ww898::utf::utf8::read(
+            [this, &index]() { return this->sf_string[index++]; });
+        if (read_res.isErr()) {
+            retval += 1;
+        } else {
+            auto ch = read_res.unwrap();
+
+            switch (ch) {
+                case '\t':
+                    do {
+                        retval += 1;
+                    } while (retval % 8);
+                    break;
+                default: {
+                    auto wcw_res = wcwidth(read_res.unwrap());
+                    if (wcw_res < 0) {
+                        wcw_res = 1;
+                    }
+                    retval += wcw_res;
+                    break;
+                }
+            }
+        }
+    }
+
+    return retval;
+}

src/base/intern_string.hh

@@ -32,6 +32,7 @@
 #ifndef intern_string_hh
 #define intern_string_hh
 
+#include <optional>
 #include <ostream>
 #include <vector>
 
@@ -40,11 +41,12 @@
 #include <sys/types.h>
 
 #include "fmt/format.h"
-#include "optional.hpp"
 #include "result.h"
 #include "scn/util/string_view.h"
 #include "strnatcmp.h"
 
+unsigned long hash_str(const char* str, size_t len);
+
 struct string_fragment {
     using iterator = const char*;
 
@@ -142,6 +144,8 @@ struct string_fragment {
 
     Result<ssize_t, const char*> utf8_length() const;
 
+    size_t column_width() const;
+
     const char* data() const { return &this->sf_string[this->sf_begin]; }
 
     const unsigned char* udata() const
@@ -176,6 +180,8 @@ struct string_fragment {
     Result<ssize_t, const char*> codepoint_to_byte_index(
         ssize_t cp_index) const;
 
+    string_fragment sub_cell_range(int cell_start, int cell_end) const;
+
     const char& operator[](int index) const
     {
         return this->sf_string[sf_begin + index];
@@ -275,10 +281,22 @@ struct string_fragment {
 
     string_fragment sub_range(int begin, int end) const
     {
+        if (this->sf_begin + begin > this->sf_end) {
+            begin = this->sf_end - this->sf_begin;
+        }
+        if (this->sf_begin + end > this->sf_end) {
+            end = this->sf_end - this->sf_begin;
+        }
         return string_fragment{
             this->sf_string, this->sf_begin + begin, this->sf_begin + end};
     }
 
+    bool contains(const string_fragment& sf) const
+    {
+        return this->sf_string == sf.sf_string && this->sf_begin <= sf.sf_begin
+            && sf.sf_end <= this->sf_end;
+    }
+
     size_t count(char ch) const
     {
         size_t retval = 0;
@@ -292,7 +310,7 @@ struct string_fragment {
         return retval;
     }
 
-    nonstd::optional<size_t> find(char ch) const
+    std::optional<int> find(char ch) const
     {
         for (int lpc = this->sf_begin; lpc < this->sf_end; lpc++) {
             if (this->sf_string[lpc] == ch) {
@@ -300,7 +318,7 @@ struct string_fragment {
             }
         }
 
-        return nonstd::nullopt;
+        return std::nullopt;
     }
 
     template<typename P>
@@ -364,21 +382,21 @@ struct string_fragment {
             start - left.sf_begin, predicate, count);
     }
 
-    nonstd::optional<std::pair<uint32_t, string_fragment>> consume_codepoint()
+    std::optional<std::pair<uint32_t, string_fragment>> consume_codepoint()
         const
     {
         auto cp = this->front_codepoint();
         auto index_res = this->codepoint_to_byte_index(1);
 
         if (index_res.isErr()) {
-            return nonstd::nullopt;
+            return std::nullopt;
         }
 
         return std::make_pair(cp, this->substr(index_res.unwrap()));
     }
 
     template<typename P>
-    nonstd::optional<string_fragment> consume(P predicate) const
+    std::optional<string_fragment> consume(P predicate) const
     {
         int consumed = 0;
         while (consumed < this->length()) {
@@ -390,7 +408,7 @@ struct string_fragment {
         }
 
         if (consumed == 0) {
-            return nonstd::nullopt;
+            return std::nullopt;
         }
 
         return string_fragment{
@@ -400,7 +418,7 @@ struct string_fragment {
         };
     }
 
-    nonstd::optional<string_fragment> consume_n(int amount) const;
+    std::optional<string_fragment> consume_n(int amount) const;
 
     template<typename P>
     string_fragment skip(P predicate) const
@@ -418,7 +436,7 @@ struct string_fragment {
     }
 
     using split_result
-        = nonstd::optional<std::pair<string_fragment, string_fragment>>;
+        = std::optional<std::pair<string_fragment, string_fragment>>;
 
     template<typename P>
     split_result split_while(P&& predicate) const
@@ -433,7 +451,7 @@ struct string_fragment {
         }
 
         if (consumed == 0) {
-            return nonstd::nullopt;
+            return std::nullopt;
         }
 
         return std::make_pair(
@@ -490,7 +508,7 @@ struct string_fragment {
         }
 
         if (consumed == this->length()) {
-            return nonstd::nullopt;
+            return std::nullopt;
         }
 
         return std::make_pair(
@@ -563,6 +581,7 @@ struct string_fragment {
     }
 
     string_fragment trim(const char* tokens) const;
+    string_fragment rtrim(const char* tokens) const;
     string_fragment trim() const;
 
     string_fragment prepend(const char* str, int amount) const
@@ -627,6 +646,11 @@ struct string_fragment {
 
     std::string to_string_with_case_style(case_style style) const;
 
+    unsigned long hash() const
+    {
+        return hash_str(this->data(), this->length());
+    }
+
     const char* sf_string;
     int sf_begin;
     int sf_end;
@@ -795,8 +819,6 @@ private:
     const intern_string* ist_interned_string;
 };
 
-unsigned long hash_str(const char* str, size_t len);
-
 namespace fmt {
 template<>
 struct formatter<string_fragment> : formatter<string_view> {

src/base/intern_string.tests.cc

@@ -184,3 +184,17 @@ World!)");
         CHECK(all_sf2 == "Hello,\nWorld!");
     }
 }
+
+TEST_CASE("string_fragment::column_width")
+{
+    {
+        const auto sf = string_fragment::from_const("Key(s)\n");
+
+        CHECK(7 == sf.column_width());
+    }
+    {
+        const auto sf = string_fragment::from_const("\u26a0");
+
+        CHECK(1 == sf.column_width());
+    }
+}

src/base/is_utf8.cc

@@ -60,7 +60,7 @@
   error.
 */
 utf8_scan_result
-is_utf8(string_fragment str, nonstd::optional<unsigned char> terminator)
+is_utf8(string_fragment str, std::optional<unsigned char> terminator)
 {
     const auto* ustr = str.udata();
     utf8_scan_result retval;
@@ -76,6 +76,7 @@ is_utf8(string_fragment str, nonstd::optional<unsigned char> terminator)
             break;
         }
 
+        retval.usr_column_width_guess += 1;
         if (retval.usr_message != nullptr) {
             i += 1;
             continue;
@@ -83,6 +84,9 @@ is_utf8(string_fragment str, nonstd::optional<unsigned char> terminator)
 
         valid_end = i;
         if (ustr[i] <= 0x7F) /* 00..7F */ {
+            if (ustr[i] == '\t') {
+                retval.usr_column_width_guess += 7;
+            }
             i += 1;
         } else if (ustr[i] >= 0xC2 && ustr[i] <= 0xDF) /* C2..DF 80..BF */ {
             if (i + 1 < str.length()) /* Expect a 2nd byte */ {

src/base/is_utf8.hh

@@ -32,14 +32,14 @@
 #include <sys/types.h>
 
 #include "intern_string.hh"
-#include "optional.hpp"
 
 struct utf8_scan_result {
     const char* usr_message{nullptr};
     size_t usr_faulty_bytes{0};
     string_fragment usr_valid_frag{string_fragment::invalid()};
-    nonstd::optional<string_fragment> usr_remaining;
+    std::optional<string_fragment> usr_remaining;
     bool usr_has_ansi{false};
+    size_t usr_column_width_guess{0};
 
     const char* remaining_ptr(const string_fragment& frag) const
     {
@@ -53,7 +53,7 @@ struct utf8_scan_result {
 };
 
 utf8_scan_result is_utf8(string_fragment frag,
-                         nonstd::optional<unsigned char> terminator
-                         = nonstd::nullopt);
+                         std::optional<unsigned char> terminator
+                         = std::nullopt);
 
 #endif /* _IS_UTF8_H */

src/base/isc.cc

@@ -53,7 +53,20 @@ service_base::run()
         mstime_t current_time = getmstime();
         auto timeout = this->compute_timeout(current_time);
 
-        this->s_port.process_for(timeout);
+        try {
+            this->s_port.process_for(timeout);
+        } catch (const std::exception& e) {
+            log_error("%s: message failed with -- %s",
+                      this->s_name.c_str(),
+                      e.what());
+            this->s_looping = false;
+            continue;
+        } catch (...) {
+            log_error("%s: message failed with non-standard exception",
+                      this->s_name.c_str());
+            this->s_looping = false;
+            continue;
+        }
         this->s_children.cleanup_children();
 
         try {

src/base/isc.hh

@@ -166,7 +166,7 @@ protected:
 template<typename T>
 class service : public service_base {
 public:
-    explicit service(std::string sub_name = "")
+    explicit service(const std::string& sub_name = "")
         : service_base(std::string(__PRETTY_FUNCTION__) + " " + sub_name)
     {
     }
@@ -174,9 +174,11 @@ public:
     template<typename F>
     void send(F msg)
     {
-        this->s_port.send({[lifetime = this->shared_from_this(), this, msg]() {
-            msg(*(static_cast<T*>(this)));
-        }});
+        this->s_port.send({
+            [lifetime = this->shared_from_this(),
+             this,
+             msg2 = std::move(msg)]() { msg2(*(static_cast<T*>(this))); },
+        });
     }
 
     template<typename F, class Rep, class Period>
@@ -185,12 +187,16 @@ public:
     {
         msg_port reply_port;
 
-        this->s_port.send(
-            {[lifetime = this->shared_from_this(), this, &reply_port, msg]() {
-                msg(*(static_cast<T*>(this)));
+        this->s_port.send({
+            [lifetime = this->shared_from_this(),
+             this,
+             &reply_port,
+             msg2 = std::move(msg)]() {
+                msg2(*(static_cast<T*>(this)));
                 reply_port.send(empty_msg());
-            }});
-        reply_port.template process_for(rel_time);
+            },
+        });
+        reply_port.process_for(rel_time);
     }
 };
 
@@ -200,7 +206,7 @@ struct to {
     {
         auto& service = injector::get<T&, Service>();
 
-        service.send(cb);
+        service.send(std::move(cb));
     }
 
     template<class Rep, class Period>
@@ -209,14 +215,14 @@ struct to {
     {
         auto& service = injector::get<T&, Service>();
 
-        service.send_and_wait(cb, rel_time);
+        service.send_and_wait(std::move(cb), rel_time);
     }
 
     void send_and_wait(std::function<void(T)> cb)
     {
         using namespace std::literals::chrono_literals;
 
-        this->send_and_wait(cb, 48h);
+        this->send_and_wait(std::move(cb), 48h);
     }
 };
 

src/base/itertools.enumerate.hh

@@ -0,0 +1,88 @@
+/**
+ * Copyright (c) 2024, Timothy Stack
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * * Neither the name of Timothy Stack nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef lnav_itertools_enumerate_hh
+#define lnav_itertools_enumerate_hh
+
+#include <tuple>
+
+namespace lnav::itertools {
+
+namespace details {
+
+template<typename Iterator, typename CounterType = size_t>
+class EnumerateIterator {
+    Iterator iter;
+    CounterType index;
+
+public:
+    EnumerateIterator(Iterator iter, CounterType index)
+        : iter(std::move(iter)), index(index)
+    {
+    }
+
+    EnumerateIterator& operator++()
+    {
+        ++iter;
+        ++index;
+        return *this;
+    }
+
+    auto operator*() { return std::forward_as_tuple(index, *iter); }
+    bool operator!=(EnumerateIterator const& rhs) const
+    {
+        return iter != rhs.iter;
+    }
+};
+
+}  // namespace details
+
+template<typename Iterable, typename CounterType = size_t>
+class enumerate {
+public:
+    enumerate(Iterable& iterable, CounterType start = 0)
+        : iterable(iterable), index(start)
+    {
+    }
+
+    auto begin()
+    {
+        return details::EnumerateIterator{std::begin(iterable), index};
+    }
+
+    auto end() { return details::EnumerateIterator{std::end(iterable), index}; }
+
+private:
+    Iterable& iterable;
+    CounterType index;
+};
+
+}  // namespace lnav::itertools
+
+#endif

src/base/itertools.hh

@@ -34,12 +34,12 @@
 #include <deque>
 #include <map>
 #include <memory>
+#include <optional>
 #include <set>
 #include <type_traits>
 #include <vector>
 
 #include "func_util.hh"
-#include "optional.hpp"
 
 namespace lnav {
 namespace itertools {
@@ -122,7 +122,7 @@ struct append {
 };
 
 struct nth {
-    nonstd::optional<size_t> a_index;
+    std::optional<size_t> a_index;
 };
 
 struct skip {
@@ -190,7 +190,7 @@ second()
 }
 
 inline details::nth
-nth(nonstd::optional<size_t> index)
+nth(std::optional<size_t> index)
 {
     return details::nth{
         index,
@@ -340,7 +340,7 @@ sum()
 }  // namespace lnav
 
 template<typename C, typename P>
-nonstd::optional<std::conditional_t<
+std::optional<std::conditional_t<
     std::is_const<typename std::remove_reference_t<C>>::value,
     typename std::remove_reference_t<C>::const_iterator,
     typename std::remove_reference_t<C>::iterator>>
@@ -348,44 +348,44 @@ operator|(C&& in, const lnav::itertools::details::find_if<P>& finder)
 {
     for (auto iter = in.begin(); iter != in.end(); ++iter) {
         if (lnav::func::invoke(finder.fi_predicate, *iter)) {
-            return nonstd::make_optional(iter);
+            return std::make_optional(iter);
         }
     }
 
-    return nonstd::nullopt;
+    return std::nullopt;
 }
 
 template<typename C, typename T>
-nonstd::optional<size_t>
+std::optional<size_t>
 operator|(const C& in, const lnav::itertools::details::find<T>& finder)
 {
     size_t retval = 0;
     for (const auto& elem : in) {
         if (elem == finder.f_value) {
-            return nonstd::make_optional(retval);
+            return std::make_optional(retval);
         }
         retval += 1;
     }
 
-    return nonstd::nullopt;
+    return std::nullopt;
 }
 
 template<typename C>
-nonstd::optional<typename C::const_iterator>
+std::optional<typename C::const_iterator>
 operator|(const C& in, const lnav::itertools::details::nth indexer)
 {
     if (!indexer.a_index.has_value()) {
-        return nonstd::nullopt;
+        return std::nullopt;
     }
 
     if (indexer.a_index.value() < in.size()) {
         auto iter = in.begin();
 
         std::advance(iter, indexer.a_index.value());
-        return nonstd::make_optional(iter);
+        return std::make_optional(iter);
     }
 
-    return nonstd::nullopt;
+    return std::nullopt;
 }
 
 template<typename C>
@@ -402,10 +402,10 @@ operator|(const C& in, const lnav::itertools::details::first indexer)
 }
 
 template<typename C>
-nonstd::optional<typename C::value_type>
+std::optional<typename C::value_type>
 operator|(const C& in, const lnav::itertools::details::max_value maxer)
 {
-    nonstd::optional<typename C::value_type> retval;
+    std::optional<typename C::value_type> retval;
 
     for (const auto& elem : in) {
         if (!retval) {
@@ -572,13 +572,13 @@ template<typename T,
          typename F,
          std::enable_if_t<lnav::func::is_invocable<F, T>::value, int> = 0>
 auto
-operator|(nonstd::optional<T> in,
+operator|(std::optional<T> in,
           const lnav::itertools::details::flat_mapper<F>& mapper) ->
     typename std::remove_const_t<typename std::remove_reference_t<
         decltype(lnav::func::invoke(mapper.fm_func, in.value()))>>
 {
     if (!in) {
-        return nonstd::nullopt;
+        return std::nullopt;
     }
 
     return lnav::func::invoke(mapper.fm_func, in.value());
@@ -588,7 +588,7 @@ template<typename T,
          typename F,
          std::enable_if_t<lnav::func::is_invocable<F, T>::value, int> = 0>
 void
-operator|(nonstd::optional<T> in,
+operator|(std::optional<T> in,
           const lnav::itertools::details::for_eacher<F>& eacher)
 {
     if (!in) {
@@ -602,7 +602,7 @@ template<typename T,
          typename F,
          std::enable_if_t<lnav::func::is_invocable<F, T>::value, int> = 0>
 void
-operator|(std::vector<std::shared_ptr<T>>& in,
+operator|(const std::vector<std::shared_ptr<T>>& in,
           const lnav::itertools::details::for_eacher<F>& eacher)
 {
     for (auto& elem : in) {
@@ -610,21 +610,33 @@ operator|(std::vector<std::shared_ptr<T>>& in,
     }
 }
 
+template<typename T,
+         typename F,
+         std::enable_if_t<lnav::func::is_invocable<F, T>::value, int> = 0>
+void
+operator|(const std::vector<T>& in,
+          const lnav::itertools::details::for_eacher<F>& eacher)
+{
+    for (auto& elem : in) {
+        lnav::func::invoke(eacher.fe_func, elem);
+    }
+}
+
 template<typename T,
          typename F,
          std::enable_if_t<lnav::func::is_invocable<F, T>::value, int> = 0>
 auto
-operator|(nonstd::optional<T> in,
+operator|(std::optional<T> in,
           const lnav::itertools::details::mapper<F>& mapper)
-    -> nonstd::optional<
+    -> std::optional<
         typename std::remove_const_t<typename std::remove_reference_t<
             decltype(lnav::func::invoke(mapper.m_func, in.value()))>>>
 {
     if (!in) {
-        return nonstd::nullopt;
+        return std::nullopt;
     }
 
-    return nonstd::make_optional(lnav::func::invoke(mapper.m_func, in.value()));
+    return std::make_optional(lnav::func::invoke(mapper.m_func, in.value()));
 }
 
 template<typename T, typename F>
@@ -813,38 +825,38 @@ template<typename T,
          typename F,
          std::enable_if_t<!lnav::func::is_invocable<F, T>::value, int> = 0>
 auto
-operator|(nonstd::optional<T> in,
+operator|(std::optional<T> in,
           const lnav::itertools::details::mapper<F>& mapper)
-    -> nonstd::optional<typename std::remove_reference_t<
+    -> std::optional<typename std::remove_reference_t<
         typename std::remove_const_t<decltype(((in.value()).*mapper.m_func))>>>
 {
     if (!in) {
-        return nonstd::nullopt;
+        return std::nullopt;
     }
 
-    return nonstd::make_optional((in.value()).*mapper.m_func);
+    return std::make_optional((in.value()).*mapper.m_func);
 }
 
 template<typename T,
          typename F,
          std::enable_if_t<!lnav::func::is_invocable<F, T>::value, int> = 0>
 auto
-operator|(nonstd::optional<T> in,
+operator|(std::optional<T> in,
           const lnav::itertools::details::mapper<F>& mapper)
-    -> nonstd::optional<
+    -> std::optional<
         typename std::remove_const_t<typename std::remove_reference_t<
             decltype(((*in.value()).*mapper.m_func))>>>
 {
     if (!in) {
-        return nonstd::nullopt;
+        return std::nullopt;
     }
 
-    return nonstd::make_optional((*in.value()).*mapper.m_func);
+    return std::make_optional((*in.value()).*mapper.m_func);
 }
 
 template<typename T>
 T
-operator|(nonstd::optional<T> in,
+operator|(std::optional<T> in,
           const lnav::itertools::details::unwrap_or<T>& unwrapper)
 {
     return in.value_or(unwrapper.uo_value);

src/base/lnav.console.cc

@@ -51,13 +51,13 @@ snippet::from_content_with_offset(intern_string_t src,
                                   size_t offset,
                                   const std::string& errmsg)
 {
-    auto content_sf = string_fragment::from_str(content.get_string());
-    auto line_with_error = content_sf.find_boundaries_around(
+    const auto content_sf = string_fragment::from_str(content.get_string());
+    const auto line_with_error = content_sf.find_boundaries_around(
         offset, string_fragment::tag1{'\n'});
-    auto line_with_context = content_sf.find_boundaries_around(
+    const auto line_with_context = content_sf.find_boundaries_around(
         offset, string_fragment::tag1{'\n'}, 3);
-    auto line_number = content_sf.sub_range(0, offset).count('\n');
-    auto erroff_in_line = offset - line_with_error.sf_begin;
+    const auto line_number = content_sf.sub_range(0, offset).count('\n');
+    const auto erroff_in_line = offset - line_with_error.sf_begin;
 
     attr_line_t pointer;
 
@@ -252,7 +252,7 @@ user_message::to_attr_line(std::set<render_flags> flags) const
     return retval;
 }
 
-static nonstd::optional<fmt::terminal_color>
+static std::optional<fmt::terminal_color>
 curses_color_to_terminal_color(int curses_color)
 {
     switch (curses_color) {
@@ -273,7 +273,7 @@ curses_color_to_terminal_color(int curses_color)
         case COLOR_RED:
             return fmt::terminal_color::red;
         default:
-            return nonstd::nullopt;
+            return std::nullopt;
     }
 }
 
@@ -326,7 +326,7 @@ println(FILE* file, const attr_line_t& al)
         }
     }
 
-    nonstd::optional<size_t> last_point;
+    std::optional<size_t> last_point;
     for (const auto& point : points) {
         if (!last_point) {
             last_point = point;
@@ -337,7 +337,7 @@ println(FILE* file, const attr_line_t& al)
         auto line_style = fmt::text_style{};
         auto fg_style = fmt::text_style{};
         auto start = last_point.value();
-        nonstd::optional<std::string> href;
+        std::optional<std::string> href;
 
         for (const auto& attr : al.get_attrs()) {
             if (!attr.sa_range.contains(start)
@@ -409,7 +409,9 @@ println(FILE* file, const attr_line_t& al)
                         default:
                             break;
                     }
-                } else if (attr.sa_type == &VC_ROLE) {
+                } else if (attr.sa_type == &VC_ROLE
+                           || attr.sa_type == &VC_ROLE_FG)
+                {
                     auto saw = string_attr_wrapper<role_t>(&attr);
                     auto role = saw.get();
 
@@ -441,6 +443,9 @@ println(FILE* file, const attr_line_t& al)
                             line_style |= fmt::emphasis::bold
                                 | fmt::fg(fmt::terminal_color::green);
                             break;
+                        case role_t::VCR_FOOTNOTE_BORDER:
+                            line_style |= fmt::fg(fmt::terminal_color::blue);
+                            break;
                         case role_t::VCR_INFO:
                         case role_t::VCR_STATUS:
                             line_style |= fmt::emphasis::bold
@@ -514,15 +519,66 @@ println(FILE* file, const attr_line_t& al)
             line_style |= default_bg_style;
         }
 
+        if (line_style.has_foreground() && line_style.has_background()
+            && !line_style.get_foreground().is_rgb
+            && !line_style.get_background().is_rgb
+            && line_style.get_foreground().value.term_color
+                == line_style.get_background().value.term_color)
+        {
+            auto new_style = fmt::text_style{};
+
+            if (line_style.has_emphasis()) {
+                new_style |= line_style.get_emphasis();
+            }
+            new_style |= fmt::fg(line_style.get_foreground());
+            if (line_style.get_background().value.term_color
+                == lnav::enums::to_underlying(fmt::terminal_color::black))
+            {
+                new_style |= fmt::bg(fmt::terminal_color::white);
+            } else {
+                new_style |= fmt::bg(fmt::terminal_color::black);
+            }
+            line_style = new_style;
+        }
+
         if (href) {
             fmt::print(file, FMT_STRING("\x1b]8;;{}\x1b\\"), href.value());
         }
         if (start < str.size()) {
             auto actual_end = std::min(str.size(), static_cast<size_t>(point));
-            fmt::print(file,
-                       line_style,
-                       FMT_STRING("{}"),
-                       str.substr(start, actual_end - start));
+            auto sub = std::string{};
+
+            for (auto lpc = start; lpc < actual_end;) {
+                auto cp_start = lpc;
+                auto read_res = ww898::utf::utf8::read(
+                    [&str, &lpc]() { return str[lpc++]; });
+
+                if (read_res.isErr()) {
+                    sub.append(fmt::format(
+                        FMT_STRING("{:?}"),
+                        fmt::string_view{&str[cp_start], lpc - cp_start}));
+                    continue;
+                }
+
+                auto ch = read_res.unwrap();
+                switch (ch) {
+                    case '\b':
+                        sub.append("\u232b");
+                        break;
+                    case '\x1b':
+                        sub.append("\u238b");
+                        break;
+                    case '\x07':
+                        sub.append("\U0001F514");
+                        break;
+
+                    default:
+                        sub.append(&str[cp_start], lpc - cp_start);
+                        break;
+                }
+            }
+
+            fmt::print(file, line_style, FMT_STRING("{}"), sub);
         }
         if (href) {
             fmt::print(file, FMT_STRING("\x1b]8;;\x1b\\"));

src/base/lnav.console.hh

@@ -94,6 +94,13 @@ struct user_message {
 
     static user_message ok(const attr_line_t& al);
 
+    user_message() = default;
+    user_message(user_message&&) = default;
+    user_message(const user_message&) = default;
+
+    user_message& operator=(user_message&&) = default;
+    user_message& operator=(const user_message&) = default;
+
     user_message& with_reason(const attr_line_t& al)
     {
         this->um_reason = al;
@@ -186,6 +193,9 @@ struct user_message {
     attr_line_t to_attr_line(std::set<render_flags> flags
                              = {render_flags::prefix}) const;
 
+    user_message move() & { return std::move(*this); }
+    user_message move() && { return std::move(*this); }
+
     level um_level{level::ok};
     attr_line_t um_message;
     std::vector<snippet> um_snippets;