[demux] add lnav_file_demux_metadata view

[demux] add a demuxer for container with type
14 changed files with 63 additions and 45 deletions
--- a/docs/source/config.rst
+++ b/docs/source/config.rst
@ -334,6 +334,7 @@ has the following known capture names:

 If there are additional captures, they will be included
 in the file metadata that can be accessed by the
+:code:`lnav_file_demux_metadata` view of the
 :code:`lnav_file_metadata` table.

 Reference
--- a/src/init.sql
+++ b/src/init.sql
@ -93,6 +93,11 @@ SELECT *
 FROM lnav_views
 WHERE name = (SELECT name FROM lnav_view_stack ORDER BY rowid DESC LIMIT 1);

+CREATE VIEW lnav_file_demux_metadata AS
+SELECT filepath, jget(content, '/demux_meta') AS metadata
+FROM lnav_file_metadata
+WHERE descriptor = 'org.lnav.piper.header';
+
 INSERT INTO lnav_example_log
 VALUES (0, NULL, '2017-02-03T04:05:06.100', '2017-02-03T04:05:06.100', 0,
        'info', 0, NULL, NULL, NULL, 'hw', 2, 1486094706000, '/tmp/log',
--- a/src/root-config.json
+++ b/src/root-config.json
@ -30,6 +30,9 @@
            },
            "recv-with-pod": {
                "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[+\\-]\\d{2}:\\d{2})) source=[a-zA-Z0-9][a-zA-Z0-9_\\.\\-]* (?<body>.*) kubernetes_host=(?<k8s_host>[a-zA-Z0-9][a-zA-Z0-9_\\.\\-]*) kubernetes_pod_name=(?<mux_id>[a-zA-Z0-9][a-zA-Z0-9_\\.\\-]*)"
+            },
+            "container-with-type": {
+                "pattern": "^(?<mux_id>[a-zA-Z][\\w\\-]{3,}) (?<container_type>[a-zA-Z][\\w\\-]{3,}) (?<body>.*)"
            }
        }
    },
--- a/test/Makefile.am
+++ b/test/Makefile.am
@ -348,6 +348,7 @@ dist_noinst_DATA = \
 	logfile_invalid_json.json \
 	logfile_invalid_json2.json \
 	logfile_mixed_json2.json \
+	logfile_mux_zookeeper.0 \
 	logfile_journald.json \
 	logfile_json.json \
 	logfile_json2.json \
--- a/test/expected/expected.am
+++ b/test/expected/expected.am
@ -2,28 +2,20 @@
 EXPECTED_FILES = \
    $(srcdir)/%reldir%/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.err \
    $(srcdir)/%reldir%/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out \
-    $(srcdir)/%reldir%/test_cli.sh_108c6922cde063429f76abc3fdb8a81ad9a2f671.err \
-    $(srcdir)/%reldir%/test_cli.sh_108c6922cde063429f76abc3fdb8a81ad9a2f671.out \
    $(srcdir)/%reldir%/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.err \
    $(srcdir)/%reldir%/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.out \
    $(srcdir)/%reldir%/test_cli.sh_17a68b798354f9a6cdfab372006caeb74038d15c.err \
    $(srcdir)/%reldir%/test_cli.sh_17a68b798354f9a6cdfab372006caeb74038d15c.out \
    $(srcdir)/%reldir%/test_cli.sh_3114508cf42fb2608ef77f4bc294a84885c97a79.err \
    $(srcdir)/%reldir%/test_cli.sh_3114508cf42fb2608ef77f4bc294a84885c97a79.out \
-    $(srcdir)/%reldir%/test_cli.sh_4327033cfae0d4c170a38a3c4a570520bfabb493.err \
-    $(srcdir)/%reldir%/test_cli.sh_4327033cfae0d4c170a38a3c4a570520bfabb493.out \
    $(srcdir)/%reldir%/test_cli.sh_4761827b538a623c60d2e7d243bae9b653bb1975.err \
    $(srcdir)/%reldir%/test_cli.sh_4761827b538a623c60d2e7d243bae9b653bb1975.out \
    $(srcdir)/%reldir%/test_cli.sh_5524542b1a6954ff9741155101497270a2f0c557.err \
    $(srcdir)/%reldir%/test_cli.sh_5524542b1a6954ff9741155101497270a2f0c557.out \
-    $(srcdir)/%reldir%/test_cli.sh_76aa57821598962e59063a40c20171040c95a731.err \
-    $(srcdir)/%reldir%/test_cli.sh_76aa57821598962e59063a40c20171040c95a731.out \
    $(srcdir)/%reldir%/test_cli.sh_97e19b9ff3775d84074455a2e8993a0611b1c269.err \
    $(srcdir)/%reldir%/test_cli.sh_97e19b9ff3775d84074455a2e8993a0611b1c269.out \
    $(srcdir)/%reldir%/test_cli.sh_a2f9a3e798e5f4f09c92eaa25ae236a2fb09dbd9.err \
    $(srcdir)/%reldir%/test_cli.sh_a2f9a3e798e5f4f09c92eaa25ae236a2fb09dbd9.out \
-    $(srcdir)/%reldir%/test_cli.sh_af3ace7762b4cc150fcdcac86083b379bded7b32.err \
-    $(srcdir)/%reldir%/test_cli.sh_af3ace7762b4cc150fcdcac86083b379bded7b32.out \
    $(srcdir)/%reldir%/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.err \
    $(srcdir)/%reldir%/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.out \
    $(srcdir)/%reldir%/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.err \
@ -274,6 +266,10 @@ EXPECTED_FILES = \
    $(srcdir)/%reldir%/test_config.sh_eec3768ebc201ca63bca1411270965f78db1abfc.out \
    $(srcdir)/%reldir%/test_demux.sh_090b34f84229c02aad7ab2f1cea20cfcf94446d5.err \
    $(srcdir)/%reldir%/test_demux.sh_090b34f84229c02aad7ab2f1cea20cfcf94446d5.out \
+    $(srcdir)/%reldir%/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.err \
+    $(srcdir)/%reldir%/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.out \
+    $(srcdir)/%reldir%/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.err \
+    $(srcdir)/%reldir%/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.out \
    $(srcdir)/%reldir%/test_demux.sh_babab3536a2fc7c2a99847fa355b50d40a51763d.err \
    $(srcdir)/%reldir%/test_demux.sh_babab3536a2fc7c2a99847fa355b50d40a51763d.out \
    $(srcdir)/%reldir%/test_demux.sh_f8cbb968fccbc0442a831c0f69c6dbdfe5413339.err \
@ -694,8 +690,6 @@ EXPECTED_FILES = \
    $(srcdir)/%reldir%/test_sql.sh_9a209f3ee1b1f543ca2587b695d2eb0e63e74c51.out \
    $(srcdir)/%reldir%/test_sql.sh_9aaaa810f883f5dcf8a4d5cda4051fdf4640a65c.err \
    $(srcdir)/%reldir%/test_sql.sh_9aaaa810f883f5dcf8a4d5cda4051fdf4640a65c.out \
-    $(srcdir)/%reldir%/test_sql.sh_9b03e9f7a1bc35e408b3a17ee90cfdadea164df6.err \
-    $(srcdir)/%reldir%/test_sql.sh_9b03e9f7a1bc35e408b3a17ee90cfdadea164df6.out \
    $(srcdir)/%reldir%/test_sql.sh_9ceccab07fbf7130bffe3c201c710719e4a3e9af.err \
    $(srcdir)/%reldir%/test_sql.sh_9ceccab07fbf7130bffe3c201c710719e4a3e9af.out \
    $(srcdir)/%reldir%/test_sql.sh_9e1d05b821822ee40e13fadb24ec558f4bfcff10.err \
--- a/test/expected/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out
+++ b/test/expected/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out
@ -5246,6 +5246,9 @@
            "container": {
                "pattern": "^(?:\\x1b\\[\\d*K)?(?<mux_id>[a-zA-Z0-9][a-zA-Z0-9_\\.\\-]*)\\s+\\| (?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{9}Z )?(?<body>.*)"
            },
+            "container-with-type": {
+                "pattern": "^(?<mux_id>[a-zA-Z][\\w\\-]{3,}) (?<container_type>[a-zA-Z][\\w\\-]{3,}) (?<body>.*)"
+            },
            "recv-with-pod": {
                "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[+\\-]\\d{2}:\\d{2})) source=[a-zA-Z0-9][a-zA-Z0-9_\\.\\-]* (?<body>.*) kubernetes_host=(?<k8s_host>[a-zA-Z0-9][a-zA-Z0-9_\\.\\-]*) kubernetes_pod_name=(?<mux_id>[a-zA-Z0-9][a-zA-Z0-9_\\.\\-]*)"
            }
--- a/test/expected/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.out
+++ b/test/expected/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.out
@ -19,43 +19,44 @@
 /log/annotations/org.lnav.test/description -> {test_dir}/configs/installed/anno-test.json:6
 /log/annotations/org.lnav.test/handler -> {test_dir}/configs/installed/anno-test.json:8
 /log/date-time/convert-zoned-to-local -> root-config.json:18
+/log/demux/container-with-type/pattern -> root-config.json:35
 /log/demux/container/pattern -> root-config.json:29
 /log/demux/recv-with-pod/pattern -> root-config.json:32
-/tuning/archive-manager/cache-ttl -> root-config.json:39
-/tuning/archive-manager/min-free-space -> root-config.json:38
-/tuning/clipboard/impls/MacOS/find/read -> root-config.json:67
-/tuning/clipboard/impls/MacOS/find/write -> root-config.json:66
-/tuning/clipboard/impls/MacOS/general/read -> root-config.json:63
-/tuning/clipboard/impls/MacOS/general/write -> root-config.json:62
-/tuning/clipboard/impls/MacOS/test -> root-config.json:60
-/tuning/clipboard/impls/NeoVim/general/read -> root-config.json:95
-/tuning/clipboard/impls/NeoVim/general/write -> root-config.json:94
-/tuning/clipboard/impls/NeoVim/test -> root-config.json:92
-/tuning/clipboard/impls/Wayland/general/read -> root-config.json:74
-/tuning/clipboard/impls/Wayland/general/write -> root-config.json:73
-/tuning/clipboard/impls/Wayland/test -> root-config.json:71
-/tuning/clipboard/impls/Windows/general/write -> root-config.json:101
-/tuning/clipboard/impls/Windows/test -> root-config.json:99
-/tuning/clipboard/impls/X11-xclip/general/read -> root-config.json:81
-/tuning/clipboard/impls/X11-xclip/general/write -> root-config.json:80
-/tuning/clipboard/impls/X11-xclip/test -> root-config.json:78
-/tuning/clipboard/impls/tmux/general/read -> root-config.json:88
-/tuning/clipboard/impls/tmux/general/write -> root-config.json:87
-/tuning/clipboard/impls/tmux/test -> root-config.json:85
-/tuning/piper/max-size -> root-config.json:53
-/tuning/piper/rotations -> root-config.json:54
-/tuning/piper/ttl -> root-config.json:55
-/tuning/remote/ssh/command -> root-config.json:43
-/tuning/remote/ssh/config/BatchMode -> root-config.json:45
-/tuning/remote/ssh/config/ConnectTimeout -> root-config.json:46
-/tuning/remote/ssh/start-command -> root-config.json:48
-/tuning/remote/ssh/transfer-command -> root-config.json:49
-/tuning/url-scheme/docker-compose/handler -> root-config.json:111
-/tuning/url-scheme/docker/handler -> root-config.json:108
+/tuning/archive-manager/cache-ttl -> root-config.json:42
+/tuning/archive-manager/min-free-space -> root-config.json:41
+/tuning/clipboard/impls/MacOS/find/read -> root-config.json:70
+/tuning/clipboard/impls/MacOS/find/write -> root-config.json:69
+/tuning/clipboard/impls/MacOS/general/read -> root-config.json:66
+/tuning/clipboard/impls/MacOS/general/write -> root-config.json:65
+/tuning/clipboard/impls/MacOS/test -> root-config.json:63
+/tuning/clipboard/impls/NeoVim/general/read -> root-config.json:98
+/tuning/clipboard/impls/NeoVim/general/write -> root-config.json:97
+/tuning/clipboard/impls/NeoVim/test -> root-config.json:95
+/tuning/clipboard/impls/Wayland/general/read -> root-config.json:77
+/tuning/clipboard/impls/Wayland/general/write -> root-config.json:76
+/tuning/clipboard/impls/Wayland/test -> root-config.json:74
+/tuning/clipboard/impls/Windows/general/write -> root-config.json:104
+/tuning/clipboard/impls/Windows/test -> root-config.json:102
+/tuning/clipboard/impls/X11-xclip/general/read -> root-config.json:84
+/tuning/clipboard/impls/X11-xclip/general/write -> root-config.json:83
+/tuning/clipboard/impls/X11-xclip/test -> root-config.json:81
+/tuning/clipboard/impls/tmux/general/read -> root-config.json:91
+/tuning/clipboard/impls/tmux/general/write -> root-config.json:90
+/tuning/clipboard/impls/tmux/test -> root-config.json:88
+/tuning/piper/max-size -> root-config.json:56
+/tuning/piper/rotations -> root-config.json:57
+/tuning/piper/ttl -> root-config.json:58
+/tuning/remote/ssh/command -> root-config.json:46
+/tuning/remote/ssh/config/BatchMode -> root-config.json:48
+/tuning/remote/ssh/config/ConnectTimeout -> root-config.json:49
+/tuning/remote/ssh/start-command -> root-config.json:51
+/tuning/remote/ssh/transfer-command -> root-config.json:52
+/tuning/url-scheme/docker-compose/handler -> root-config.json:114
+/tuning/url-scheme/docker/handler -> root-config.json:111
 /tuning/url-scheme/hw/handler -> {test_dir}/configs/installed/hw-url-handler.json:6
-/tuning/url-scheme/journald/handler -> root-config.json:114
-/tuning/url-scheme/piper/handler -> root-config.json:117
-/tuning/url-scheme/podman/handler -> root-config.json:120
+/tuning/url-scheme/journald/handler -> root-config.json:117
+/tuning/url-scheme/piper/handler -> root-config.json:120
+/tuning/url-scheme/podman/handler -> root-config.json:123
 /ui/clock-format -> root-config.json:4
 /ui/default-colors -> root-config.json:6
 /ui/dim-text -> root-config.json:5
--- a/test/expected/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.err
+++ b/test/expected/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.err
--- a/test/expected/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.out
+++ b/test/expected/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.out
@ -0,0 +1,2 @@
+[1m[4m                                 filepath                                 [0m[1m[4m [0m[1m[4m        metadata         [0m[1m[4m [0m
+{test_dir}/logfile_mux_zookeeper.0/zk-server-default-1 {"k8s_host":"k8s-node-1"} 
--- a/test/expected/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.err
+++ b/test/expected/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.err
--- a/test/expected/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.out
+++ b/test/expected/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.out
@ -0,0 +1 @@
+2024-04-04 18:30:54,422 [myid:] - INFO [nioEventLoopGroup-4-1:o.a.z.s.NettyServerCnxn@294] - Processing srvr command from /127.0.0.1:49034
--- a/test/expected/test_regex101.sh_7de76c174c58d67bf93e8f01d6d55ebb6a023f10.err
+++ b/test/expected/test_regex101.sh_7de76c174c58d67bf93e8f01d6d55ebb6a023f10.err
@ -1,3 +1,3 @@
 [1m[31m✘ error[0m: unknown regex: [1ms[0m
 [36m =[0m [36mnote[0m: did you mean one of the following?
-            [1mstd[0m
+         [1mstd[0m
--- a/test/logfile_mux_zookeeper.0
+++ b/test/logfile_mux_zookeeper.0
@ -0,0 +1 @@
+2024-04-04T20:30:55.424+02:00 source=k8s-node-1 2024-04-04 18:30:54,422 [myid:] - INFO [nioEventLoopGroup-4-1:o.a.z.s.NettyServerCnxn@294] - Processing srvr command from /127.0.0.1:49034 kubernetes_host=k8s-node-1 kubernetes_pod_name=zk-server-default-1
--- a/test/test_demux.sh
+++ b/test/test_demux.sh
@ -10,3 +10,9 @@ cat ${test_dir}/logfile_docker_compose_with_ts.0 | run_cap_test env TEST_COMMENT
     ${lnav_test} -n

 run_cap_test ${lnav_test} -n ${test_dir}/logfile_docker_compose_with_ts.0
+
+run_cap_test ${lnav_test} -n ${test_dir}/logfile_mux_zookeeper.0
+
+run_cap_test ${lnav_test} -n \
+    -c ';SELECT * FROM lnav_file_demux_metadata' \
+    ${test_dir}/logfile_mux_zookeeper.0
Author	SHA1	Message	Date
Tim Stack	8aa72b16e4	[demux] add lnav_file_demux_metadata view	2 weeks ago
Tim Stack	30c8beeb2e	[demux] add a demuxer for container with type	3 weeks ago
				`@ -0,0 +1 @@`
				`2024-04-04 18:30:54,422 [myid:] - INFO [nioEventLoopGroup-4-1:o.a.z.s.NettyServerCnxn@294] - Processing srvr command from /127.0.0.1:49034`
				`@ -0,0 +1 @@`
				`2024-04-04T20:30:55.424+02:00 source=k8s-node-1 2024-04-04 18:30:54,422 [myid:] - INFO [nioEventLoopGroup-4-1:o.a.z.s.NettyServerCnxn@294] - Processing srvr command from /127.0.0.1:49034 kubernetes_host=k8s-node-1 kubernetes_pod_name=zk-server-default-1`