From 8aa72b16e48a11c67659d2a6ec86c823f11aeaff Mon Sep 17 00:00:00 2001 From: Tim Stack Date: Wed, 1 May 2024 15:18:59 -0700 Subject: [PATCH] [demux] add lnav_file_demux_metadata view --- docs/source/config.rst | 1 + src/init.sql | 5 +++++ test/Makefile.am | 1 + test/expected/expected.am | 14 ++++---------- ...sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.err | 0 ...sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.out | 2 ++ ...sh_b640a3b46494e680a84cf042514ed465e39d8f38.err | 0 ...sh_b640a3b46494e680a84cf042514ed465e39d8f38.out | 1 + ...sh_7de76c174c58d67bf93e8f01d6d55ebb6a023f10.err | 2 +- test/logfile_mux_zookeeper.0 | 1 + test/test_demux.sh | 6 ++++++ 11 files changed, 22 insertions(+), 11 deletions(-) create mode 100644 test/expected/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.err create mode 100644 test/expected/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.out create mode 100644 test/expected/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.err create mode 100644 test/expected/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.out create mode 100644 test/logfile_mux_zookeeper.0 diff --git a/docs/source/config.rst b/docs/source/config.rst index 2d1fe6a3..e6756c6e 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -334,6 +334,7 @@ has the following known capture names: If there are additional captures, they will be included in the file metadata that can be accessed by the +:code:`lnav_file_demux_metadata` view of the :code:`lnav_file_metadata` table. Reference diff --git a/src/init.sql b/src/init.sql index f5042098..fce829ad 100644 --- a/src/init.sql +++ b/src/init.sql @@ -93,6 +93,11 @@ SELECT * FROM lnav_views WHERE name = (SELECT name FROM lnav_view_stack ORDER BY rowid DESC LIMIT 1); +CREATE VIEW lnav_file_demux_metadata AS +SELECT filepath, jget(content, '/demux_meta') AS metadata +FROM lnav_file_metadata +WHERE descriptor = 'org.lnav.piper.header'; + INSERT INTO lnav_example_log VALUES (0, NULL, '2017-02-03T04:05:06.100', '2017-02-03T04:05:06.100', 0, 'info', 0, NULL, NULL, NULL, 'hw', 2, 1486094706000, '/tmp/log', diff --git a/test/Makefile.am b/test/Makefile.am index f88e19ab..b300f0db 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -348,6 +348,7 @@ dist_noinst_DATA = \ logfile_invalid_json.json \ logfile_invalid_json2.json \ logfile_mixed_json2.json \ + logfile_mux_zookeeper.0 \ logfile_journald.json \ logfile_json.json \ logfile_json2.json \ diff --git a/test/expected/expected.am b/test/expected/expected.am index 8e935eaf..9b18fb0a 100644 --- a/test/expected/expected.am +++ b/test/expected/expected.am @@ -2,28 +2,20 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.err \ $(srcdir)/%reldir%/test_cli.sh_0b3639753916f71254e8c9cce4ebb8bfd9978d3e.out \ - $(srcdir)/%reldir%/test_cli.sh_108c6922cde063429f76abc3fdb8a81ad9a2f671.err \ - $(srcdir)/%reldir%/test_cli.sh_108c6922cde063429f76abc3fdb8a81ad9a2f671.out \ $(srcdir)/%reldir%/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.err \ $(srcdir)/%reldir%/test_cli.sh_10c33e465ef7681c6b5519d05d557426b26cd43d.out \ $(srcdir)/%reldir%/test_cli.sh_17a68b798354f9a6cdfab372006caeb74038d15c.err \ $(srcdir)/%reldir%/test_cli.sh_17a68b798354f9a6cdfab372006caeb74038d15c.out \ $(srcdir)/%reldir%/test_cli.sh_3114508cf42fb2608ef77f4bc294a84885c97a79.err \ $(srcdir)/%reldir%/test_cli.sh_3114508cf42fb2608ef77f4bc294a84885c97a79.out \ - $(srcdir)/%reldir%/test_cli.sh_4327033cfae0d4c170a38a3c4a570520bfabb493.err \ - $(srcdir)/%reldir%/test_cli.sh_4327033cfae0d4c170a38a3c4a570520bfabb493.out \ $(srcdir)/%reldir%/test_cli.sh_4761827b538a623c60d2e7d243bae9b653bb1975.err \ $(srcdir)/%reldir%/test_cli.sh_4761827b538a623c60d2e7d243bae9b653bb1975.out \ $(srcdir)/%reldir%/test_cli.sh_5524542b1a6954ff9741155101497270a2f0c557.err \ $(srcdir)/%reldir%/test_cli.sh_5524542b1a6954ff9741155101497270a2f0c557.out \ - $(srcdir)/%reldir%/test_cli.sh_76aa57821598962e59063a40c20171040c95a731.err \ - $(srcdir)/%reldir%/test_cli.sh_76aa57821598962e59063a40c20171040c95a731.out \ $(srcdir)/%reldir%/test_cli.sh_97e19b9ff3775d84074455a2e8993a0611b1c269.err \ $(srcdir)/%reldir%/test_cli.sh_97e19b9ff3775d84074455a2e8993a0611b1c269.out \ $(srcdir)/%reldir%/test_cli.sh_a2f9a3e798e5f4f09c92eaa25ae236a2fb09dbd9.err \ $(srcdir)/%reldir%/test_cli.sh_a2f9a3e798e5f4f09c92eaa25ae236a2fb09dbd9.out \ - $(srcdir)/%reldir%/test_cli.sh_af3ace7762b4cc150fcdcac86083b379bded7b32.err \ - $(srcdir)/%reldir%/test_cli.sh_af3ace7762b4cc150fcdcac86083b379bded7b32.out \ $(srcdir)/%reldir%/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.err \ $(srcdir)/%reldir%/test_cli.sh_c69c835a3c43210225cf62564b3e9584c899af20.out \ $(srcdir)/%reldir%/test_cli.sh_cc06341dd560f927512e92c7c0985ed8b25827ae.err \ @@ -274,6 +266,10 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_config.sh_eec3768ebc201ca63bca1411270965f78db1abfc.out \ $(srcdir)/%reldir%/test_demux.sh_090b34f84229c02aad7ab2f1cea20cfcf94446d5.err \ $(srcdir)/%reldir%/test_demux.sh_090b34f84229c02aad7ab2f1cea20cfcf94446d5.out \ + $(srcdir)/%reldir%/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.err \ + $(srcdir)/%reldir%/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.out \ + $(srcdir)/%reldir%/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.err \ + $(srcdir)/%reldir%/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.out \ $(srcdir)/%reldir%/test_demux.sh_babab3536a2fc7c2a99847fa355b50d40a51763d.err \ $(srcdir)/%reldir%/test_demux.sh_babab3536a2fc7c2a99847fa355b50d40a51763d.out \ $(srcdir)/%reldir%/test_demux.sh_f8cbb968fccbc0442a831c0f69c6dbdfe5413339.err \ @@ -694,8 +690,6 @@ EXPECTED_FILES = \ $(srcdir)/%reldir%/test_sql.sh_9a209f3ee1b1f543ca2587b695d2eb0e63e74c51.out \ $(srcdir)/%reldir%/test_sql.sh_9aaaa810f883f5dcf8a4d5cda4051fdf4640a65c.err \ $(srcdir)/%reldir%/test_sql.sh_9aaaa810f883f5dcf8a4d5cda4051fdf4640a65c.out \ - $(srcdir)/%reldir%/test_sql.sh_9b03e9f7a1bc35e408b3a17ee90cfdadea164df6.err \ - $(srcdir)/%reldir%/test_sql.sh_9b03e9f7a1bc35e408b3a17ee90cfdadea164df6.out \ $(srcdir)/%reldir%/test_sql.sh_9ceccab07fbf7130bffe3c201c710719e4a3e9af.err \ $(srcdir)/%reldir%/test_sql.sh_9ceccab07fbf7130bffe3c201c710719e4a3e9af.out \ $(srcdir)/%reldir%/test_sql.sh_9e1d05b821822ee40e13fadb24ec558f4bfcff10.err \ diff --git a/test/expected/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.err b/test/expected/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.out b/test/expected/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.out new file mode 100644 index 00000000..5fdb1876 --- /dev/null +++ b/test/expected/test_demux.sh_ac2a283ce1a4a3d99b0a9de4e9b72bbd9637cf61.out @@ -0,0 +1,2 @@ + filepath   metadata   +{test_dir}/logfile_mux_zookeeper.0/zk-server-default-1 {"k8s_host":"k8s-node-1"} diff --git a/test/expected/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.err b/test/expected/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.err new file mode 100644 index 00000000..e69de29b diff --git a/test/expected/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.out b/test/expected/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.out new file mode 100644 index 00000000..b937e359 --- /dev/null +++ b/test/expected/test_demux.sh_b640a3b46494e680a84cf042514ed465e39d8f38.out @@ -0,0 +1 @@ +2024-04-04 18:30:54,422 [myid:] - INFO [nioEventLoopGroup-4-1:o.a.z.s.NettyServerCnxn@294] - Processing srvr command from /127.0.0.1:49034 diff --git a/test/expected/test_regex101.sh_7de76c174c58d67bf93e8f01d6d55ebb6a023f10.err b/test/expected/test_regex101.sh_7de76c174c58d67bf93e8f01d6d55ebb6a023f10.err index 8b825ab0..119b17bf 100644 --- a/test/expected/test_regex101.sh_7de76c174c58d67bf93e8f01d6d55ebb6a023f10.err +++ b/test/expected/test_regex101.sh_7de76c174c58d67bf93e8f01d6d55ebb6a023f10.err @@ -1,3 +1,3 @@ ✘ error: unknown regex: s  = note: did you mean one of the following? - std + std diff --git a/test/logfile_mux_zookeeper.0 b/test/logfile_mux_zookeeper.0 new file mode 100644 index 00000000..9184c422 --- /dev/null +++ b/test/logfile_mux_zookeeper.0 @@ -0,0 +1 @@ +2024-04-04T20:30:55.424+02:00 source=k8s-node-1 2024-04-04 18:30:54,422 [myid:] - INFO [nioEventLoopGroup-4-1:o.a.z.s.NettyServerCnxn@294] - Processing srvr command from /127.0.0.1:49034 kubernetes_host=k8s-node-1 kubernetes_pod_name=zk-server-default-1 diff --git a/test/test_demux.sh b/test/test_demux.sh index 5e074dd0..77d614b6 100644 --- a/test/test_demux.sh +++ b/test/test_demux.sh @@ -10,3 +10,9 @@ cat ${test_dir}/logfile_docker_compose_with_ts.0 | run_cap_test env TEST_COMMENT ${lnav_test} -n run_cap_test ${lnav_test} -n ${test_dir}/logfile_docker_compose_with_ts.0 + +run_cap_test ${lnav_test} -n ${test_dir}/logfile_mux_zookeeper.0 + +run_cap_test ${lnav_test} -n \ + -c ';SELECT * FROM lnav_file_demux_metadata' \ + ${test_dir}/logfile_mux_zookeeper.0