diff --git a/docs/_posts/2024-03-29-prql-support.md b/docs/_posts/2024-03-29-prql-support.md new file mode 100644 index 00000000..98df8e68 --- /dev/null +++ b/docs/_posts/2024-03-29-prql-support.md @@ -0,0 +1,51 @@ +--- +layout: post +title: Support for the PRQL in the database query prompt +excerpt: >- + PRQL is a database query language that is pipeline-oriented + and easier to use interactively +--- + +The v0.12.1 release of lnav includes support for +[PRQL](https://prql-lang.org). PRQL is a database query language +that has a pipeline-oriented syntax. The main advantage of PRQL, +in the context of lnav, is that it is easier to work with +interactively compared to SQL. For example, lnav can provide +previews of different stages of the pipeline and provide more +accurate tab-completions for the columns in the result set. I'm +hoping that the ease-of-use will make doing log analysis in lnav +much easier. + +You can execute a PRQL query using the existing database prompt +(press `;`). A query is interpreted as PRQL if it starts with +the [`from`](https://prql-lang.org/book/reference/data/from.html) +keyword. After `from`, the database table should be provided. +The table for the focused log message will be suggested by default. +You can accept the suggestion by pressing TAB. To add a new stage +to the pipeline, enter a pipe symbol (`|`), followed by a +[PRQL transform](https://prql-lang.org/book/reference/stdlib/transforms/index.html) +and its arguments. In addition to the standard set of transforms, +lnav provides some convenience transforms in the `stats` and `utils` +namespaces. For example, `stats.count_by` can be passed one or more +column names to group by and count, with the result sorted by most +to least. + +As you enter a query, lnav will update various panels on the display +to show help, preview data, and errors. The following is a +screenshot of lnav viewing a web access log with a query in progress: + +![Screenshot of PRQL in action](/assets/images/lnav-prql-preview.png) + +The top half is the usual log message view. Below that is the online +help panel showing the documentation for the `stats.count_by` PRQL +function. lnav will show the help for what is currently under the +cursor. The next panel shows the preview data for the pipeline stage +that precedes the stage where the cursor is. In this case, the +results of `from access_log`, which is the contents of the access +log table. The second preview window shows the result of the +pipeline stage where the cursor is located. + +There is still a lot of work to be done on the integration and PRQL +itself, but I'm very hopeful this will work out well in the long +term. Many thanks to the PRQL team for starting the project and +keeping it going, it's not easy competing with SQL. diff --git a/docs/assets/images/lnav-prql-preview.png b/docs/assets/images/lnav-prql-preview.png new file mode 100644 index 00000000..1c484d70 Binary files /dev/null and b/docs/assets/images/lnav-prql-preview.png differ diff --git a/src/plain_text_source.cc b/src/plain_text_source.cc index 22fd834e..88b8e15d 100644 --- a/src/plain_text_source.cc +++ b/src/plain_text_source.cc @@ -528,6 +528,9 @@ plain_text_source::adjacent_anchor(vis_line_t vl, text_anchors::direction dir) if (neighbors_res->cnr_next) { return this->line_for_offset( neighbors_res->cnr_next.value()->hn_start); + } else if (!md.m_sections_root->hn_children.empty()) { + return this->line_for_offset( + md.m_sections_root->hn_children[0]->hn_start); } break; } diff --git a/src/prql/prql.am b/src/prql/prql.am index a1780799..f53487ad 100644 --- a/src/prql/prql.am +++ b/src/prql/prql.am @@ -1,4 +1,5 @@ PRQL_FILES = \ $(srcdir)/%reldir%/stats.prql \ + $(srcdir)/%reldir%/utils.prql \ $() diff --git a/src/prql/stats.prql b/src/prql/stats.prql index b29dabc0..66c4bb0a 100644 --- a/src/prql/stats.prql +++ b/src/prql/stats.prql @@ -3,3 +3,17 @@ let count_by = func column rel -> ( group {column} (aggregate {total = count this}) sort {-total} ) + +let average_of = func column rel -> ( + rel + aggregate {value = average column} +) + +let sum_of = func column rel -> ( + (rel | aggregate {total = sum column}) +) + +let by = func column values rel -> ( + rel + group {column} (aggregate values) +) diff --git a/src/prql/utils.prql b/src/prql/utils.prql new file mode 100644 index 00000000..326f5ba4 --- /dev/null +++ b/src/prql/utils.prql @@ -0,0 +1,5 @@ +let distinct = func column rel -> ( + rel + select {column} + group {column} (take 1) +) diff --git a/src/readline_callbacks.cc b/src/readline_callbacks.cc index 3eed553d..bc81a5a0 100644 --- a/src/readline_callbacks.cc +++ b/src/readline_callbacks.cc @@ -583,7 +583,7 @@ rl_search_internal(readline_curses* rc, ln_mode_t mode, bool complete = false) continue; } curr_stage_prql.insert(riter->sa_range.lr_start, - "| take 1000 "); + "| take 10000 "); } curr_stage_prql.rtrim(); curr_stage_prql.append(" | take 5"); @@ -608,7 +608,7 @@ rl_search_internal(readline_curses* rc, ln_mode_t mode, bool complete = false) continue; } prev_stage_prql.insert(riter->sa_range.lr_start, - "| take 1000 "); + "| take 10000 "); } prev_stage_prql.append(" | take 5"); diff --git a/src/sql_commands.cc b/src/sql_commands.cc index bacaf771..33fa1384 100644 --- a/src/sql_commands.cc +++ b/src/sql_commands.cc @@ -572,7 +572,7 @@ static readline_context::command_t sql_commands[] = { .with_grouping("(", ")")) .with_example({ "To group by log_level and count the rows in each partition", - "from db.lnav_example_log | group { log_level } (aggregate { " + "from lnav_example_log | group { log_level } (aggregate { " "count this })", help_example::language::prql, }), @@ -623,6 +623,21 @@ static readline_context::command_t sql_commands[] = { "prql-source", {"prql-source"}, }, + { + "stats.average_of", + prql_cmd_sort, + help_text("stats.average_of", "Compute the average of col") + .prql_function() + .with_parameter(help_text{"col", "The column to average"}) + .with_example({ + "To get the average of a", + "from [{a=1}, {a=1}, {a=2}] | stats.average_of a", + help_example::language::prql, + }), + nullptr, + "prql-source", + {"prql-source"}, + }, { "stats.count_by", prql_cmd_sort, @@ -642,6 +657,38 @@ static readline_context::command_t sql_commands[] = { "prql-source", {"prql-source"}, }, + { + "stats.sum_of", + prql_cmd_sort, + help_text("stats.sum_of", "Compute the sum of col") + .prql_function() + .with_parameter(help_text{"col", "The column to sum"}) + .with_example({ + "To get the sum of a", + "from [{a=1}, {a=1}, {a=2}] | stats.sum_of a", + help_example::language::prql, + }), + nullptr, + "prql-source", + {"prql-source"}, + }, + { + "stats.by", + prql_cmd_sort, + help_text("stats.by", "A shorthand for grouping and aggregating") + .prql_function() + .with_parameter(help_text{"col", "The column to sum"}) + .with_parameter(help_text{"values", "The aggregations to perform"}) + .with_example({ + "To partition by a and get the sum of b", + "from [{a=1, b=1}, {a=1, b=1}, {a=2, b=1}] | stats.by a " + "{sum b}", + help_example::language::prql, + }), + nullptr, + "prql-source", + {"prql-source"}, + }, { "sort", prql_cmd_sort, @@ -681,6 +728,22 @@ static readline_context::command_t sql_commands[] = { "prql-source", {"prql-source"}, }, + { + "utils.distinct", + prql_cmd_sort, + help_text("utils.distinct", + "A shorthand for getting distinct values of col") + .prql_function() + .with_parameter(help_text{"col", "The column to sum"}) + .with_example({ + "To get the distinct values of a", + "from [{a=1}, {a=1}, {a=2}] | utils.distinct a", + help_example::language::prql, + }), + nullptr, + "prql-source", + {"prql-source"}, + }, }; static readline_context::command_map_t sql_cmd_map; diff --git a/src/sql_util.cc b/src/sql_util.cc index 92b6d4ad..6b1459a4 100644 --- a/src/sql_util.cc +++ b/src/sql_util.cc @@ -1145,6 +1145,9 @@ annotate_sql_statement(attr_line_t& al) std::vector find_sql_help_for_line(const attr_line_t& al, size_t x) { + static const auto* sql_cmd_map + = injector::get(); + std::vector retval; const auto& sa = al.get_attrs(); std::string name; @@ -1152,10 +1155,6 @@ find_sql_help_for_line(const attr_line_t& al, size_t x) x = al.nearest_text(x); { - const auto* sql_cmd_map - = injector::get(); - auto sa_opt = get_string_attr(al.get_attrs(), &SQL_COMMAND_ATTR); if (sa_opt) { auto cmd_name = al.get_substring((*sa_opt)->sa_range); @@ -1182,6 +1181,11 @@ find_sql_help_for_line(const attr_line_t& al, size_t x) al.get_attrs(), &lnav::sql ::PRQL_FQID_ATTR, x); if (prql_fqid_iter != al.get_attrs().end()) { auto fqid = al.get_substring(prql_fqid_iter->sa_range); + auto cmd_iter = sql_cmd_map->find(fqid); + if (cmd_iter != sql_cmd_map->end()) { + return {&cmd_iter->second->c_help}; + } + auto func_pair = lnav::sql::prql_functions.equal_range(fqid); for (auto func_iter = func_pair.first; func_iter != func_pair.second; diff --git a/src/textfile_sub_source.cc b/src/textfile_sub_source.cc index eafefbbf..87dce712 100644 --- a/src/textfile_sub_source.cc +++ b/src/textfile_sub_source.cc @@ -1257,6 +1257,9 @@ textfile_sub_source::adjacent_anchor(vis_line_t vl, text_anchors::direction dir) if (neighbors_res->cnr_next) { return to_vis_line( lf, neighbors_res->cnr_next.value()->hn_start); + } else if (!md.m_sections_root->hn_children.empty()) { + return to_vis_line( + lf, md.m_sections_root->hn_children[0]->hn_start); } break; } diff --git a/src/view_helpers.cc b/src/view_helpers.cc index 2185c504..7a0828db 100644 --- a/src/view_helpers.cc +++ b/src/view_helpers.cc @@ -1019,7 +1019,9 @@ execute_examples() } for (auto cmd_pair : *sql_cmd_map) { if (cmd_pair.second->c_help.ht_context - != help_context_t::HC_PRQL_TRANSFORM) + != help_context_t::HC_PRQL_TRANSFORM + && cmd_pair.second->c_help.ht_context + != help_context_t::HC_PRQL_FUNCTION) { continue; }