[blog] add a post about PRQL

docs/_posts/2024-03-29-prql-support.md

@@ -0,0 +1,51 @@
+---
+layout: post
+title: Support for the PRQL in the database query prompt
+excerpt: >-
+  PRQL is a database query language that is pipeline-oriented
+  and easier to use interactively
+---
+
+The v0.12.1 release of lnav includes support for
+[PRQL](https://prql-lang.org).  PRQL is a database query language
+that has a pipeline-oriented syntax.  The main advantage of PRQL,
+in the context of lnav, is that it is easier to work with
+interactively compared to SQL.  For example, lnav can provide
+previews of different stages of the pipeline and provide more
+accurate tab-completions for the columns in the result set.  I'm
+hoping that the ease-of-use will make doing log analysis in lnav
+much easier.
+
+You can execute a PRQL query using the existing database prompt
+(press `;`).  A query is interpreted as PRQL if it starts with
+the [`from`](https://prql-lang.org/book/reference/data/from.html)
+keyword.  After `from`, the database table should be provided.
+The table for the focused log message will be suggested by default.
+You can accept the suggestion by pressing TAB.  To add a new stage
+to the pipeline, enter a pipe symbol (`|`), followed by a
+[PRQL transform](https://prql-lang.org/book/reference/stdlib/transforms/index.html)
+and its arguments.  In addition to the standard set of transforms,
+lnav provides some convenience transforms in the `stats` and `utils`
+namespaces.  For example, `stats.count_by` can be passed one or more
+column names to group by and count, with the result sorted by most
+to least.
+
+As you enter a query, lnav will update various panels on the display
+to show help, preview data, and errors.  The following is a
+screenshot of lnav viewing a web access log with a query in progress:
+
+![Screenshot of PRQL in action](/assets/images/lnav-prql-preview.png)
+
+The top half is the usual log message view.  Below that is the online
+help panel showing the documentation for the `stats.count_by` PRQL
+function.  lnav will show the help for what is currently under the
+cursor.  The next panel shows the preview data for the pipeline stage
+that precedes the stage where the cursor is.  In this case, the
+results of `from access_log`, which is the contents of the access
+log table.  The second preview window shows the result of the
+pipeline stage where the cursor is located.
+
+There is still a lot of work to be done on the integration and PRQL
+itself, but I'm very hopeful this will work out well in the long
+term.  Many thanks to the PRQL team for starting the project and
+keeping it going, it's not easy competing with SQL.

src/plain_text_source.cc

@@ -528,6 +528,9 @@ plain_text_source::adjacent_anchor(vis_line_t vl, text_anchors::direction dir)
                 if (neighbors_res->cnr_next) {
                     return this->line_for_offset(
                         neighbors_res->cnr_next.value()->hn_start);
+                } else if (!md.m_sections_root->hn_children.empty()) {
+                    return this->line_for_offset(
+                        md.m_sections_root->hn_children[0]->hn_start);
                 }
                 break;
             }

src/prql/prql.am

@@ -1,4 +1,5 @@
 
 PRQL_FILES = \
     $(srcdir)/%reldir%/stats.prql \
+    $(srcdir)/%reldir%/utils.prql \
     $()

src/prql/stats.prql

@@ -3,3 +3,17 @@ let count_by = func column rel <relation> -> <relation> (
   group {column} (aggregate {total = count this})
   sort {-total}
 )
+
+let average_of = func column rel <relation> -> <relation> (
+  rel
+  aggregate {value = average column}
+)
+
+let sum_of = func column rel <relation> -> <relation> (
+  (rel | aggregate {total = sum column})
+)
+
+let by = func column values rel <relation> -> <relation> (
+  rel
+  group {column} (aggregate values)
+)

src/prql/utils.prql

@@ -0,0 +1,5 @@
+let distinct = func column rel <relation> -> <relation> (
+  rel
+  select {column}
+  group {column} (take 1)
+)

src/readline_callbacks.cc

@@ -583,7 +583,7 @@ rl_search_internal(readline_curses* rc, ln_mode_t mode, bool complete = false)
                         continue;
                     }
                     curr_stage_prql.insert(riter->sa_range.lr_start,
-                                           "| take 1000 ");
+                                           "| take 10000 ");
                 }
                 curr_stage_prql.rtrim();
                 curr_stage_prql.append(" | take 5");
@@ -608,7 +608,7 @@ rl_search_internal(readline_curses* rc, ln_mode_t mode, bool complete = false)
                             continue;
                         }
                         prev_stage_prql.insert(riter->sa_range.lr_start,
-                                               "| take 1000 ");
+                                               "| take 10000 ");
                     }
                     prev_stage_prql.append(" | take 5");
 

src/sql_commands.cc

@@ -572,7 +572,7 @@ static readline_context::command_t sql_commands[] = {
                     .with_grouping("(", ")"))
             .with_example({
                 "To group by log_level and count the rows in each partition",
-                "from db.lnav_example_log | group { log_level } (aggregate { "
+                "from lnav_example_log | group { log_level } (aggregate { "
                 "count this })",
                 help_example::language::prql,
             }),
@@ -623,6 +623,21 @@ static readline_context::command_t sql_commands[] = {
         "prql-source",
         {"prql-source"},
     },
+    {
+        "stats.average_of",
+        prql_cmd_sort,
+        help_text("stats.average_of", "Compute the average of col")
+            .prql_function()
+            .with_parameter(help_text{"col", "The column to average"})
+            .with_example({
+                "To get the average of a",
+                "from [{a=1}, {a=1}, {a=2}] | stats.average_of a",
+                help_example::language::prql,
+            }),
+        nullptr,
+        "prql-source",
+        {"prql-source"},
+    },
     {
         "stats.count_by",
         prql_cmd_sort,
@@ -642,6 +657,38 @@ static readline_context::command_t sql_commands[] = {
         "prql-source",
         {"prql-source"},
     },
+    {
+        "stats.sum_of",
+        prql_cmd_sort,
+        help_text("stats.sum_of", "Compute the sum of col")
+            .prql_function()
+            .with_parameter(help_text{"col", "The column to sum"})
+            .with_example({
+                "To get the sum of a",
+                "from [{a=1}, {a=1}, {a=2}] | stats.sum_of a",
+                help_example::language::prql,
+            }),
+        nullptr,
+        "prql-source",
+        {"prql-source"},
+    },
+    {
+        "stats.by",
+        prql_cmd_sort,
+        help_text("stats.by", "A shorthand for grouping and aggregating")
+            .prql_function()
+            .with_parameter(help_text{"col", "The column to sum"})
+            .with_parameter(help_text{"values", "The aggregations to perform"})
+            .with_example({
+                "To partition by a and get the sum of b",
+                "from [{a=1, b=1}, {a=1, b=1}, {a=2, b=1}] | stats.by a "
+                "{sum b}",
+                help_example::language::prql,
+            }),
+        nullptr,
+        "prql-source",
+        {"prql-source"},
+    },
     {
         "sort",
         prql_cmd_sort,
@@ -681,6 +728,22 @@ static readline_context::command_t sql_commands[] = {
         "prql-source",
         {"prql-source"},
     },
+    {
+        "utils.distinct",
+        prql_cmd_sort,
+        help_text("utils.distinct",
+                  "A shorthand for getting distinct values of col")
+            .prql_function()
+            .with_parameter(help_text{"col", "The column to sum"})
+            .with_example({
+                "To get the distinct values of a",
+                "from [{a=1}, {a=1}, {a=2}] | utils.distinct a",
+                help_example::language::prql,
+            }),
+        nullptr,
+        "prql-source",
+        {"prql-source"},
+    },
 };
 
 static readline_context::command_map_t sql_cmd_map;

src/sql_util.cc

@@ -1145,6 +1145,9 @@ annotate_sql_statement(attr_line_t& al)
 std::vector<const help_text*>
 find_sql_help_for_line(const attr_line_t& al, size_t x)
 {
+    static const auto* sql_cmd_map
+        = injector::get<readline_context::command_map_t*, sql_cmd_map_tag>();
+
     std::vector<const help_text*> retval;
     const auto& sa = al.get_attrs();
     std::string name;
@@ -1152,10 +1155,6 @@ find_sql_help_for_line(const attr_line_t& al, size_t x)
     x = al.nearest_text(x);
 
     {
-        const auto* sql_cmd_map
-            = injector::get<readline_context::command_map_t*,
-                            sql_cmd_map_tag>();
-
         auto sa_opt = get_string_attr(al.get_attrs(), &SQL_COMMAND_ATTR);
         if (sa_opt) {
             auto cmd_name = al.get_substring((*sa_opt)->sa_range);
@@ -1182,6 +1181,11 @@ find_sql_help_for_line(const attr_line_t& al, size_t x)
         al.get_attrs(), &lnav::sql ::PRQL_FQID_ATTR, x);
     if (prql_fqid_iter != al.get_attrs().end()) {
         auto fqid = al.get_substring(prql_fqid_iter->sa_range);
+        auto cmd_iter = sql_cmd_map->find(fqid);
+        if (cmd_iter != sql_cmd_map->end()) {
+            return {&cmd_iter->second->c_help};
+        }
+
         auto func_pair = lnav::sql::prql_functions.equal_range(fqid);
 
         for (auto func_iter = func_pair.first; func_iter != func_pair.second;

src/textfile_sub_source.cc

@@ -1257,6 +1257,9 @@ textfile_sub_source::adjacent_anchor(vis_line_t vl, text_anchors::direction dir)
                 if (neighbors_res->cnr_next) {
                     return to_vis_line(
                         lf, neighbors_res->cnr_next.value()->hn_start);
+                } else if (!md.m_sections_root->hn_children.empty()) {
+                    return to_vis_line(
+                        lf, md.m_sections_root->hn_children[0]->hn_start);
                 }
                 break;
             }

src/view_helpers.cc

@@ -1019,7 +1019,9 @@ execute_examples()
     }
     for (auto cmd_pair : *sql_cmd_map) {
         if (cmd_pair.second->c_help.ht_context
-            != help_context_t::HC_PRQL_TRANSFORM)
+                != help_context_t::HC_PRQL_TRANSFORM
+            && cmd_pair.second->c_help.ht_context
+                != help_context_t::HC_PRQL_FUNCTION)
         {
             continue;
         }