[sql] some SQL prompt cleanup

src/base/string_util.cc

@@ -273,7 +273,7 @@ is_blank(const std::string& str)
 }
 
 std::string
-scrub_ws(const char* in)
+scrub_ws(const char* in, ssize_t len)
 {
     static const std::string TAB_SYMBOL = "\u21e5";
     static const std::string LF_SYMBOL = "\u240a";
@@ -281,7 +281,9 @@ scrub_ws(const char* in)
 
     std::string retval;
 
-    for (size_t lpc = 0; in[lpc]; lpc++) {
+    for (size_t lpc = 0; (len == -1 && in[lpc]) || (len >= 0 && lpc < len);
+         lpc++)
+    {
         auto ch = in[lpc];
 
         switch (ch) {

src/base/string_util.hh

@@ -100,7 +100,12 @@ endswith(const std::string& str, const char (&suffix)[N])
 
 void truncate_to(std::string& str, size_t max_char_len);
 
-std::string scrub_ws(const char* in);
+std::string scrub_ws(const char* in, ssize_t len = -1);
+inline std::string
+scrub_ws(const string_fragment& sf)
+{
+    return scrub_ws(sf.data(), sf.length());
+}
 
 inline std::string
 trim(const std::string& str)

src/field_overlay_source.cc

@@ -329,17 +329,16 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
             }
             str = "   " + field_name;
         } else {
-            auto_mem<char, sqlite3_free> jgetter;
+            str = lnav::sql::mprintf("   jget(%s, '/%q')",
-
+                                     meta.lvm_struct_name.get(),
-            jgetter = sqlite3_mprintf("   jget(%s, '/%q')",
+                                     meta.lvm_name.get());
-                                      meta.lvm_struct_name.get(),
-                                      meta.lvm_name.get());
-            str = jgetter;
         }
         str.append(this->fos_known_key_size - (str.length() - 3), ' ');
-        str += " = " + value_str;
 
         al.with_string(str);
+        readline_sqlite_highlighter(al, 0);
+
+        al.append(" = ").append(scrub_ws(value_str.c_str()));
         if (meta.lvm_struct_name.empty()) {
             auto prefix_len = field_name.length() - orig_field_name.length();
             al.with_attr(string_attr(
@@ -365,8 +364,8 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
                 .append(this->fos_known_key_size - meta.lvm_name.size() - 9 + 3,
                         ' ')
                 .append(" = ")
-                .append(
+                .append(scrub_ws(string_fragment::from_bytes(js.js_content.in(),
-                    string_fragment::from_bytes(js.js_content.in(), js.js_len));
+                                                             js.js_len)));
             this->fos_lines.emplace_back(al);
             this->add_key_line_attrs(this->fos_known_key_size);
         }
@@ -379,15 +378,13 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
     }
 
     for (const auto& extra_pair : this->fos_log_helper.ldh_extra_json) {
-        auto_mem<char, sqlite3_free> qname;
+        auto qname = lnav::sql::mprintf("%Q", extra_pair.first.c_str());
-
-        qname = sqlite3_mprintf("%Q", extra_pair.first.c_str());
-
         auto key_line = attr_line_t("   jget(log_raw_text, ")
                             .append(qname.in())
                             .append(")");
+        readline_sqlite_highlighter(key_line, 0);
         auto key_size = key_line.length();
-        key_line.append(" = ").append(extra_pair.second);
+        key_line.append(" = ").append(scrub_ws(extra_pair.second));
         this->fos_lines.emplace_back(key_line);
         this->add_key_line_attrs(key_size - 3);
     }
@@ -398,9 +395,9 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
         for (size_t lpc = 0; lpc < jpairs.size(); lpc++) {
             auto key_line = attr_line_t("   ").append(
                 this->fos_log_helper.format_json_getter(jpairs_map.first, lpc));
-
+            readline_sqlite_highlighter(key_line, 0);
             auto key_size = key_line.length();
-            key_line.append(" = ").append(jpairs[lpc].wt_value);
+            key_line.append(" = ").append(scrub_ws(jpairs[lpc].wt_value));
             this->fos_lines.emplace_back(key_line);
             this->add_key_line_attrs(key_size - 3);
         }
@@ -411,15 +408,18 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
     }
 
     for (const auto& xml_pair : this->fos_log_helper.ldh_xml_pairs) {
-        auto_mem<char, sqlite3_free> qname;
+        auto qname = sql_quote_ident(xml_pair.first.first.get());
-        auto_mem<char, sqlite3_free> xp_call;
+        auto xp_call = lnav::sql::mprintf(
-
+            "xpath(%Q, %s.%s)",
-        qname = sql_quote_ident(xml_pair.first.first.get());
+            xml_pair.first.second.c_str(),
-        xp_call = sqlite3_mprintf(
+            this->fos_log_helper.ldh_file->get_format()->get_name().c_str(),
-            "xpath(%Q, %s)", xml_pair.first.second.c_str(), qname.in());
+            qname.in());
-        this->fos_lines.emplace_back(fmt::format(
+        auto key_line = attr_line_t("   ").append(xp_call.in());
-            FMT_STRING("   {} = {}"), xp_call.in(), xml_pair.second));
+        readline_sqlite_highlighter(key_line, 0);
-        this->add_key_line_attrs(strlen(xp_call));
+        auto key_size = key_line.length();
+        key_line.append(" = ").append(scrub_ws(xml_pair.second));
+        this->fos_lines.emplace_back(key_line);
+        this->add_key_line_attrs(key_size - 3);
     }
 
     if (!this->fos_contexts.empty()

src/lnav.cc

@@ -296,7 +296,7 @@ setup_logline_table(exec_context& ec)
         nullptr,
     };
 
-    textview_curses& log_view = lnav_data.ld_views[LNV_LOG];
+    auto& log_view = lnav_data.ld_views[LNV_LOG];
     bool retval = false;
     bool update_possibilities
         = (lnav_data.ld_rl_view != nullptr && ec.ec_local_vars.size() == 1);
@@ -312,8 +312,8 @@ setup_logline_table(exec_context& ec)
 
     if (log_view.get_inner_height()) {
         static intern_string_t logline = intern_string::lookup("logline");
-        vis_line_t vl = log_view.get_selection();
+        auto vl = log_view.get_selection();
-        content_line_t cl = lnav_data.ld_log_source.at_base(vl);
+        auto cl = lnav_data.ld_log_source.at_base(vl);
 
         lnav_data.ld_vtab_manager->unregister_vtab(logline);
         lnav_data.ld_vtab_manager->register_vtab(
@@ -327,19 +327,27 @@ setup_logline_table(exec_context& ec)
 
             ldh.parse_line(cl);
 
-            std::map<const intern_string_t,
+            for (const auto& jextra : ldh.ldh_extra_json) {
-                     json_ptr_walk::walk_list_t>::const_iterator pair_iter;
+                lnav_data.ld_rl_view->add_possibility(
-            for (pair_iter = ldh.ldh_json_pairs.begin();
+                    ln_mode_t::SQL,
-                 pair_iter != ldh.ldh_json_pairs.end();
+                    "*",
-                 ++pair_iter)
+                    lnav::sql::mprintf("%Q", jextra.first.c_str()).in());
-            {
+            }
-                for (size_t lpc = 0; lpc < pair_iter->second.size(); lpc++) {
+            for (const auto& jpair : ldh.ldh_json_pairs) {
+                for (const auto& wt : jpair.second) {
                     lnav_data.ld_rl_view->add_possibility(
                         ln_mode_t::SQL,
                         "*",
-                        ldh.format_json_getter(pair_iter->first, lpc));
+                        lnav::sql::mprintf("%Q", wt.wt_ptr.c_str()).in());
                 }
             }
+            for (const auto& xml_pair : ldh.ldh_xml_pairs) {
+                lnav_data.ld_rl_view->add_possibility(
+                    ln_mode_t::SQL,
+                    "*",
+                    lnav::sql::mprintf("%Q", xml_pair.first.second.c_str())
+                        .in());
+            }
         }
 
         retval = true;

src/log_data_helper.cc

@@ -224,15 +224,13 @@ log_data_helper::get_line_bounds(size_t& line_index_out,
 std::string
 log_data_helper::format_json_getter(const intern_string_t field, int index)
 {
-    auto_mem<char, sqlite3_free> qname;
-    auto_mem<char, sqlite3_free> jget;
     std::string retval;
 
-    qname = sql_quote_ident(field.get());
+    auto qname = sql_quote_ident(field.get());
-    jget = sqlite3_mprintf("jget(%s,%Q)",
+    retval
-                           qname.in(),
+        = lnav::sql::mprintf("jget(%s,%Q)",
-                           this->ldh_json_pairs[field][index].wt_ptr.c_str());
+                             qname.in(),
-    retval = std::string(jget);
+                             this->ldh_json_pairs[field][index].wt_ptr.c_str());
 
     return retval;
 }

src/log_vtab_impl.cc

@@ -111,8 +111,6 @@ log_vtab_impl::get_table_statement()
         max_name_len = std::max(max_name_len, iter->vc_name.length());
     }
     for (iter = cols.begin(); iter != cols.end(); iter++) {
-        auto_mem<char, sqlite3_free> coldecl;
-        auto_mem<char, sqlite3_free> colname;
         std::string comment;
 
         require(!iter->vc_name.empty());
@@ -121,8 +119,8 @@ log_vtab_impl::get_table_statement()
             comment.append(" -- ").append(iter->vc_comment);
         }
 
-        colname = sql_quote_ident(iter->vc_name.c_str());
+        auto colname = sql_quote_ident(iter->vc_name.c_str());
-        coldecl = sqlite3_mprintf(
+        auto coldecl = lnav::sql::mprintf(
             "  %-*s %-7s %s COLLATE %-15Q,%s\n",
             max_name_len,
             colname.in(),

src/readline_callbacks.cc

@@ -906,6 +906,6 @@ rl_blur(readline_curses* rc)
                    .get_overlay_source();
 
     fos->fos_contexts.pop();
-    tc->set_sync_selection_and_top(fos->fos_contexts.top().c_show);
+    tc->set_sync_selection_and_top(false);
     lnav_data.ld_preview_generation += 1;
 }

src/readline_possibilities.cc

@@ -152,9 +152,8 @@ add_text_possibilities(readline_curses* rlc,
         switch (tq) {
             case text_quoting::sql: {
                 auto token_value = tok_res->to_string();
-                auto_mem<char, sqlite3_free> quoted_token;
+                auto quoted_token
-
+                    = lnav::sql::mprintf("%Q", token_value.c_str());
-                quoted_token = sqlite3_mprintf("%Q", token_value.c_str());
                 rlc->add_possibility(context, type, std::string(quoted_token));
                 break;
             }
@@ -265,9 +264,7 @@ add_filter_expr_possibilities(readline_curses* rlc,
                 continue;
             }
 
-            auto_mem<char> ident(sqlite3_free);
+            auto ident = sql_quote_ident(lv.lv_meta.lvm_name.get());
-
-            ident = sql_quote_ident(lv.lv_meta.lvm_name.get());
             auto bound_name = fmt::format(FMT_STRING(":{}"), ident.in());
             rlc->add_possibility(context, type, bound_name);
             switch (lv.lv_meta.lvm_kind) {

src/sql_util.cc

@@ -35,6 +35,7 @@
 #include "sql_util.hh"
 
 #include <ctype.h>
+#include <stdarg.h>
 #include <string.h>
 
 #include "base/auto_mem.hh"
@@ -573,12 +574,12 @@ sql_ident_needs_quote(const char* ident)
     return false;
 }
 
-char*
+auto_mem<char, sqlite3_free>
 sql_quote_ident(const char* ident)
 {
     bool needs_quote = false;
     size_t quote_count = 0, alloc_size;
-    char* retval;
+    auto_mem<char, sqlite3_free> retval;
 
     for (int lpc = 0; ident[lpc]; lpc++) {
         if ((lpc == 0 && isdigit(ident[lpc]))
@@ -592,8 +593,8 @@ sql_quote_ident(const char* ident)
     }
 
     alloc_size = strlen(ident) + quote_count * 2 + (needs_quote ? 2 : 0) + 1;
-    if ((retval = (char*) sqlite3_malloc(alloc_size)) == NULL) {
+    if ((retval = (char*) sqlite3_malloc(alloc_size)) == nullptr) {
-        retval = NULL;
+        retval = nullptr;
     } else {
         char* curr = retval;
 
@@ -1221,3 +1222,22 @@ find_sql_help_for_line(const attr_line_t& al, size_t x)
 
     return retval;
 }
+
+namespace lnav {
+namespace sql {
+
+auto_mem<char, sqlite3_free>
+mprintf(const char* fmt, ...)
+{
+    auto_mem<char, sqlite3_free> retval;
+    va_list args;
+
+    va_start(args, fmt);
+    retval = sqlite3_vmprintf(fmt, args);
+    va_end(args);
+
+    return retval;
+}
+
+}  // namespace sql
+}  // namespace lnav

src/sql_util.hh

@@ -104,7 +104,7 @@ void sql_install_logger();
 
 bool sql_ident_needs_quote(const char* ident);
 
-char* sql_quote_ident(const char* ident);
+auto_mem<char, sqlite3_free> sql_quote_ident(const char* ident);
 
 std::string sql_safe_ident(const string_fragment& ident);
 
@@ -125,4 +125,12 @@ int sqlite_authorizer(void* pUserData,
                       const char* detail3,
                       const char* detail4);
 
+namespace lnav {
+namespace sql {
+
+auto_mem<char, sqlite3_free> mprintf(const char* fmt, ...);
+
+}
+}  // namespace lnav
+
 #endif

test/expected/expected.am

@@ -1094,6 +1094,10 @@ EXPECTED_FILES = \
     $(srcdir)/%reldir%/test_sql_views_vtab.sh_28e23f4e98b1acd6478e39844fd9306b444550c3.out \
     $(srcdir)/%reldir%/test_sql_views_vtab.sh_32acc1a8bb5028636fdbf08f077f9a835ab51bec.err \
     $(srcdir)/%reldir%/test_sql_views_vtab.sh_32acc1a8bb5028636fdbf08f077f9a835ab51bec.out \
+    $(srcdir)/%reldir%/test_sql_views_vtab.sh_4363d60040424a573ed79ee4260a32e3cd72f62c.err \
+    $(srcdir)/%reldir%/test_sql_views_vtab.sh_4363d60040424a573ed79ee4260a32e3cd72f62c.out \
+    $(srcdir)/%reldir%/test_sql_views_vtab.sh_45dbef06572b43cb997682436e753a13e003f792.err \
+    $(srcdir)/%reldir%/test_sql_views_vtab.sh_45dbef06572b43cb997682436e753a13e003f792.out \
     $(srcdir)/%reldir%/test_sql_views_vtab.sh_485a6ac7c69bd4b5d34d3399a9c17f6a2dc89ad3.err \
     $(srcdir)/%reldir%/test_sql_views_vtab.sh_485a6ac7c69bd4b5d34d3399a9c17f6a2dc89ad3.out \
     $(srcdir)/%reldir%/test_sql_views_vtab.sh_62d15cb9d5a9259f198aa01ca8ed200d6da38d68.err \

test/expected/test_sql_views_vtab.sh_4363d60040424a573ed79ee4260a32e3cd72f62c.out

@@ -0,0 +1,46 @@
+2023-03-24T14:26:16.243 renovate[7] INFO     Dependency extraction complete
+ Received Time: 2023-03-24T14:26:16.243 — in the future  Format: %Y-%m-%dT%H:%M:%S.%L%z
+ Known message fields for table bunyan_log:
+   name     = renovate
+   hostname = renovate-gitlab-67c4bcb5-9ggbv
+   pid      = 7
+   level    = 30
+   v        = 0
+ JSON fields:
+   jget(log_raw_text, '/baseBranch') = main
+   jget(log_raw_text, '/logContext') = qjifsaDDI
+   jget(log_raw_text, '/repository') = webgui/custom-icons-transformer
+   jget(log_raw_text, '/stats') = {"managers":{"gitlabci":{"fileCount":1,"depCount":1},"gitlabci-include":{"fileCount":1,"depCount":1},"npm":{"fileCount":1,"depCount":15}},"total":{"fileCount":3,"depCount":17}}
+ No discovered message fields
+  logContext: qjifsaDDI
+  repository: webgui/custom-icons-transformer
+  baseBranch: main
+  stats: {"managers":{"gitlabci":{"fileCount":1,"depCount":1},"gitlabci-include":{"fileCount":1,"depCount":1},"npm":{"fileCount":1,"depCount":15}},"total":{"fileCount":3,"depCount":17}}
+2023-03-24T14:26:16.390 renovate[7] DEBUG    Dependency node has unsupported/unversioned value lts-bullseye-slim (versioning=docker)
+  logContext: qjifsaDDI
+  repository: webgui/custom-icons-transformer
+2023-03-24T14:26:17.493 renovate[7] DEBUG    Release 2.8.7 is pending status checks
+  logContext: qjifsaDDI
+  repository: webgui/custom-icons-transformer
+  depName: prettier
+  check: stabilityDays
+2023-03-24T14:26:17.897 renovate[7] DEBUG    Release 4.4.1 is pending status checks
+  logContext: qjifsaDDI
+  repository: webgui/custom-icons-transformer
+  depName: rimraf
+  check: stabilityDays
+2023-03-24T14:26:17.897 renovate[7] DEBUG    All releases are pending - using latest
+  logContext: qjifsaDDI
+  repository: webgui/custom-icons-transformer
+  depName: rimraf
+  bucket: non-major
+2023-03-24T14:26:18.330 renovate[7] DEBUG    Release 2.10.0 is pending status checks
+  logContext: qjifsaDDI
+  repository: webgui/custom-icons-transformer
+  depName: prettier-plugin-svelte
+  check: stabilityDays
+2023-03-24T14:26:18.331 renovate[7] DEBUG    All releases are pending - using latest
+  logContext: qjifsaDDI
+  repository: webgui/custom-icons-transformer
+  depName: prettier-plugin-svelte
+  bucket: non-major

test/expected/test_sql_views_vtab.sh_45dbef06572b43cb997682436e753a13e003f792.out

@@ -0,0 +1,47 @@
+[2020-12-10 06:56:41,092] DEBUG [connect.client:69] Full request text:
+ Received Time: 2020-12-10T06:56:41.092 — in the future  Format: %Y-%m-%d %H:%M:%S,%L
+ Pattern: /xml_msg_log/regex/std = ^\[(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]\s+(?<level>\w+)\s+\[(?<module>[^:]*):(?<line>\d+)\]\s*(?<body>[^\n]*)\n?(?<msg_data>.*)
+ Known message fields for table xml_msg_log:
+   module   = connect.client
+   line     = 69
+   msg_data = <?xml version='1.0' encoding='iso-8859-2'?>␊<a-request>␊  <head>␊    x␊  </head>␊  <source>␊    x␊  </source>␊  <request id="1">␊    <name>␊      x␊    </name>␊  </request>␊</a-request>␊
+ XML fields:
+   xpath('/a-request/head/text()', xml_msg_log.msg_data) = x
+   xpath('/a-request/request/@id', xml_msg_log.msg_data) = 1
+   xpath('/a-request/request/name/text()', xml_msg_log.msg_data) = x
+   xpath('/a-request/source/text()', xml_msg_log.msg_data) = x
+ No discovered message fields
+ └ #xml-req
+<?xml version='1.0' encoding='iso-8859-2'?>
+<a-request>
+  <head>
+    x
+  </head>
+  <source>
+    x
+  </source>
+  <request id="1">
+    <name>
+      x
+    </name>
+  </request>
+</a-request>
+
+[2020-12-10 06:56:41,099] DEBUG [m:85] Full reply text:
+<?xml version='1.0' encoding='iso-8859-2'?>
+<a-reply>
+  <head>
+    x
+  </head>
+  <reply id="2">
+    <status>
+      <result>OK</result>
+    </status>
+    <name>
+      x
+    </name>
+  </reply>
+  <technical-track>
+    x
+  </technical-track>
+</a-reply>

test/test_sql_views_vtab.sh

@@ -179,6 +179,16 @@ run_cap_test ${lnav_test} -n \
     -c ":write-json-to -" \
     ${test_dir}/logfile_xml_msg.0
 
+run_cap_test ${lnav_test} -n -I ${test_dir} \
+    -c ";UPDATE lnav_views SET options = json_object('row-details', 'show') WHERE name = 'log'" \
+    -c ":goto 2" \
+    ${test_dir}/logfile_xml_msg.0
+
+run_cap_test ${lnav_test} -n -I ${test_dir} \
+    -c ";UPDATE lnav_views SET options = json_object('row-details', 'show') WHERE name = 'log'" \
+    -c ":goto 9" \
+    ${test_dir}/logfile_bunyan.0
+
 run_cap_test ${lnav_test} -n \
     -c ";UPDATE lnav_views SET top_meta = json_object('file', 'bad') WHERE name = 'text'" \
     ${test_dir}/textfile_ansi.0