[sql] add a spooky_hash() function

Fixes #567

NEWS

@@ -13,6 +13,8 @@ lnav v0.8.5:
        given file.  This command is mostly useful in scripts where one might
        want to redirect all output from commands like ":echo" and ":write-to -"
        to a single file.
+     * Added 'spooky_hash()' and 'group_spooky_hash()' SQL functions to
+       generate a hash of their parameters.
 
      Interface Changes:
      * The auto-complete behavior in the prompt has been modified to fall back

docs/source/sqlext.rst

@@ -128,6 +128,10 @@ Additional string comparison and manipulation functions:
 * extract(str) - Parse and extract values from a string using the same
   algorithm as the *logline* table (see :ref:`data-ext`).  The discovered
   data is returned as a JSON-object that you can do further processing on.
+* spooky_hash(str1, ...) - Compute the hash value for the given arguments using
+  the "spooky" hash algorithm.
+* group_spooky_hash(str1, ...) - An aggregate version of the "spooky_hash()"
+  function.
 
 File Paths
 ----------

src/sqlite-extension-func.cc

@@ -92,6 +92,8 @@ int register_sqlite_funcs(sqlite3 *db, sqlite_registration_func_t *reg_funcs)
         }
 
         for (i = 0; agg_funcs && agg_funcs[i].zName; i++) {
+            struct FuncDefAgg &fda = agg_funcs[i];
+
             //sqlite3CreateFunc
             sqlite3_create_function(db,
                                     agg_funcs[i].zName,
@@ -101,6 +103,13 @@ int register_sqlite_funcs(sqlite3 *db, sqlite_registration_func_t *reg_funcs)
                                     0,
                                     agg_funcs[i].xStep,
                                     agg_funcs[i].xFinalize);
+
+            if (fda.fda_help.ht_context != HC_NONE) {
+                help_text &ht = fda.fda_help;
+
+                sqlite_function_help.insert(make_pair(ht.ht_name, &ht));
+                ht.index_tags();
+            }
         }
     }
 

src/sqlite-extension-func.hh

@@ -55,6 +55,7 @@ struct FuncDefAgg {
     uint8_t needCollSeq;
     void (*xStep)(sqlite3_context*,int,sqlite3_value**);
     void (*xFinalize)(sqlite3_context*);
+    help_text fda_help;
 };
 
 typedef int (*sqlite_registration_func_t)(struct FuncDef **basic_funcs,

src/string-extension-functions.cc

@@ -198,6 +198,64 @@ string regexp_replace(const char *str, const char *re, const char *repl)
     return dest;
 }
 
+static
+string spooky_hash(const vector<const char *> args)
+{
+    byte_array<2, uint64> hash;
+    SpookyHash context;
+
+    context.Init(0, 0);
+    for (const auto arg : args) {
+        int64_t len = arg != nullptr ? strlen(arg) : 0;
+
+        context.Update(&len, sizeof(len));
+        if (arg == nullptr) {
+            continue;
+        }
+        context.Update(arg, len);
+    }
+    context.Final(hash.out(0), hash.out(1));
+
+    return hash.to_string();
+}
+
+static void sql_spooky_hash_step(sqlite3_context *context,
+                                 int argc,
+                                 sqlite3_value **argv)
+{
+    auto *hasher = (SpookyHash *)sqlite3_aggregate_context(context,
+        sizeof(SpookyHash));
+
+    for (int lpc = 0; lpc < argc; lpc++) {
+        auto value = sqlite3_value_text(argv[lpc]);
+        int64_t len = value != nullptr ? strlen((const char *) value) : 0;
+
+        hasher->Update(&len, sizeof(len));
+        if (value == nullptr) {
+            continue;
+        }
+        hasher->Update(value, len);
+    }
+}
+
+static void sql_spooky_hash_final(sqlite3_context *context)
+{
+    auto *hasher = (SpookyHash *)sqlite3_aggregate_context(
+        context, sizeof(SpookyHash));
+
+    if (hasher == nullptr) {
+        sqlite3_result_null(context);
+    } else {
+        byte_array<2, uint64> hash;
+
+        hasher->Final(hash.out(0), hash.out(1));
+
+        string hex = hash.to_string();
+        sqlite3_result_text(context, hex.c_str(), hex.length(),
+                            SQLITE_TRANSIENT);
+    }
+}
+
 int string_extension_functions(struct FuncDef **basic_funcs,
                                struct FuncDefAgg **agg_funcs)
 {
@@ -270,10 +328,41 @@ int string_extension_functions(struct FuncDef **basic_funcs,
                 .with_example({"SELECT endswith('notbad.png', '.jpg')"})
         ),
 
-        { NULL }
+        sqlite_func_adapter<decltype(&spooky_hash), spooky_hash>::builder(
+            help_text("spooky_hash",
+                      "Compute the hash value for the given arguments.")
+                .sql_function()
+                .with_parameter(help_text("str", "The string to hash")
+                                    .one_or_more())
+                .with_tags({"string"})
+                .with_example({"SELECT spooky_hash('Hello, World!')"})
+                .with_example({"SELECT spooky_hash('Hello, World!', NULL)"})
+                .with_example({"SELECT spooky_hash('Hello, World!', '')"})
+                .with_example({"SELECT spooky_hash('Hello, World!', 123)"})
+        ),
+
+        {nullptr}
+    };
+
+    static struct FuncDefAgg str_agg_funcs[] = {
+        {"group_spooky_hash", -1, 0,
+            sql_spooky_hash_step, sql_spooky_hash_final,
+            help_text("group_spooky_hash",
+                      "Compute the hash value for the given arguments")
+                .sql_function()
+                .with_parameter(help_text("str", "The string to hash")
+                                    .one_or_more())
+                .with_tags({"string"})
+                .with_example({"SELECT spooky_hash('abc', '123')"})
+                .with_example(
+                    {"SELECT group_spooky_hash(column1) FROM (VALUES ('abc'), ('123'))"})
+        },
+
+        {nullptr}
     };
 
     *basic_funcs = string_funcs;
+    *agg_funcs = str_agg_funcs;
 
     return SQLITE_OK;
 }

test/expected_help.txt

@@ -1047,9 +1047,10 @@ Synopsis
 Parameter
   X   The unicode code point values
 See Also
-  endswith(), extract(), group_concat(), instr(), leftstr(), length(), lower(), 
-  ltrim(), printf(), regexp_match(), regexp_replace(), replace(), replicate(), 
-  reverse(), rightstr(), rtrim(), startswith(), substr(), trim(), upper()
+  endswith(), extract(), group_concat(), group_spooky_hash(), instr(), leftstr(), 
+  length(), lower(), ltrim(), printf(), regexp_match(), regexp_replace(), 
+  replace(), replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Example
 #1 ;SELECT char(0x48, 0x49)
    
@@ -1148,9 +1149,10 @@ Parameters
   str      The string to test
   suffix   The suffix to check in the string
 See Also
-  char(), extract(), group_concat(), instr(), leftstr(), length(), lower(), 
-  ltrim(), printf(), regexp_match(), regexp_replace(), replace(), replicate(), 
-  reverse(), rightstr(), rtrim(), startswith(), substr(), trim(), upper()
+  char(), extract(), group_concat(), group_spooky_hash(), instr(), leftstr(), 
+  length(), lower(), ltrim(), printf(), regexp_match(), regexp_replace(), 
+  replace(), replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Examples
 #1 ;SELECT endswith('notbad.jpg', '.jpg')
    
@@ -1164,9 +1166,10 @@ Synopsis
 Parameter
   str   The string to parse
 See Also
-  char(), endswith(), group_concat(), instr(), leftstr(), length(), lower(), 
-  ltrim(), printf(), regexp_match(), regexp_replace(), replace(), replicate(), 
-  reverse(), rightstr(), rtrim(), startswith(), substr(), trim(), upper()
+  char(), endswith(), group_concat(), group_spooky_hash(), instr(), leftstr(), 
+  length(), lower(), ltrim(), printf(), regexp_match(), regexp_replace(), 
+  replace(), replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Examples
 #1 ;SELECT extract('foo=1 bar=2 name="Rolo Tomassi"')
    
@@ -1215,9 +1218,10 @@ Parameters
   X     The value to concatenate.
   sep   The separator to place between the values.
 See Also
-  char(), endswith(), extract(), instr(), leftstr(), length(), lower(), ltrim(), 
-  printf(), regexp_match(), regexp_replace(), replace(), replicate(), reverse(), 
-  rightstr(), rtrim(), startswith(), substr(), trim(), upper()
+  char(), endswith(), extract(), group_spooky_hash(), instr(), leftstr(), 
+  length(), lower(), ltrim(), printf(), regexp_match(), regexp_replace(), 
+  replace(), replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Examples
 #1 ;SELECT group_concat(ex_procname) FROM lnav_example_log
    
@@ -1229,6 +1233,24 @@ Examples
    
 
 
+Synopsis
+  group_spooky_hash(str, ...) -- Compute the hash value for the given arguments
+    
+Parameter
+  str   The string to hash
+See Also
+  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
+  lower(), ltrim(), printf(), regexp_match(), regexp_replace(), replace(), 
+  replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), startswith(), 
+  substr(), trim(), upper()
+Examples
+#1 ;SELECT spooky_hash('abc', '123')
+   
+
+#2 ;SELECT group_spooky_hash(column1) FROM (VALUES ('abc'), ('123'))
+   
+
+
 Synopsis
   hex(X) -- Returns a string which is the upper-case hexadecimal rendering of 
     the content of its argument.
@@ -1260,9 +1282,10 @@ Parameters
   haystack   The string to search within
   needle     The string to look for in the haystack
 See Also
-  char(), endswith(), extract(), group_concat(), leftstr(), length(), lower(), 
-  ltrim(), printf(), regexp_match(), regexp_replace(), replace(), replicate(), 
-  reverse(), rightstr(), rtrim(), startswith(), substr(), trim(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), leftstr(), 
+  length(), lower(), ltrim(), printf(), regexp_match(), regexp_replace(), 
+  replace(), replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Example
 #1 ;SELECT instr('abc', 'b')
    
@@ -1351,9 +1374,10 @@ Parameters
   str   The string to return subset.
   N     The number of characters from the left side of the string to return.
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), length(), lower(), 
-  ltrim(), printf(), regexp_match(), regexp_replace(), replace(), replicate(), 
-  reverse(), rightstr(), rtrim(), startswith(), substr(), trim(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  length(), lower(), ltrim(), printf(), regexp_match(), regexp_replace(), 
+  replace(), replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Examples
 #1 ;SELECT leftstr('abc', 1)
    
@@ -1368,9 +1392,10 @@ Synopsis
 Parameter
   str   The string to determine the length of
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), lower(), 
-  ltrim(), printf(), regexp_match(), regexp_replace(), replace(), replicate(), 
-  reverse(), rightstr(), rtrim(), startswith(), substr(), trim(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), lower(), ltrim(), printf(), regexp_match(), regexp_replace(), 
+  replace(), replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Example
 #1 ;SELECT length('abc')
    
@@ -1430,9 +1455,10 @@ Synopsis
 Parameter
   str   The string to convert.
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  ltrim(), printf(), regexp_match(), regexp_replace(), replace(), replicate(), 
-  reverse(), rightstr(), rtrim(), startswith(), substr(), trim(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), ltrim(), printf(), regexp_match(), regexp_replace(), 
+  replace(), replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Example
 #1 ;SELECT lower('AbC')
    
@@ -1446,9 +1472,10 @@ Parameters
   str     The string to trim characters from the left side
   chars   The characters to trim.  Defaults to spaces.
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  lower(), printf(), regexp_match(), regexp_replace(), replace(), replicate(), 
-  reverse(), rightstr(), rtrim(), startswith(), substr(), trim(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), printf(), regexp_match(), regexp_replace(), 
+  replace(), replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Examples
 #1 ;SELECT ltrim('   abc')
    
@@ -1512,9 +1539,10 @@ Parameters
   format   The format of the string to return.
   X        The argument to substitute at a given position in the format.
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  lower(), ltrim(), regexp_match(), regexp_replace(), replace(), replicate(), 
-  reverse(), rightstr(), rtrim(), startswith(), substr(), trim(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), ltrim(), regexp_match(), regexp_replace(), 
+  replace(), replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Examples
 #1 ;SELECT printf('Hello, %s!', 'World')
    
@@ -1580,10 +1608,10 @@ Parameters
   re    The regular expression to use
   str   The string to test against the regular expression
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  lower(), ltrim(), printf(), regexp_replace(), regexp_replace(), replace(), 
-  replicate(), reverse(), rightstr(), rtrim(), startswith(), substr(), trim(), 
-  upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), ltrim(), printf(), regexp_replace(), 
+  regexp_replace(), replace(), replicate(), reverse(), rightstr(), rtrim(), 
+  spooky_hash(), startswith(), substr(), trim(), upper()
 Examples
 #1 ;SELECT regexp_match('(\d+)', '123')
    
@@ -1604,10 +1632,10 @@ Parameters
   repl   The replacement string.  You can reference capture groups with a 
          backslash followed by the number of the group, starting with 1.
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  lower(), ltrim(), printf(), regexp_match(), regexp_match(), replace(), 
-  replicate(), reverse(), rightstr(), rtrim(), startswith(), substr(), trim(), 
-  upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), ltrim(), printf(), regexp_match(), regexp_match(), 
+  replace(), replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Examples
 #1 ;SELECT regexp_replace('Hello, World!', '^(\w+)', 'Goodbye')
    
@@ -1625,9 +1653,10 @@ Parameters
   old           The string to be replaced.
   replacement   The string to replace any occurrences of the old string with.
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  lower(), ltrim(), printf(), regexp_match(), regexp_replace(), replicate(), 
-  reverse(), rightstr(), rtrim(), startswith(), substr(), trim(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), ltrim(), printf(), regexp_match(), 
+  regexp_replace(), replicate(), reverse(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Examples
 #1 ;SELECT replace('abc', 'x', 'z')
    
@@ -1642,9 +1671,10 @@ Parameters
   str   The string to replicate.
   N     The number of times to replicate the string.
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  lower(), ltrim(), printf(), regexp_match(), regexp_replace(), replace(), 
-  reverse(), rightstr(), rtrim(), startswith(), substr(), trim(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), ltrim(), printf(), regexp_match(), 
+  regexp_replace(), replace(), reverse(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Example
 #1 ;SELECT replicate('abc', 3)
    
@@ -1655,9 +1685,10 @@ Synopsis
 Parameter
   str   The string to reverse.
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  lower(), ltrim(), printf(), regexp_match(), regexp_replace(), replace(), 
-  replicate(), rightstr(), rtrim(), startswith(), substr(), trim(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), ltrim(), printf(), regexp_match(), 
+  regexp_replace(), replace(), replicate(), rightstr(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Example
 #1 ;SELECT reverse('abc')
    
@@ -1670,9 +1701,10 @@ Parameters
   str   The string to return subset.
   N     The number of characters from the right side of the string to return.
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  lower(), ltrim(), printf(), regexp_match(), regexp_replace(), replace(), 
-  replicate(), reverse(), rtrim(), startswith(), substr(), trim(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), ltrim(), printf(), regexp_match(), 
+  regexp_replace(), replace(), replicate(), reverse(), rtrim(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Examples
 #1 ;SELECT rightstr('abc', 1)
    
@@ -1708,9 +1740,10 @@ Parameters
   str     The string to trim characters from the right side
   chars   The characters to trim.  Defaults to spaces.
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  lower(), ltrim(), printf(), regexp_match(), regexp_replace(), replace(), 
-  replicate(), reverse(), rightstr(), startswith(), substr(), trim(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), ltrim(), printf(), regexp_match(), 
+  regexp_replace(), replace(), replicate(), reverse(), rightstr(), spooky_hash(), 
+  startswith(), substr(), trim(), upper()
 Examples
 #1 ;SELECT ltrim('abc   ')
    
@@ -1719,6 +1752,29 @@ Examples
    
 
 
+Synopsis
+  spooky_hash(str, ...) -- Compute the hash value for the given arguments.
+Parameter
+  str   The string to hash
+See Also
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), ltrim(), printf(), regexp_match(), 
+  regexp_replace(), replace(), replicate(), reverse(), rightstr(), rtrim(), 
+  startswith(), substr(), trim(), upper()
+Examples
+#1 ;SELECT spooky_hash('Hello, World!')
+   
+
+#2 ;SELECT spooky_hash('Hello, World!', NULL)
+   
+
+#3 ;SELECT spooky_hash('Hello, World!', '')
+   
+
+#4 ;SELECT spooky_hash('Hello, World!', 123)
+   
+
+
 Synopsis
   sqlite_compileoption_get(N) -- Returns the N-th compile-time option used to 
     build SQLite or NULL if N is out of range.
@@ -1753,9 +1809,10 @@ Parameters
   str      The string to test
   prefix   The prefix to check in the string
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  lower(), ltrim(), printf(), regexp_match(), regexp_replace(), replace(), 
-  replicate(), reverse(), rightstr(), rtrim(), substr(), trim(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), ltrim(), printf(), regexp_match(), 
+  regexp_replace(), replace(), replicate(), reverse(), rightstr(), rtrim(), 
+  spooky_hash(), substr(), trim(), upper()
 Examples
 #1 ;SELECT startswith('foobar', 'foo')
    
@@ -1797,9 +1854,10 @@ Parameters
           the end of the string are returned.  If the value is negative, then 
           the characters before the start are returned.
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  lower(), ltrim(), printf(), regexp_match(), regexp_replace(), replace(), 
-  replicate(), reverse(), rightstr(), rtrim(), startswith(), trim(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), ltrim(), printf(), regexp_match(), 
+  regexp_replace(), replace(), replicate(), reverse(), rightstr(), rtrim(), 
+  spooky_hash(), startswith(), trim(), upper()
 Examples
 #1 ;SELECT substr('abc', 2)
    
@@ -1900,9 +1958,10 @@ Parameters
   str     The string to trim characters from the left and right sides.
   chars   The characters to trim.  Defaults to spaces.
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  lower(), ltrim(), printf(), regexp_match(), regexp_replace(), replace(), 
-  replicate(), reverse(), rightstr(), rtrim(), startswith(), substr(), upper()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), ltrim(), printf(), regexp_match(), 
+  regexp_replace(), replace(), replicate(), reverse(), rightstr(), rtrim(), 
+  spooky_hash(), startswith(), substr(), upper()
 Examples
 #1 ;SELECT trim('    abc   ')
    
@@ -1948,9 +2007,10 @@ Synopsis
 Parameter
   str   The string to convert.
 See Also
-  char(), endswith(), extract(), group_concat(), instr(), leftstr(), length(), 
-  lower(), ltrim(), printf(), regexp_match(), regexp_replace(), replace(), 
-  replicate(), reverse(), rightstr(), rtrim(), startswith(), substr(), trim()
+  char(), endswith(), extract(), group_concat(), group_spooky_hash(), instr(), 
+  leftstr(), length(), lower(), ltrim(), printf(), regexp_match(), 
+  regexp_replace(), replace(), replicate(), reverse(), rightstr(), rtrim(), 
+  spooky_hash(), startswith(), substr(), trim()
 Example
 #1 ;SELECT upper('aBc')