From 6f4791ec35063bedd6467c3851f2238c84e0f28b Mon Sep 17 00:00:00 2001 From: Timothy Stack Date: Tue, 23 Mar 2021 22:23:08 -0700 Subject: [PATCH] [sqlite] include generate_series() extension --- NEWS | 3 + aminclude_static.am | 2 +- src/CMakeLists.txt | 3 + src/Makefile.am | 5 + src/internals/sql-ref.rst | 51 ++++ src/sqlite-extension-func.cc | 38 ++- src/third-party/sqlite/ext/series.c | 448 ++++++++++++++++++++++++++++ test/expected_help.txt | 24 ++ test/test_sql.sh | 16 + 9 files changed, 588 insertions(+), 2 deletions(-) create mode 100644 src/third-party/sqlite/ext/series.c diff --git a/NEWS b/NEWS index 8a0f903f..1a96da0f 100644 --- a/NEWS +++ b/NEWS @@ -50,6 +50,9 @@ lnav v0.9.1: functions that operate on XML/JSON data, like xpath() or json_tree(). * Added an "lnav_top_view" SQL VIEW that returns the row for the top view in the lnav_views table. + * The "generate_series()" SQLite extension is now included by default. + One change from the standard implementation is that both the start and + stop are required parameters. Interface Changes: * When copying log lines, the file name and time offset will be included diff --git a/aminclude_static.am b/aminclude_static.am index 6a928aca..209c8bb5 100644 --- a/aminclude_static.am +++ b/aminclude_static.am @@ -1,6 +1,6 @@ # aminclude_static.am generated automatically by Autoconf -# from AX_AM_MACROS_STATIC on Tue Mar 23 09:31:31 PDT 2021 +# from AX_AM_MACROS_STATIC on Tue Mar 23 22:11:31 PDT 2021 # Code coverage diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9b6f005b..9757c845 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -357,6 +357,8 @@ add_library(diag STATIC yajl/yajl_version.c spookyhash/SpookyV2.cpp + third-party/sqlite/ext/series.c + all_logs_vtab.hh archive_manager.hh archive_manager.cfg.hh @@ -511,6 +513,7 @@ target_include_directories( ) target_link_libraries(diag ${lnav_LIBS}) +target_compile_definitions(diag PRIVATE SQLITE_OMIT_LOAD_EXTENSION) check_library_exists(util openpty "" HAVE_LIBUTIL) diff --git a/src/Makefile.am b/src/Makefile.am index 8a4ca63f..f7df5d17 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -115,6 +115,7 @@ AM_LDFLAGS = \ AM_CPPFLAGS = \ -DSYSCONFDIR='"$(sysconfdir)"' \ + -DSQLITE_OMIT_LOAD_EXTENSION \ -I$(srcdir)/fmtlib \ -Wall \ $(CODE_COVERAGE_CPPFLAGS) \ @@ -291,7 +292,11 @@ noinst_HEADERS = \ nodist_libdiag_a_SOURCES = \ $(LNAV_BUILT_FILES) +THIRD_PARTY_SRCS = \ + third-party/sqlite/ext/series.c + libdiag_a_SOURCES = \ + $(THIRD_PARTY_SRCS) \ all_logs_vtab.cc \ ansi_scrubber.cc \ archive_manager.cc \ diff --git a/src/internals/sql-ref.rst b/src/internals/sql-ref.rst index 3db7f70b..8467fd4e 100644 --- a/src/internals/sql-ref.rst +++ b/src/internals/sql-ref.rst @@ -1168,6 +1168,57 @@ floor(*num*) ---- +.. _generate_series: + +generate_series(*start*, *stop*, *\[step\]*) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + A table-valued-function that returns the whole numbers between a lower and upper bound, inclusive + + **Parameters** + * **start\*** --- The starting point of the series + * **stop\*** --- The stopping point of the series + * **step** --- The increment between each value + + **Examples** + To generate the numbers in the range [10, 14]: + + .. code-block:: custsqlite + + ;SELECT value FROM generate_series(10, 14) + value + 10 + 11 + 12 + 13 + 14 + + To generate every other number in the range [10, 14]: + + .. code-block:: custsqlite + + ;SELECT value FROM generate_series(10, 14, 2) + value + 10 + 12 + 14 + + To count down from five to 1: + + .. code-block:: custsqlite + + ;SELECT value FROM generate_series(1, 5, -1) + value + 5 + 4 + 3 + 2 + 1 + + +---- + + .. _gethostbyaddr: gethostbyaddr(*hostname*) diff --git a/src/sqlite-extension-func.cc b/src/sqlite-extension-func.cc index c438fb12..fdcc1139 100644 --- a/src/sqlite-extension-func.cc +++ b/src/sqlite-extension-func.cc @@ -35,12 +35,21 @@ #include "base/string_util.hh" #include "base/lnav_log.hh" +#include "auto_mem.hh" #include "sql_util.hh" #include "sqlite-extension-func.hh" using namespace std; +extern "C" { +int sqlite3_series_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +); +} + sqlite_registration_func_t sqlite_registration_funcs[] = { common_extension_functions, state_extension_functions, @@ -59,7 +68,13 @@ int register_sqlite_funcs(sqlite3 *db, sqlite_registration_func_t *reg_funcs) assert(db != nullptr); assert(reg_funcs != nullptr); - + + { + auto_mem errmsg(sqlite3_free); + + sqlite3_series_init(db, errmsg.out(), nullptr); + } + for (lpc = 0; reg_funcs[lpc]; lpc++) { struct FuncDef *basic_funcs = nullptr; struct FuncDefAgg *agg_funcs = nullptr; @@ -679,6 +694,27 @@ int register_sqlite_funcs(sqlite3 *db, sqlite_registration_func_t *reg_funcs) "SELECT total(ex_duration) FROM lnav_example_log" }), + help_text("generate_series", + "A table-valued-function that returns the whole numbers " + "between a lower and upper bound, inclusive") + .sql_table_valued_function() + .with_parameter({"start", "The starting point of the series"}) + .with_parameter({"stop", "The stopping point of the series"}) + .with_parameter(help_text("step", "The increment between each value") + .optional()) + .with_result({"value", "The number in the series"}) + .with_example({ + "To generate the numbers in the range [10, 14]", + "SELECT value FROM generate_series(10, 14)" + }) + .with_example({ + "To generate every other number in the range [10, 14]", + "SELECT value FROM generate_series(10, 14, 2)" + }) + .with_example({ + "To count down from five to 1", + "SELECT value FROM generate_series(1, 5, -1)" + }) }; for (auto &ht : builtin_funcs) { diff --git a/src/third-party/sqlite/ext/series.c b/src/third-party/sqlite/ext/series.c new file mode 100644 index 00000000..4b34f842 --- /dev/null +++ b/src/third-party/sqlite/ext/series.c @@ -0,0 +1,448 @@ +/* +** 2015-08-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file demonstrates how to create a table-valued-function using +** a virtual table. This demo implements the generate_series() function +** which gives similar results to the eponymous function in PostgreSQL. +** Examples: +** +** SELECT * FROM generate_series(0,100,5); +** +** The query above returns integers from 0 through 100 counting by steps +** of 5. +** +** SELECT * FROM generate_series(0,100); +** +** Integers from 0 through 100 with a step size of 1. +** +** SELECT * FROM generate_series(20) LIMIT 10; +** +** Integers 20 through 29. +** +** HOW IT WORKS +** +** The generate_series "function" is really a virtual table with the +** following schema: +** +** CREATE TABLE generate_series( +** value, +** start HIDDEN, +** stop HIDDEN, +** step HIDDEN +** ); +** +** Function arguments in queries against this virtual table are translated +** into equality constraints against successive hidden columns. In other +** words, the following pairs of queries are equivalent to each other: +** +** SELECT * FROM generate_series(0,100,5); +** SELECT * FROM generate_series WHERE start=0 AND stop=100 AND step=5; +** +** SELECT * FROM generate_series(0,100); +** SELECT * FROM generate_series WHERE start=0 AND stop=100; +** +** SELECT * FROM generate_series(20) LIMIT 10; +** SELECT * FROM generate_series WHERE start=20 LIMIT 10; +** +** The generate_series virtual table implementation leaves the xCreate method +** set to NULL. This means that it is not possible to do a CREATE VIRTUAL +** TABLE command with "generate_series" as the USING argument. Instead, there +** is a single generate_series virtual table that is always available without +** having to be created first. +** +** The xBestIndex method looks for equality constraints against the hidden +** start, stop, and step columns, and if present, it uses those constraints +** to bound the sequence of generated values. If the equality constraints +** are missing, it uses 0 for start, 4294967295 for stop, and 1 for step. +** xBestIndex returns a small cost when both start and stop are available, +** and a very large cost if either start or stop are unavailable. This +** encourages the query planner to order joins such that the bounds of the +** series are well-defined. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + + +/* series_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct series_cursor series_cursor; +struct series_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + int isDesc; /* True to count down rather than up */ + sqlite3_int64 iRowid; /* The rowid */ + sqlite3_int64 iValue; /* Current value ("value") */ + sqlite3_int64 mnValue; /* Mimimum value ("start") */ + sqlite3_int64 mxValue; /* Maximum value ("stop") */ + sqlite3_int64 iStep; /* Increment ("step") */ +}; + +/* +** The seriesConnect() method is invoked to create a new +** series_vtab that describes the generate_series virtual table. +** +** Think of this routine as the constructor for series_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the series_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against generate_series will look like. +*/ +static int seriesConnect( + sqlite3 *db, + void *pUnused, + int argcUnused, const char *const*argvUnused, + sqlite3_vtab **ppVtab, + char **pzErrUnused +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define SERIES_COLUMN_VALUE 0 +#define SERIES_COLUMN_START 1 +#define SERIES_COLUMN_STOP 2 +#define SERIES_COLUMN_STEP 3 + + (void)pUnused; + (void)argcUnused; + (void)argvUnused; + (void)pzErrUnused; + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(value,start hidden,stop hidden,step hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); +#ifdef SQLITE_VTAB_INNOCUOUS + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); +#endif + } + return rc; +} + +/* +** This method is the destructor for series_cursor objects. +*/ +static int seriesDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new series_cursor object. +*/ +static int seriesOpen(sqlite3_vtab *pUnused, sqlite3_vtab_cursor **ppCursor){ + series_cursor *pCur; + (void)pUnused; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a series_cursor. +*/ +static int seriesClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a series_cursor to its next row of output. +*/ +static int seriesNext(sqlite3_vtab_cursor *cur){ + series_cursor *pCur = (series_cursor*)cur; + if( pCur->isDesc ){ + pCur->iValue -= pCur->iStep; + }else{ + pCur->iValue += pCur->iStep; + } + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int seriesColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + series_cursor *pCur = (series_cursor*)cur; + sqlite3_int64 x = 0; + switch( i ){ + case SERIES_COLUMN_START: x = pCur->mnValue; break; + case SERIES_COLUMN_STOP: x = pCur->mxValue; break; + case SERIES_COLUMN_STEP: x = pCur->iStep; break; + default: x = pCur->iValue; break; + } + sqlite3_result_int64(ctx, x); + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** first row returned is assigned rowid value 1, and each subsequent +** row a value 1 more than that of the previous. +*/ +static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + series_cursor *pCur = (series_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int seriesEof(sqlite3_vtab_cursor *cur){ + series_cursor *pCur = (series_cursor*)cur; + if( pCur->isDesc ){ + return pCur->iValue < pCur->mnValue; + }else{ + return pCur->iValue > pCur->mxValue; + } +} + +/* True to cause run-time checking of the start=, stop=, and/or step= +** parameters. The only reason to do this is for testing the +** constraint checking logic for virtual tables in the SQLite core. +*/ +#ifndef SQLITE_SERIES_CONSTRAINT_VERIFY +# define SQLITE_SERIES_CONSTRAINT_VERIFY 0 +#endif + +/* +** This method is called to "rewind" the series_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to seriesColumn() or seriesRowid() or +** seriesEof(). +** +** The query plan selected by seriesBestIndex is passed in the idxNum +** parameter. (idxStr is not used in this implementation.) idxNum +** is a bitmask showing which constraints are available: +** +** 1: start=VALUE +** 2: stop=VALUE +** 4: step=VALUE +** +** Also, if bit 8 is set, that means that the series should be output +** in descending order rather than in ascending order. If bit 16 is +** set, then output must appear in ascending order. +** +** This routine should initialize the cursor and position it so that it +** is pointing at the first row, or pointing off the end of the table +** (so that seriesEof() will return true) if the table is empty. +*/ +static int seriesFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStrUnused, + int argc, sqlite3_value **argv +){ + series_cursor *pCur = (series_cursor *)pVtabCursor; + int i = 0; + (void)idxStrUnused; + if( idxNum & 1 ){ + pCur->mnValue = sqlite3_value_int64(argv[i++]); + }else{ + pCur->mnValue = 0; + } + if( idxNum & 2 ){ + pCur->mxValue = sqlite3_value_int64(argv[i++]); + }else{ + pCur->mxValue = 0xffffffff; + } + if( idxNum & 4 ){ + pCur->iStep = sqlite3_value_int64(argv[i++]); + if( pCur->iStep==0 ){ + pCur->iStep = 1; + }else if( pCur->iStep<0 ){ + pCur->iStep = -pCur->iStep; + if( (idxNum & 16)==0 ) idxNum |= 8; + } + }else{ + pCur->iStep = 1; + } + for(i=0; imnValue = 1; + pCur->mxValue = 0; + break; + } + } + if( idxNum & 8 ){ + pCur->isDesc = 1; + pCur->iValue = pCur->mxValue; + if( pCur->iStep>0 ){ + pCur->iValue -= (pCur->mxValue - pCur->mnValue)%pCur->iStep; + } + }else{ + pCur->isDesc = 0; + pCur->iValue = pCur->mnValue; + } + pCur->iRowid = 1; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the generate_series virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** The query plan is represented by bits in idxNum: +** +** (1) start = $value -- constraint exists +** (2) stop = $value -- constraint exists +** (4) step = $value -- constraint exists +** (8) output in descending order +*/ +static int seriesBestIndex( + sqlite3_vtab *vtab, + sqlite3_index_info *pIdxInfo +){ + int i, j; /* Loop over constraints */ + int idxNum = 0; /* The query plan bitmask */ + int unusableMask = 0; /* Mask of unusable constraints */ + int nArg = 0; /* Number of arguments that seriesFilter() expects */ + int aIdx[3]; /* Constraints on start, stop, and step */ + const struct sqlite3_index_constraint *pConstraint; + + /* This implementation assumes that the start, stop, and step columns + ** are the last three columns in the virtual table. */ + assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 ); + assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 ); + aIdx[0] = aIdx[1] = aIdx[2] = -1; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + int iCol; /* 0 for start, 1 for stop, 2 for step */ + int iMask; /* bitmask for those column */ + if( pConstraint->iColumniColumn - SERIES_COLUMN_START; + assert( iCol>=0 && iCol<=2 ); + iMask = 1 << iCol; + if( pConstraint->usable==0 ){ + unusableMask |= iMask; + continue; + }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + idxNum |= iMask; + aIdx[iCol] = i; + } + } + for(i=0; i<3; i++){ + if( (j = aIdx[i])>=0 ){ + pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[j].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY; + } + } + if( (unusableMask & ~idxNum)!=0 ){ + /* The start, stop, and step columns are inputs. Therefore if there + ** are unusable constraints on any of start, stop, or step then + ** this plan is unusable */ + return SQLITE_CONSTRAINT; + } + if( (idxNum & 3)==3 ){ + /* Both start= and stop= boundaries are available. This is the + ** the preferred case */ + pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0)); + pIdxInfo->estimatedRows = 1000; + if( pIdxInfo->nOrderBy==1 ){ + if( pIdxInfo->aOrderBy[0].desc ){ + idxNum |= 8; + }else{ + idxNum |= 16; + } + pIdxInfo->orderByConsumed = 1; + } + }else{ + if (!(idxNum & 1)) { + vtab->zErrMsg = sqlite3_mprintf("the start parameter is required"); + } else { + vtab->zErrMsg = sqlite3_mprintf("the stop parameter is required"); + } + return SQLITE_ERROR; + } + pIdxInfo->idxNum = idxNum; + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** generate_series virtual table. +*/ +static sqlite3_module seriesModule = { + 0, /* iVersion */ + 0, /* xCreate */ + seriesConnect, /* xConnect */ + seriesBestIndex, /* xBestIndex */ + seriesDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + seriesOpen, /* xOpen - open a cursor */ + seriesClose, /* xClose - close a cursor */ + seriesFilter, /* xFilter - configure scan constraints */ + seriesNext, /* xNext - advance a cursor */ + seriesEof, /* xEof - check for end of scan */ + seriesColumn, /* xColumn - read data */ + seriesRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_series_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3_libversion_number()<3008012 ){ + *pzErrMsg = sqlite3_mprintf( + "generate_series() requires SQLite 3.8.12 or later"); + return SQLITE_ERROR; + } + rc = sqlite3_create_module(db, "generate_series", &seriesModule, 0); +#endif + return rc; +} diff --git a/test/expected_help.txt b/test/expected_help.txt index e92700fe..85b13fc9 100644 --- a/test/expected_help.txt +++ b/test/expected_help.txt @@ -1479,6 +1479,30 @@ Example +Synopsis + generate_series(start, stop, [step]) -- A table-valued-function that returns + the whole numbers between a lower and upper bound, inclusive +Parameters + start The starting point of the series + stop The stopping point of the series + step The increment between each value +Result + value The number in the series + +Examples +#1 To generate the numbers in the range [10, 14]: + ;SELECT value FROM generate_series(10, 14) + + +#2 To generate every other number in the range [10, 14]: + ;SELECT value FROM generate_series(10, 14, 2) + + +#3 To count down from five to 1: + ;SELECT value FROM generate_series(1, 5, -1) + + + Synopsis gethostbyaddr(hostname) -- Get the hostname for the given IP address Parameter diff --git a/test/test_sql.sh b/test/test_sql.sh index df69eae1..0acb7b09 100644 --- a/test/test_sql.sh +++ b/test/test_sql.sh @@ -76,6 +76,22 @@ check_output "w3c headers are not captured?" <