Archives
/
lnav
mirror of https://github.com/tstack/lnav
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[sql] change extract() to regexp_match() and make extract() an interface to the data_parser

Browse Source
pull/306/merge
Timothy Stack 8 years ago
parent 0f67f8d1f0
commit 85eee6514b
16 changed files with 2482 additions and 1911 deletions
Show Stats Download Patch File Download Diff File

6
NEWS
Unescape Escape View File

@ -9,8 +9,10 @@ lnav v0.8.1:
object or array, respectively.
* The SQL view will now graph values found in JSON objects/arrays in
addition to the regular columns in the result.
* Added an 'extract(<re>, <str>)' SQL function that can be used to
* Added an 'regexp_match(<re>, <str>)' SQL function that can be used to
extract values from a string using a regular expression.
* Added an 'extract(<str>)' SQL function that extracts values using the
same data discover/extraction parser used in the 'logline' table.
* Pressing 'V' in the DB view will now check for a column with a
timestamp and move to the corresponding time in the log view.
* Added ":hide-lines-before", ":hide-lines-after", and
@ -23,6 +25,8 @@ lnav v0.8.1:
* Added an ":echo" command that can be useful for scripts to message
the user.
* The "log_part" column can now be set with an SQL UPDATE statement.
* Added a "log_body" hidden column that returns the body of the log
message.
* Added ":config", ":reset-config", and ":save-config" commands to change
configuration options, reset to default, and save them for future
executions.

15
docs/source/sqlext.rst
Unescape Escape View File

@ -119,12 +119,15 @@ Additional string comparison and manipulation functions:
starts with the given prefix.
* endswith(s1,suffix) - Given a string and suffix, return 1 if the string ends
with the given suffix.
* extract(re,str) - Extract values from a string using a regular expression.
The "re" argument should be a PCRE with captures. If there is a single
capture, that captured value will be directly returned. If there is more
than one capture, a JSON object will be returned with field names matching
the named capture groups or 'col_N' where 'N' is the index of the capture.
If the expression does not match the string, NULL is returned.
* regexp_match(re,str) - Match and extract values from a string using a regular
expression. The "re" argument should be a PCRE with captures. If there is
a single capture, that captured value will be directly returned. If there
is more than one capture, a JSON object will be returned with field names
matching the named capture groups or 'col_N' where 'N' is the index of the
capture. If the expression does not match the string, NULL is returned.
* extract(str) - Parse and extract values from a string using the same
algorithm as the *logline* table (see :ref:`data-ext`). The discovered
data is returned as a JSON-object that you can do further processing on.
File Paths

206
src/data_parser.hh
Unescape Escape View File

@ -35,6 +35,7 @@
#include "spookyhash/SpookyV2.h"
#include <list>
#include <stack>
#include <vector>
#include <iterator>
#include <algorithm>
@ -75,17 +76,23 @@ enum data_format_state_t {
};
struct data_format {
data_format(const char *name,
data_format(const char *name = NULL,
data_token_t appender = DT_INVALID,
data_token_t terminator = DT_INVALID)
: df_name(name),
df_appender(appender),
df_terminator(terminator)
df_terminator(terminator),
df_qualifier(DT_INVALID),
df_separator(DT_COLON),
df_prefix_terminator(DT_INVALID)
{};
const char * df_name;
const data_token_t df_appender;
const data_token_t df_terminator;
data_token_t df_appender;
data_token_t df_terminator;
data_token_t df_qualifier;
data_token_t df_separator;
data_token_t df_prefix_terminator;
};
data_format_state_t dfs_prefix_next(data_format_state_t state,
@ -197,6 +204,10 @@ public:
LIST_INIT_TRACE;
};
element_list_t(const element_list_t &other) : std::list<element>(other) {
this->el_format = other.el_format;
}
~element_list_t()
{
const char *fn = __FILE__;
@ -251,6 +262,8 @@ public:
this->std::list<element>::splice(pos, other, first, last);
}
data_format el_format;
};
struct element {
@ -302,6 +315,7 @@ public:
if (this->e_sub_elements == NULL) {
this->e_sub_elements = new element_list_t("_sub_", __FILE__,
__LINE__);
this->e_sub_elements->el_format = subs.el_format;
}
this->e_sub_elements->swap(subs);
this->update_capture();
@ -333,6 +347,9 @@ public:
this->e_sub_elements->size() == 1) {
retval = this->e_sub_elements->front().e_token;
}
else {
retval = DT_SYMBOL;
}
}
else {
retval = this->e_token;
@ -410,13 +427,72 @@ private:
data_token_t ei_token;
};
struct discover_format_state {
discover_format_state() {
memset(this->dfs_hist, 0, sizeof(this->dfs_hist));
this->dfs_prefix_state = DFS_INIT;
this->dfs_semi_state = DFS_INIT;
this->dfs_comma_state = DFS_INIT;
}
void update_for_element(const element &elem) {
this->dfs_prefix_state = dfs_prefix_next(this->dfs_prefix_state, elem.e_token);
this->dfs_semi_state = dfs_semi_next(this->dfs_semi_state, elem.e_token);
this->dfs_comma_state = dfs_comma_next(this->dfs_comma_state, elem.e_token);
if (this->dfs_prefix_state != DFS_ERROR) {
if (this->dfs_semi_state == DFS_ERROR) {
this->dfs_semi_state = DFS_INIT;
}
if (this->dfs_comma_state == DFS_ERROR) {
this->dfs_comma_state = DFS_INIT;
}
}
this->dfs_hist[elem.e_token] += 1;
}
void finalize() {
data_token_t qualifier = this->dfs_format.df_qualifier;
data_token_t separator = this->dfs_format.df_separator;
data_token_t prefix_term = this->dfs_format.df_prefix_terminator;
this->dfs_format = FORMAT_PLAIN;
if (this->dfs_hist[DT_EQUALS]) {
qualifier = DT_COLON;
separator = DT_EQUALS;
}
if (this->dfs_semi_state != DFS_ERROR && this->dfs_hist[DT_SEMI]) {
this->dfs_format = FORMAT_SEMI;
}
else if (this->dfs_comma_state != DFS_ERROR) {
this->dfs_format = FORMAT_COMMA;
if (separator == DT_COLON && this->dfs_hist[DT_COMMA] > 0) {
if (!((this->dfs_hist[DT_COLON] == this->dfs_hist[DT_COMMA]) ||
((this->dfs_hist[DT_COLON] - 1) == this->dfs_hist[DT_COMMA]))) {
separator = DT_INVALID;
if (this->dfs_hist[DT_COLON] == 1) {
prefix_term = DT_COLON;
}
}
}
}
this->dfs_format.df_qualifier = qualifier;
this->dfs_format.df_separator = separator;
this->dfs_format.df_prefix_terminator = prefix_term;
};
data_format_state_t dfs_prefix_state;
data_format_state_t dfs_semi_state;
data_format_state_t dfs_comma_state;
int dfs_hist[DT_TERMINAL_MAX];
data_format dfs_format;
};
data_parser(data_scanner *ds)
: dp_errors("dp_errors", __FILE__, __LINE__),
dp_pairs("dp_pairs", __FILE__, __LINE__),
dp_format(NULL),
dp_qualifier(DT_INVALID),
dp_separator(DT_INVALID),
dp_prefix_terminator(DT_INVALID),
dp_msg_format(NULL),
dp_msg_format_begin(ds->get_input().pi_offset),
dp_scanner(ds)
@ -434,6 +510,8 @@ private:
ELEMENT_LIST_T(prefix);
SpookyHash context;
require(in_list.el_format.df_name != NULL);
POINT_TRACE("pairup_start");
for (element_list_t::iterator iter = in_list.begin();
@ -448,20 +526,20 @@ private:
}
}
if (this->dp_prefix_terminator != DT_INVALID) {
if (iter->e_token == this->dp_prefix_terminator) {
this->dp_prefix_terminator = DT_INVALID;
if (in_list.el_format.df_prefix_terminator != DT_INVALID) {
if (iter->e_token == in_list.el_format.df_prefix_terminator) {
in_list.el_format.df_prefix_terminator = DT_INVALID;
}
else {
el_stack.PUSH_BACK(*iter);
}
}
else if (iter->e_token == this->dp_format->df_terminator) {
this->end_of_value(el_stack, key_comps, value);
else if (iter->e_token == in_list.el_format.df_terminator) {
this->end_of_value(el_stack, key_comps, value, in_list);
key_comps.PUSH_BACK(*iter);
}
else if (iter->e_token == this->dp_qualifier) {
else if (iter->e_token == in_list.el_format.df_qualifier) {
value.SPLICE(value.end(),
key_comps,
key_comps.begin(),
@ -471,13 +549,13 @@ private:
el_stack.PUSH_BACK(element(value, DNT_VALUE));
}
}
else if (iter->e_token == this->dp_separator) {
else if (iter->e_token == in_list.el_format.df_separator) {
element_list_t::iterator key_iter = key_comps.end();
bool found = false, key_is_values = true;
do {
--key_iter;
if (key_iter->e_token == this->dp_format->df_appender) {
if (key_iter->e_token == in_list.el_format.df_appender) {
++key_iter;
value.SPLICE(value.end(),
key_comps,
@ -487,7 +565,7 @@ private:
found = true;
}
else if (key_iter->e_token ==
this->dp_format->df_terminator) {
in_list.el_format.df_terminator) {
std::vector<element> key_copy;
value.SPLICE(value.end(),
@ -512,8 +590,8 @@ private:
element_list_t::iterator value_iter;
if (el_stack.size() > 1 &&
this->dp_format->df_appender != DT_INVALID &&
this->dp_format->df_terminator != DT_INVALID) {
in_list.el_format.df_appender != DT_INVALID &&
in_list.el_format.df_terminator != DT_INVALID) {
/* If we're expecting a terminator and haven't found it */
/* then this is part of the value. */
continue;
@ -563,7 +641,7 @@ private:
key_comps, key_comps.begin(), key_comps.end());
}
else {
this->end_of_value(el_stack, key_comps, value);
this->end_of_value(el_stack, key_comps, value, in_list);
}
POINT_TRACE("pairup_stack");
@ -785,20 +863,13 @@ private:
void discover_format(void)
{
pcre_context_static<30> pc;
int hist[DT_TERMINAL_MAX];
std::stack<discover_format_state> state_stack;
struct element elem;
this->dp_group_token.push_back(DT_INVALID);
this->dp_group_stack.resize(1);
this->dp_qualifier = DT_INVALID;
this->dp_separator = DT_COLON;
this->dp_prefix_terminator = DT_INVALID;
data_format_state_t prefix_state = DFS_INIT;
data_format_state_t semi_state = DFS_INIT;
data_format_state_t comma_state = DFS_INIT;
memset(hist, 0, sizeof(hist));
state_stack.push(discover_format_state());
while (this->dp_scanner->tokenize2(pc, elem.e_token)) {
pcre_context::iterator pc_iter;
@ -810,18 +881,7 @@ private:
require(elem.e_capture.c_begin != -1);
require(elem.e_capture.c_end != -1);
prefix_state = dfs_prefix_next(prefix_state, elem.e_token);
semi_state = dfs_semi_next(semi_state, elem.e_token);
comma_state = dfs_comma_next(comma_state, elem.e_token);
if (prefix_state != DFS_ERROR) {
if (semi_state == DFS_ERROR) {
semi_state = DFS_INIT;
}
if (comma_state == DFS_ERROR) {
comma_state = DFS_INIT;
}
}
hist[elem.e_token] += 1;
state_stack.top().update_for_element(elem);
switch (elem.e_token) {
case DT_LPAREN:
case DT_LANGLE:
@ -831,6 +891,7 @@ private:
this->dp_group_stack.push_back(element_list_t("_anon_",
__FILE__,
__LINE__));
state_stack.push(discover_format_state());
break;
case DT_RPAREN:
@ -844,6 +905,9 @@ private:
this->dp_group_stack.rbegin();
++riter;
if (!this->dp_group_stack.back().empty()) {
state_stack.top().finalize();
this->dp_group_stack.back().el_format = state_stack.top().dfs_format;
state_stack.pop();
(*riter).PUSH_BACK(element(this->dp_group_stack.back(),
DNT_GROUP));
}
@ -867,43 +931,28 @@ private:
this->dp_group_stack.rbegin();
++riter;
if (!this->dp_group_stack.back().empty()) {
state_stack.top().finalize();
this->dp_group_stack.back().el_format = state_stack.top().dfs_format;
state_stack.pop();
(*riter).PUSH_BACK(element(this->dp_group_stack.back(),
DNT_GROUP));
}
this->dp_group_stack.pop_back();
}
if (hist[DT_EQUALS]) {
this->dp_qualifier = DT_COLON;
this->dp_separator = DT_EQUALS;
}
if (semi_state != DFS_ERROR && hist[DT_SEMI]) {
this->dp_format = &FORMAT_SEMI;
}
else if (comma_state != DFS_ERROR) {
this->dp_format = &FORMAT_COMMA;
if (this->dp_separator == DT_COLON && hist[DT_COMMA] > 0) {
if (!((hist[DT_COLON] == hist[DT_COMMA]) ||
((hist[DT_COLON] - 1) == hist[DT_COMMA]))) {
this->dp_separator = DT_INVALID;
if (hist[DT_COLON] == 1) {
this->dp_prefix_terminator = DT_COLON;
}
}
}
}
else {
this->dp_format = &FORMAT_PLAIN;
}
state_stack.top().finalize();
this->dp_group_stack.back().el_format = state_stack.top().dfs_format;
};
void end_of_value(element_list_t &el_stack,
element_list_t &key_comps,
element_list_t &value) {
key_comps.remove_if(element_if(this->dp_format->df_terminator));
element_list_t &value,
const element_list_t &in_list) {
bool key_added = false;
key_comps.remove_if(element_if(in_list.el_format.df_terminator));
key_comps.remove_if(element_if(DT_COMMA));
value.remove_if(element_if(this->dp_format->df_terminator));
value.remove_if(element_if(in_list.el_format.df_terminator));
value.remove_if(element_if(DT_COMMA));
strip(key_comps, element_if(DT_WHITE));
strip(value, element_if(DT_WHITE));
@ -943,6 +992,7 @@ private:
strip(key_comps, element_if(DT_WHITE));
if (!key_comps.empty()) {
el_stack.PUSH_BACK(element(key_comps, DNT_KEY, false));
key_added = true;
}
key_comps.CLEAR();
}
@ -956,7 +1006,23 @@ private:
strip(value, element_if(DT_COLON));
strip(value, element_if(DT_WHITE));
if (!value.empty()) {
el_stack.PUSH_BACK(element(value, DNT_VALUE));
if (key_added && value.front().e_token == DNT_GROUP) {
element_list_t::iterator end_iter = el_stack.end();
--end_iter;
value.SPLICE(value.begin(),
el_stack,
end_iter,
el_stack.end());
element_list_t ELEMENT_LIST_T(group_pair);
group_pair.PUSH_BACK(element(value, DNT_PAIR));
el_stack.PUSH_BACK(element(group_pair, DNT_VALUE));
}
else {
el_stack.PUSH_BACK(element(value, DNT_VALUE));
}
}
value.CLEAR();
};
@ -1040,10 +1106,6 @@ private:
element_list_t dp_pairs;
schema_id_t dp_schema_id;
data_format * dp_format;
data_token_t dp_qualifier;
data_token_t dp_separator;
data_token_t dp_prefix_terminator;
std::string *dp_msg_format;
int dp_msg_format_begin;

3816
src/data_scanner_re.cc
View File

File diff suppressed because it is too large Load Diff

3
src/data_scanner_re.re
Unescape Escape View File

@ -202,7 +202,8 @@ bool data_scanner::tokenize2(pcre_context &pc, data_token_t &token_out)
[a-zA-Z0-9\._%+-]+"@"[a-zA-Z0-9\.-]+"."[a-zA-Z]+ { RET(DT_EMAIL); }
("true"|"True"|"TRUE"|"false"|"False"|"FALSE"|"None"|"null"|"NULL") { RET(DT_CONSTANT); }
"true"|"True"|"TRUE"|"false"|"False"|"FALSE"|"None"|"null"|"NULL"/([\r\n\t \(\)!\*:;'\"\?,]|[\.\!,\?]SPACE|EOF) { RET(DT_CONSTANT); }
("re-")?[a-zA-Z][a-z']+/([\r\n\t \(\)!\*:;'\"\?,]|[\.\!,\?]SPACE|EOF) { RET(DT_WORD); }
[^\x00"; \t\r\n:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\\]+("::"[^\x00"; \r\n\t:=,\(\)\{\}\[\]\+#!%\^&\*'\?<>\~`\|\\]+)* {

2
src/dhclient-summary.lnav
Unescape Escape View File

@ -14,7 +14,7 @@
FROM
(WITH lease_times AS
(SELECT min(log_time) AS start_time, ip FROM
(SELECT log_time,extract('bound to (\S+) --', log_text) AS ip FROM syslog_log WHERE ip IS NOT NULL)
(SELECT log_time, regexp_match('bound to (\S+) --', log_text) AS ip FROM syslog_log WHERE ip IS NOT NULL)
GROUP BY ip ORDER BY start_time ASC)
SELECT start_time,
(SELECT lt2.start_time AS end_time FROM lease_times AS lt2 WHERE lt1.start_time < lt2.start_time LIMIT 1) AS end_time,

1
src/lnav.cc
Unescape Escape View File

@ -239,6 +239,7 @@ bool setup_logline_table()
static const char *hidden_table_columns[] = {
"log_path",
"log_text",
"log_body",
NULL
};

2
src/lnav_commands.cc
Unescape Escape View File

@ -607,7 +607,7 @@ static string com_save_to(string cmdline, vector<string> &args)
for (iter = row_iter->begin();
iter != row_iter->end();
++iter) {
csv_write_string(outfile, *iter);
fputs(*iter, outfile);
}
fprintf(outfile, "\n");
}

61
src/log_vtab_impl.cc
Unescape Escape View File

@ -90,7 +90,8 @@ std::string log_vtab_impl::get_table_statement(void)
oss << coldecl;
}
oss << " log_path text hidden collate naturalnocase,\n"
<< " log_text text hidden\n"
<< " log_text text hidden,\n"
<< " log_body text hidden\n"
<< ");\n";
return oss.str();
@ -368,22 +369,52 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col)
(VT_COL_MAX + vt->vi->vi_column_count -
1) - 1;
if (post_col_number == 0) {
const string &fn = lf->get_filename();
switch (post_col_number) {
case 0: {
const string &fn = lf->get_filename();
sqlite3_result_text(ctx,
fn.c_str(),
fn.length(),
SQLITE_STATIC);
}
else {
shared_buffer_ref line;
sqlite3_result_text(ctx,
fn.c_str(),
fn.length(),
SQLITE_STATIC);
break;
}
case 1: {
shared_buffer_ref line;
lf->read_full_message(ll, line);
sqlite3_result_text(ctx,
line.get_data(),
line.length(),
SQLITE_TRANSIENT);
lf->read_full_message(ll, line);
sqlite3_result_text(ctx,
line.get_data(),
line.length(),
SQLITE_TRANSIENT);
break;
}
case 2: {
if (vc->line_values.empty()) {
logfile::iterator line_iter;
line_iter = lf->begin() + cl;
lf->read_full_message(line_iter, vc->log_msg);
vt->vi->extract(lf, vc->log_msg, vc->line_values);
}
struct line_range body_range;
body_range = find_string_attr_range(
vt->vi->vi_attrs, &textview_curses::SA_BODY);
if (!body_range.is_valid()) {
sqlite3_result_null(ctx);
}
else {
const char *msg_start = vc->log_msg.get_data();
sqlite3_result_text(ctx,
&msg_start[body_range.lr_start],
body_range.length(),
SQLITE_TRANSIENT);
}
break;
}
}
}
else {

4
src/log_vtab_impl.hh
Unescape Escape View File

@ -110,7 +110,7 @@ public:
shared_buffer_ref &line,
std::vector<logline_value> &values)
{
log_format * format = lf->get_format();
log_format *format = lf->get_format();
this->vi_attrs.clear();
format->annotate(line, this->vi_attrs, values);
@ -118,9 +118,9 @@ public:
bool vi_supports_indexes;
int vi_column_count;
string_attrs_t vi_attrs;
protected:
const intern_string_t vi_name;
string_attrs_t vi_attrs;
};
class log_format_vtab_impl : public log_vtab_impl {

144
src/string-extension-functions.cc
Unescape Escape View File

@ -21,6 +21,8 @@
#include "column_namer.hh"
#include "yajl/api/yajl_gen.h"
#include "sqlite-extension-func.h"
#include "data_scanner.hh"
#include "data_parser.hh"
typedef struct {
char * s;
@ -119,7 +121,7 @@ void regexp(sqlite3_context *ctx, int argc, sqlite3_value **argv)
}
static
void extract(sqlite3_context *ctx, int argc, sqlite3_value **argv)
void regexp_match(sqlite3_context *ctx, int argc, sqlite3_value **argv)
{
const char *re, *str;
cache_entry *reobj;
@ -134,7 +136,7 @@ void extract(sqlite3_context *ctx, int argc, sqlite3_value **argv)
str = (const char *)sqlite3_value_text(argv[1]);
if (!str) {
sqlite3_result_error(ctx, "no string", -1);
sqlite3_result_null(ctx);
return;
}
@ -242,6 +244,141 @@ void extract(sqlite3_context *ctx, int argc, sqlite3_value **argv)
#endif
}
static
void elements_to_json(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el);
static
void element_to_json(yajl_gen gen, data_parser &dp, const data_parser::element &elem)
{
size_t value_len;
const char *value_str = dp.get_element_string(elem, value_len);
switch (elem.value_token()) {
case DT_NUMBER: {
yajl_gen_number(gen, value_str, value_len);
break;
}
case DNT_GROUP: {
elements_to_json(gen, dp, elem.e_sub_elements);
break;
}
case DNT_PAIR: {
const data_parser::element &pair_elem = elem.e_sub_elements->front();
yajlpp_map singleton_map(gen);
singleton_map.gen(dp.get_element_string(pair_elem.e_sub_elements->front()));
element_to_json(gen, dp, pair_elem.get_pair_value());
break;
}
case DT_CONSTANT: {
if (strncasecmp("true", value_str, value_len) == 0) {
yajl_gen_bool(gen, true);
}
else if (strncasecmp("false", value_str, value_len) == 0) {
yajl_gen_bool(gen, false);
}
else {
yajl_gen_null(gen);
}
break;
}
default:
yajl_gen_pstring(gen, value_str, value_len);
break;
}
}
static
void map_elements_to_json(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el)
{
yajlpp_map root_map(gen);
column_namer cn;
for (data_parser::element_list_t::iterator iter = el->begin();
iter != el->end();
++iter) {
const data_parser::element &pvalue = iter->get_pair_value();
if (pvalue.value_token() == DT_INVALID) {
log_debug("invalid!!");
// continue;
}
std::string key_str = dp.get_element_string(
iter->e_sub_elements->front());
string colname = cn.add_column(key_str);
root_map.gen(colname);
element_to_json(gen, dp, pvalue);
}
}
static
void list_elements_to_json(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el)
{
yajlpp_array root_array(gen);
for (data_parser::element_list_t::iterator iter = el->begin();
iter != el->end();
++iter) {
element_to_json(gen, dp, *iter);
}
}
static
void elements_to_json(yajl_gen gen, data_parser &dp, data_parser::element_list_t *el)
{
if (el->empty()) {
yajl_gen_null(gen);
}
else {
switch (el->front().e_token) {
case DNT_PAIR:
map_elements_to_json(gen, dp, el);
break;
default:
list_elements_to_json(gen, dp, el);
break;
}
}
}
static
void extract(sqlite3_context *ctx, int argc, sqlite3_value **argv)
{
const char *str;
assert(argc == 1);
str = (const char *)sqlite3_value_text(argv[0]);
if (!str) {
sqlite3_result_null(ctx);
return;
}
data_scanner ds(str);
data_parser dp(&ds);
dp.parse();
// dp.print(stderr, dp.dp_pairs);
auto_mem<yajl_gen_t> gen(yajl_gen_free);
gen = yajl_gen_alloc(NULL);
yajl_gen_config(gen.in(), yajl_gen_beautify, false);
elements_to_json(gen, dp, &dp.dp_pairs);
const unsigned char *buf;
size_t len;
yajl_gen_get_buf(gen, &buf, &len);
sqlite3_result_text(ctx, (const char *) buf, len, SQLITE_TRANSIENT);
#ifdef HAVE_SQLITE3_VALUE_SUBTYPE
sqlite3_result_subtype(ctx, JSON_SUBTYPE);
#endif
}
static
void regexp_replace(sqlite3_context *ctx, int argc, sqlite3_value **argv)
{
@ -338,8 +475,9 @@ int string_extension_functions(const struct FuncDef **basic_funcs,
static const struct FuncDef string_funcs[] = {
{ "regexp", 2, 0, SQLITE_UTF8, 0, regexp },
{ "regexp_replace", 3, 0, SQLITE_UTF8, 0, regexp_replace },
{ "regexp_match", 2, 0, SQLITE_UTF8, 0, regexp_match },
{ "extract", 2, 0, SQLITE_UTF8, 0, extract },
{ "extract", 1, 0, SQLITE_UTF8, 0, extract },
{ "startswith", 2, 0, SQLITE_UTF8, 0, sql_startswith },
{ "endswith", 2, 0, SQLITE_UTF8, 0, sql_endswith },

43
test/datafile_simple.14
Unescape Escape View File

@ -0,0 +1,43 @@
FSChange(Direction.DOWNLOAD, Action.CREATE, name=Baby Names, route=[CloudEntry(doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names)])
key 0:0
sym 0:8 ^------^ FSChange
pair 0:8 ^------^ FSChange
key 9:9 ^
key 9:9 ^
sym 9:27 ^----------------^ Direction.DOWNLOAD
val 9:27 ^----------------^ Direction.DOWNLOAD
pair 9:27 ^----------------^ Direction.DOWNLOAD
key 29:29 ^
sym 29:42 ^-----------^ Action.CREATE
val 29:42 ^-----------^ Action.CREATE
pair 29:42 ^-----------^ Action.CREATE
key 44:48 ^--^ name
word 49:53 ^--^ Baby
wspc 53:54 ^
word 54:59 ^---^ Names
val 49:59 ^--------^ Baby Names
pair 44:59 ^-------------^ name=Baby Names
key 61:66 ^---^ route
key 68:68 ^
sym 68:78 ^--------^ CloudEntry
pair 68:78 ^--------^ CloudEntry
key 79:79 ^
key 79:85 ^----^ doc_id
sym 86:130 ^------------------------------------------^ 1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg
val 86:130 ^------------------------------------------^ 1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg
pair 79:130 ^-------------------------------------------------^ doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg
key 131:139 ^------^ filename
word 140:144 ^--^ Baby
wspc 144:145 ^
word 145:150 ^---^ Names
val 140:150 ^--------^ Baby Names
pair 131:150 ^-----------------^ filename=Baby Names
grp 79:150 ^---------------------------------------------------------------------^ doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names
pair 79:150 ^---------------------------------------------------------------------^ doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names
grp 68:150 ^--------------------------------------------------------------------------------^ CloudEntry(doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names
val 68:150 ^--------------------------------------------------------------------------------^ CloudEntry(doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names
pair 61:150 ^---------------------------------------------------------------------------------------^ route=[CloudEntry(doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names
grp 9:150 ^-------------------------------------------------------------------------------------------------------------------------------------------^ Direction.DOWNLOAD, Action.CREATE, name=Baby Names, route=[CloudEntry(doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names
pair 9:150 ^-------------------------------------------------------------------------------------------------------------------------------------------^ Direction.DOWNLOAD, Action.CREATE, name=Baby Names, route=[CloudEntry(doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names
msg :FSChange(Direction.DOWNLOAD, Action.CREATE, name=Baby Names, route=[CloudEntry(doc_id=1g5Yho6JmysVGRO-Xmfurra_cQRFb0nTIfZRhGompweg,filename=Baby Names)])
format :#(#)])

52
test/datafile_simple.15
Unescape Escape View File

@ -0,0 +1,52 @@
Worker successfully completed [ImmutableChange(Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False)]
key 31:31 ^
key 31:31 ^
sym 31:46 ^-------------^ ImmutableChange
pair 31:46 ^-------------^ ImmutableChange
key 47:47 ^
key 47:47 ^
sym 47:63 ^--------------^ Direction.UPLOAD
val 47:63 ^--------------^ Direction.UPLOAD
pair 47:63 ^--------------^ Direction.UPLOAD
key 65:65 ^
sym 65:78 ^-----------^ Action.CREATE
val 65:78 ^-----------^ Action.CREATE
pair 65:78 ^-----------^ Action.CREATE
key 80:83 ^-^ ino
key 84:91 ^-----^ LocalID
key 92:97 ^---^ inode
num 98:105 ^-----^ 5567236
val 98:105 ^-----^ 5567236
pair 92:105 ^-----------^ inode=5567236
grp 92:105 ^-----------^ inode=5567236
pair 84:105 ^-------------------^ LocalID(inode=5567236
val 84:105 ^-------------------^ LocalID(inode=5567236
pair 80:105 ^-----------------------^ ino=LocalID(inode=5567236
key 108:112 ^--^ path
quot 115:140 ^-----------------------^ /Users/stack/Google Drive
val 115:140 ^-----------------------^ /Users/stack/Google Drive
pair 108:140 ^------------------------------^ path=u'/Users/stack/Google Drive
key 143:147 ^--^ name
quot 150:167 ^---------------^ pyjsonpath1.patch
val 150:167 ^---------------^ pyjsonpath1.patch
pair 143:167 ^----------------------^ name=u'pyjsonpath1.patch
key 170:180 ^--------^ parent_ino
key 181:188 ^-----^ LocalID
key 189:194 ^---^ inode
num 195:203 ^------^ 46166734
val 195:203 ^------^ 46166734
pair 189:203 ^------------^ inode=46166734
grp 189:203 ^------------^ inode=46166734
pair 181:203 ^--------------------^ LocalID(inode=46166734
val 181:203 ^--------------------^ LocalID(inode=46166734
pair 170:203 ^-------------------------------^ parent_ino=LocalID(inode=46166734
key 206:215 ^-------^ is_folder
cnst 216:221 ^---^ False
val 216:221 ^---^ False
pair 206:221 ^-------------^ is_folder=False
grp 47:221 ^----------------------------------------------------------------------------------------------------------------------------------------------------------------------------^ Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False
pair 47:221 ^----------------------------------------------------------------------------------------------------------------------------------------------------------------------------^ Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False
grp 31:221 ^--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------^ ImmutableChange(Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False
pair 31:221 ^--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------^ ImmutableChange(Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False
msg :Worker successfully completed [ImmutableChange(Direction.UPLOAD, Action.CREATE, ino=LocalID(inode=5567236), path=u'/Users/stack/Google Drive', name=u'pyjsonpath1.patch', parent_ino=LocalID(inode=46166734), is_folder=False)]
format :Worker successfully completed [#)]

2
test/datafile_simple.8
Unescape Escape View File

@ -23,9 +23,11 @@ pair 160:165
key 168:168 ^
key 168:168 ^
num 168:174 ^----^ 100003
val 168:174 ^----^ 100003
pair 168:174 ^----^ 100003
key 175:175 ^
num 175:176 ^ 1
val 175:176 ^ 1
pair 175:176 ^ 1
grp 168:176 ^------^ 100003,1
pair 168:176 ^------^ 100003,1

13
test/parser_debugger.py
Unescape Escape View File

@ -175,8 +175,21 @@ while True:
if not cmd or cmd[0] == '':
pass
elif cmd[0] == 'h':
print 'Help:'
print ' q - quit'
print ' s - Start over'
print ' n - Next step'
print ' r - Previous step'
print ' b - Previous breakpoint'
print ' c - Next breakpoint'
print ' p - Print state'
print ' w <var> - Add a variable to the watch list'
print ' u <var> - Remove a variable from the watch list'
elif cmd[0] == 'q':
break
elif cmd[0] == 's':
index = 0
elif cmd[0] == 'n':
if index < len(ops):
index += 1

23
test/test_sql_str_func.sh
Unescape Escape View File

@ -78,25 +78,26 @@ Row 0:
EOF
run_test ./drive_sql "select extract('abc', 'abc')"
run_test ./drive_sql "select regexp_match('abc', 'abc')"
check_error_output "" <<EOF
error: sqlite3_exec failed -- regular expression does not have any captures
EOF
run_test ./drive_sql "select extract(null, 'abc')"
run_test ./drive_sql "select regexp_match(null, 'abc')"
check_error_output "" <<EOF
error: sqlite3_exec failed -- no regexp
EOF
run_test ./drive_sql "select extract('abc', null)"
run_test ./drive_sql "select regexp_match('abc', null) as result"
check_error_output "" <<EOF
error: sqlite3_exec failed -- no string
check_output "" <<EOF
Row 0:
Column result: (null)
EOF
run_test ./drive_sql "select typeof(result), result from (select extract('(\d*)abc', 'abc') as result)"
run_test ./drive_sql "select typeof(result), result from (select regexp_match('(\d*)abc', 'abc') as result)"
check_output "" <<EOF
Row 0:
@ -104,7 +105,7 @@ Row 0:
Column result:
EOF
run_test ./drive_sql "select typeof(result), result from (select extract('(\d*)abc(\d*)', 'abc') as result)"
run_test ./drive_sql "select typeof(result), result from (select regexp_match('(\d*)abc(\d*)', 'abc') as result)"
check_output "" <<EOF
Row 0:
@ -112,7 +113,7 @@ Row 0:
Column result: {"col_0":"","col_1":""}
EOF
run_test ./drive_sql "select typeof(result), result from (select extract('(\d+)', '123') as result)"
run_test ./drive_sql "select typeof(result), result from (select regexp_match('(\d+)', '123') as result)"
check_output "" <<EOF
Row 0:
@ -120,7 +121,7 @@ Row 0:
Column result: 123
EOF
run_test ./drive_sql "select typeof(result), result from (select extract('a(\d+\.\d+)a', 'a123.456a') as result)"
run_test ./drive_sql "select typeof(result), result from (select regexp_match('a(\d+\.\d+)a', 'a123.456a') as result)"
check_output "" <<EOF
Row 0:
@ -128,14 +129,14 @@ Row 0:
Column result: 123.456
EOF
run_test ./drive_sql "select extract('foo=(?<foo>\w+); (\w+)', 'foo=abc; 123') as result"
run_test ./drive_sql "select regexp_match('foo=(?<foo>\w+); (\w+)', 'foo=abc; 123') as result"
check_output "" <<EOF
Row 0:
Column result: {"foo":"abc","col_0":123}
EOF
run_test ./drive_sql "select extract('foo=(?<foo>\w+); (\w+\.\w+)', 'foo=abc; 123.456') as result"
run_test ./drive_sql "select regexp_match('foo=(?<foo>\w+); (\w+\.\w+)', 'foo=abc; 123.456') as result"
check_output "" <<EOF
Row 0:

Write Preview
Loading…
Cancel
Save