From a3e1fd27b8d1db003200a91a5dade450270e6a2c Mon Sep 17 00:00:00 2001 From: Timothy Stack Date: Wed, 6 Jul 2022 21:10:23 -0700 Subject: [PATCH] [db] charting of JSON values was mistakenly limited to very short values --- src/base/intern_string.hh | 17 +++++++++++ src/db_sub_source.cc | 5 ++-- src/hist_source.hh | 21 +++++++------- src/json-extension-functions.cc | 51 ++++++++++++++++++++++++++++++++- src/vtab_module.hh | 3 +- src/vtab_module_json.hh | 12 ++++++++ src/yajlpp/json_op.hh | 4 ++- 7 files changed, 97 insertions(+), 16 deletions(-) diff --git a/src/base/intern_string.hh b/src/base/intern_string.hh index 2fa65ce7..0f7c49f7 100644 --- a/src/base/intern_string.hh +++ b/src/base/intern_string.hh @@ -138,6 +138,23 @@ struct string_fragment { return *prefix == '\0'; } + bool endswith(const char* suffix) const + { + auto suffix_len = strlen(suffix); + + if (suffix_len > this->length()) { + return false; + } + + const auto* curr = this->end() - suffix_len; + while (*suffix != '\0' && *curr == *suffix) { + suffix += 1; + curr += 1; + } + + return *suffix == '\0'; + } + string_fragment substr(int begin) const { return string_fragment{ diff --git a/src/db_sub_source.cc b/src/db_sub_source.cc index 2e6b0bc9..3b070e5e 100644 --- a/src/db_sub_source.cc +++ b/src/db_sub_source.cc @@ -40,6 +40,7 @@ const char* db_label_source::NULL_STR = ""; constexpr size_t MAX_COLUMN_WIDTH = 120; +constexpr size_t MAX_JSON_WIDTH = 16 * 1024; void db_label_source::text_value_for_line(textview_curses& tc, @@ -134,7 +135,7 @@ db_label_source::text_attrs_for_line(textview_curses& tc, tc, left, this->dls_headers[lpc].hm_name, num_value, sa); } } - if (row_len > 2 && row_len < MAX_COLUMN_WIDTH + if (row_len > 2 && row_len < MAX_JSON_WIDTH && ((row_value[0] == '{' && row_value[row_len - 1] == '}') || (row_value[0] == '[' && row_value[row_len - 1] == ']'))) { @@ -143,7 +144,7 @@ db_label_source::text_attrs_for_line(textview_curses& tc, if (jpw.parse(row_value, row_len) == yajl_status_ok && jpw.complete_parse() == yajl_status_ok) { - for (auto& jpw_value : jpw.jpw_values) { + for (const auto& jpw_value : jpw.jpw_values) { double num_value; if (jpw_value.wt_type == yajl_t_number diff --git a/src/hist_source.hh b/src/hist_source.hh index 52d247c7..89aa86d2 100644 --- a/src/hist_source.hh +++ b/src/hist_source.hh @@ -203,7 +203,7 @@ public: lr.lr_start = left; - const struct chart_ident& ci = this->sbc_idents[ident_index]; + const auto& ci = this->sbc_idents[ident_index]; int amount; if (value == 0.0) { @@ -240,8 +240,9 @@ public: struct bucket_stats_t { bucket_stats_t() - : bs_min_value(std::numeric_limits::max()), - bs_max_value(0){}; + : bs_min_value(std::numeric_limits::max()), bs_max_value(0) + { + } void merge(const bucket_stats_t& rhs, bool do_stacking) { @@ -252,18 +253,18 @@ public: this->bs_max_value = std::max(this->bs_max_value, rhs.bs_max_value); } - }; + } double width() const { return std::fabs(this->bs_max_value - this->bs_min_value); - }; + } void update(double value) { this->bs_max_value = std::max(this->bs_max_value, value); this->bs_min_value = std::min(this->bs_min_value, value); - }; + } double bs_min_value; double bs_max_value; @@ -278,7 +279,7 @@ public: protected: struct chart_ident { - chart_ident(const T& ident) : ci_ident(ident) {} + explicit chart_ident(const T& ident) : ci_ident(ident) {} T ci_ident; int ci_attrs{0}; @@ -287,12 +288,10 @@ protected: struct chart_ident& find_ident(const T& ident) { - typename std::map::iterator iter; - - iter = this->sbc_ident_lookup.find(ident); + auto iter = this->sbc_ident_lookup.find(ident); if (iter == this->sbc_ident_lookup.end()) { this->sbc_ident_lookup[ident] = this->sbc_idents.size(); - this->sbc_idents.push_back(ident); + this->sbc_idents.emplace_back(ident); return this->sbc_idents.back(); } return this->sbc_idents[iter->second]; diff --git a/src/json-extension-functions.cc b/src/json-extension-functions.cc index 963ca6ad..2294e7dc 100644 --- a/src/json-extension-functions.cc +++ b/src/json-extension-functions.cc @@ -517,7 +517,7 @@ json_concat(nonstd::optional json_in, array.gen(sqlite3_value_double(val)); break; case SQLITE3_TEXT: { - auto text_val = sqlite3_value_text(val); + const auto* text_val = sqlite3_value_text(val); if (sqlite3_value_subtype(val) == JSON_SUBTYPE) { concat_gen_elements( @@ -534,6 +534,47 @@ json_concat(nonstd::optional json_in, return json_string(gen); } +#if 0 +static flattened_json_string +sql_flatten_json_object(string_fragment sf) +{ + yajlpp_gen gen; + + { + json_ptr jp("/"); + json_op jo(jp); + auto_mem handle(yajl_free); + + jo.jo_ptr_data = gen.get_handle(); + yajl_gen_config(gen, yajl_gen_beautify, false); + handle.reset(yajl_alloc(&json_op::gen_callbacks, nullptr, &jo)); + switch (yajl_parse( + handle.in(), (const unsigned char*) sf.data(), sf.length())) { + case yajl_status_error: + case yajl_status_client_canceled: + throw yajlpp_error(handle.in(), sf.data(), sf.length()); + case yajl_status_ok: + break; + } + switch (yajl_complete_parse(handle.in())) { + case yajl_status_error: + case yajl_status_client_canceled: + throw yajlpp_error(handle.in(), sf.data(), sf.length()); + case yajl_status_ok: + break; + } + } + + auto result = gen.to_string_fragment(); + if (!result.startswith("{") || !result.endswith("}")) { + throw std::runtime_error( + "flatten_json_object() requires a JSON object"); + } + + return flattened_json_string(gen); +} +#endif + struct json_agg_context { yajl_gen_t* jac_yajl_gen; }; @@ -797,6 +838,14 @@ json_extension_functions(struct FuncDef** basic_funcs, }), }, +#if 0 + sqlite_func_adapter:: + builder(help_text("flatten_json_object", "hello") + .sql_function() + .with_parameter({"json", "The JSON"})), +#endif + {nullptr}, }; diff --git a/src/vtab_module.hh b/src/vtab_module.hh index f5996ce7..0c465894 100644 --- a/src/vtab_module.hh +++ b/src/vtab_module.hh @@ -282,7 +282,8 @@ to_sqlite(sqlite3_context* ctx, double val) sqlite3_result_double(ctx, val); } -#define JSON_SUBTYPE 74 /* Ascii for "J" */ +#define JSON_SUBTYPE 74 /* Ascii for "J" */ +#define FLATTEN_SUBTYPE 0x5f template inline void diff --git a/src/vtab_module_json.hh b/src/vtab_module_json.hh index 30877aad..6e7cd9de 100644 --- a/src/vtab_module_json.hh +++ b/src/vtab_module_json.hh @@ -35,6 +35,18 @@ #include "vtab_module.hh" #include "yajlpp/yajlpp.hh" +struct flattened_json_string : json_string { + explicit flattened_json_string(yajl_gen_t* gen) : json_string(gen) {} +}; + +inline void +to_sqlite(sqlite3_context* ctx, flattened_json_string& val) +{ + sqlite3_result_text( + ctx, (const char*) val.js_content.release(), val.js_len, free); + sqlite3_result_subtype(ctx, FLATTEN_SUBTYPE); +} + inline void to_sqlite(sqlite3_context* ctx, json_string& val) { diff --git a/src/yajlpp/json_op.hh b/src/yajlpp/json_op.hh index f2736942..4f3399be 100644 --- a/src/yajlpp/json_op.hh +++ b/src/yajlpp/json_op.hh @@ -59,7 +59,9 @@ public: const static yajl_callbacks ptr_callbacks; explicit json_op(const json_ptr& ptr) - : jo_ptr(ptr), jo_ptr_callbacks(gen_callbacks){}; + : jo_ptr(ptr), jo_ptr_callbacks(gen_callbacks) + { + } bool check_index(bool primitive = true) {