From 92e20ffd515e3d80097391726a28adeacc9c2090 Mon Sep 17 00:00:00 2001 From: Timothy Stack Date: Sun, 21 Mar 2021 08:53:21 -0700 Subject: [PATCH] [reltime] add support for weekdays and having timeslice() return NULL when a time is out-of-range --- src/base/time_util.cc | 13 ++ src/base/time_util.hh | 9 +- src/big_array.hh | 4 + src/hotkeys.cc | 2 +- src/init.sql | 9 +- src/internals/sql-ref.rst | 18 +- src/lnav_commands.cc | 13 +- src/logfile_sub_source.cc | 11 +- src/logfile_sub_source.hh | 19 +- src/relative_time.cc | 315 ++++++++++++++++++++++++++++++-- src/relative_time.hh | 137 ++++---------- src/time-extension-functions.cc | 57 ++++-- src/vtab_module.hh | 10 + src/yajlpp/yajlpp_def.hh | 4 +- test/expected_help.txt | 10 +- test/test_reltime.cc | 114 +++++++++++- test/test_sql_time_func.sh | 47 ++++- 17 files changed, 604 insertions(+), 188 deletions(-) diff --git a/src/base/time_util.cc b/src/base/time_util.cc index 96e2cd6f..86ce9bc0 100644 --- a/src/base/time_util.cc +++ b/src/base/time_util.cc @@ -31,6 +31,7 @@ #include "config.h" +#include #include "time_util.hh" static time_t BAD_DATE = -1; @@ -182,3 +183,15 @@ struct tm *secs2tm(time_t *tim_p, struct tm *res) return (res); } + +struct timeval exttm::to_timeval() const +{ + struct timeval retval; + + retval.tv_sec = tm2sec(&this->et_tm); + retval.tv_usec = + std::chrono::duration_cast( + std::chrono::nanoseconds(this->et_nsec)).count(); + + return retval; +} diff --git a/src/base/time_util.hh b/src/base/time_util.hh index 7575008d..52901f69 100644 --- a/src/base/time_util.hh +++ b/src/base/time_util.hh @@ -73,14 +73,7 @@ struct exttm { return memcmp(this, &other, sizeof(exttm)) == 0; }; - struct timeval to_timeval() const { - struct timeval retval; - - retval.tv_sec = tm2sec(&this->et_tm); - retval.tv_usec = this->et_nsec * 1000; - - return retval; - }; + struct timeval to_timeval() const; }; inline diff --git a/src/big_array.hh b/src/big_array.hh index b8db5705..fefd3734 100644 --- a/src/big_array.hh +++ b/src/big_array.hh @@ -91,6 +91,10 @@ struct big_array { return this->ba_ptr[index]; }; + const T &operator[](size_t index) const { + return this->ba_ptr[index]; + }; + T &back() { return this->ba_ptr[this->ba_size - 1]; } diff --git a/src/hotkeys.cc b/src/hotkeys.cc index 2b5f932e..ad7028e4 100644 --- a/src/hotkeys.cc +++ b/src/hotkeys.cc @@ -925,7 +925,7 @@ bool handle_paging_key(int ch) logline *ll = lnav_data.ld_log_source.find_line(cl); ll->to_exttm(tm); do { - rt.add(tm); + tm = rt.adjust(tm); new_vl = lnav_data.ld_log_source.find_from_time(tm); if (new_vl == 0_vl || new_vl != vl || !rt.is_relative()) { diff --git a/src/init.sql b/src/init.sql index 6136ac99..d12d7ddc 100644 --- a/src/init.sql +++ b/src/init.sql @@ -80,6 +80,7 @@ CREATE TABLE lnav_example_log ( ex_procname text collate 'BINARY', ex_duration integer, + log_time_msecs int hidden, log_path text hidden collate naturalnocase, log_text text hidden, log_body text hidden @@ -90,7 +91,7 @@ CREATE VIEW lnav_top_view AS SELECT name FROM lnav_view_stack ORDER BY rowid DESC LIMIT 1); INSERT INTO lnav_example_log VALUES - (0, null, '2017-02-03T04:05:06.100', '2017-02-03T04:05:06.100', 0, 'info', 0, null, null, null, 'hw', 2, '/tmp/log', '2017-02-03T04:05:06.100 hw(2): Hello, World!', 'Hello, World!'), - (1, null, '2017-02-03T04:05:06.200', '2017-02-03T04:05:06.200', 100, 'error', 0, null, null, null, 'gw', 4, '/tmp/log', '2017-02-03T04:05:06.200 gw(4): Goodbye, World!', 'Goodbye, World!'), - (2, 'new', '2017-02-03T04:25:06.200', '2017-02-03T04:25:06.200', 1200000, 'warn', 0, null, null, null, 'gw', 1, '/tmp/log', '2017-02-03T04:25:06.200 gw(1): Goodbye, World!', 'Goodbye, World!'), - (3, 'new', '2017-02-03T04:55:06.200', '2017-02-03T04:55:06.200', 1800000, 'debug', 0, null, null, null, 'gw', 10, '/tmp/log', '2017-02-03T04:55:06.200 gw(10): Goodbye, World!', 'Goodbye, World!'); + (0, null, '2017-02-03T04:05:06.100', '2017-02-03T04:05:06.100', 0, 'info', 0, null, null, null, 'hw', 2, 1486094706000, '/tmp/log', '2017-02-03T04:05:06.100 hw(2): Hello, World!', 'Hello, World!'), + (1, null, '2017-02-03T04:05:06.200', '2017-02-03T04:05:06.200', 100, 'error', 0, null, null, null, 'gw', 4, 1486094706000, '/tmp/log', '2017-02-03T04:05:06.200 gw(4): Goodbye, World!', 'Goodbye, World!'), + (2, 'new', '2017-02-03T04:25:06.200', '2017-02-03T04:25:06.200', 1200000, 'warn', 0, null, null, null, 'gw', 1, 1486095906000, '/tmp/log', '2017-02-03T04:25:06.200 gw(1): Goodbye, World!', 'Goodbye, World!'), + (3, 'new', '2017-02-03T04:55:06.200', '2017-02-03T04:55:06.200', 1800000, 'debug', 0, null, null, null, 'gw', 10, 1486097706000, '/tmp/log', '2017-02-03T04:55:06.200 gw(10): Goodbye, World!', 'Goodbye, World!'); diff --git a/src/internals/sql-ref.rst b/src/internals/sql-ref.rst index ff1f45c7..3db7f70b 100644 --- a/src/internals/sql-ref.rst +++ b/src/internals/sql-ref.rst @@ -3251,7 +3251,7 @@ timediff(*time1*, *time2*) timeslice(*time*, *slice*) ^^^^^^^^^^^^^^^^^^^^^^^^^^ - Return the start of the slice of time that the given timestamp falls in. + Return the start of the slice of time that the given timestamp falls in. If the time falls outside of the slice, NULL is returned. **Parameters** * **time\*** --- The timestamp to get the time slice for. @@ -3269,8 +3269,20 @@ timeslice(*time*, *slice*) .. code-block:: custsqlite - ;SELECT timeslice(log_time_msecs, '5m') AS slice, count(*) FROM lnav_example_log GROUP BY slice - 2017-01-01 05:00:00.000 + ;SELECT timeslice(log_time_msecs, '5m') AS slice, count(1) FROM lnav_example_log GROUP BY slice + slice count(1) + 2017-02-03 04:05:00.000 2 + 2017-02-03 04:25:00.000 1 + 2017-02-03 04:55:00.000 1 + + To group log messages by those before 4:30am and after: + + .. code-block:: custsqlite + + ;SELECT timeslice(log_time_msecs, 'before 4:30am') AS slice, count(1) FROM lnav_example_log GROUP BY slice + slice count(1) + 3 + 2017-02-03 04:30:00.000 1 **See Also** :ref:`date`, :ref:`datetime`, :ref:`julianday`, :ref:`strftime`, :ref:`time`, :ref:`timediff` diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index 1774438b..58f2beb7 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -172,9 +172,9 @@ static Result com_adjust_log_time(exec_context &ec, string cmdli auto parse_res = relative_time::from_str(args[1]); if (parse_res.isOk()) { - new_time = parse_res.unwrap().add(top_time).to_timeval(); + new_time = parse_res.unwrap().adjust(top_time).to_timeval(); } - else if (dts.scan(args[1].c_str(), args[1].size(), NULL, &tm, new_time) != NULL) { + else if (dts.scan(args[1].c_str(), args[1].size(), nullptr, &tm, new_time) != nullptr) { // nothing to do } else { return ec.make_error("could not parse timestamp -- {}", args[1]); @@ -316,7 +316,7 @@ static Result com_goto(exec_context &ec, string cmdline, vector< } do { - struct exttm tm = rt.add(tv); + auto tm = rt.adjust(tv); tv = tm.to_timeval(); new_vl = vis_line_t(ttt->row_for_time(tv)); @@ -2708,7 +2708,7 @@ static Result com_pt_time(exec_context &ec, string cmdline, vect dts.set_base_time(now); if (parse_res.isOk()) { tm.et_tm = *gmtime(&now); - parse_res.unwrap().add(tm); + tm = parse_res.unwrap().adjust(tm); new_time.tv_sec = timegm(&tm.et_tm); } else { @@ -3296,10 +3296,7 @@ static Result com_hide_line(exec_context &ec, string cmdline, ve cl = lnav_data.ld_log_source.at(vl); ll = lnav_data.ld_log_source.find_line(cl); ll->to_exttm(tm); - parse_res.unwrap().add(tm); - - tv.tv_sec = timegm(&tm.et_tm); - tv.tv_usec = tm.et_nsec / 1000; + tv = parse_res.unwrap().adjust(tm).to_timeval(); tv_set = true; } diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc index c79b5161..6d5f1ace 100644 --- a/src/logfile_sub_source.cc +++ b/src/logfile_sub_source.cc @@ -134,15 +134,14 @@ shared_ptr logfile_sub_source::find(const char *fn, return retval; } -vis_line_t logfile_sub_source::find_from_time(const struct timeval &start) +vis_line_t logfile_sub_source::find_from_time(const struct timeval &start) const { - vector::iterator lb; vis_line_t retval(-1); - lb = lower_bound(this->lss_filtered_index.begin(), - this->lss_filtered_index.end(), - start, - filtered_logline_cmp(*this)); + auto lb = lower_bound(this->lss_filtered_index.begin(), + this->lss_filtered_index.end(), + start, + filtered_logline_cmp(*this)); if (lb != this->lss_filtered_index.end()) { retval = vis_line_t(lb - this->lss_filtered_index.begin()); } diff --git a/src/logfile_sub_source.hh b/src/logfile_sub_source.hh index d7cfa399..dff7de9e 100644 --- a/src/logfile_sub_source.hh +++ b/src/logfile_sub_source.hh @@ -491,7 +491,7 @@ public: std::shared_ptr find(const char *fn, content_line_t &line_base); - std::shared_ptr find(content_line_t &line) + std::shared_ptr find(content_line_t &line) const { std::shared_ptr retval; @@ -509,7 +509,7 @@ public: return retval; }; - logline *find_line(content_line_t line) + logline *find_line(content_line_t line) const { logline *retval = nullptr; std::shared_ptr lf = this->find(line); @@ -523,7 +523,7 @@ public: return retval; }; - vis_line_t find_from_time(const struct timeval &start); + vis_line_t find_from_time(const struct timeval &start) const; vis_line_t find_from_time(time_t start) { struct timeval tv = { start, 0 }; @@ -531,13 +531,8 @@ public: return this->find_from_time(tv); }; - vis_line_t find_from_time(exttm &etm) { - struct timeval tv; - - tv.tv_sec = timegm(&etm.et_tm); - tv.tv_usec = etm.et_nsec / 1000; - - return this->find_from_time(tv); + vis_line_t find_from_time(const exttm &etm) const { + return this->find_from_time(etm.to_timeval()); }; nonstd::optional find_from_content(content_line_t cl) { @@ -893,7 +888,7 @@ private: }; struct filtered_logline_cmp { - filtered_logline_cmp(logfile_sub_source & lc) + filtered_logline_cmp(const logfile_sub_source & lc) : llss_controller(lc) { }; bool operator()(const uint32_t &lhs, const uint32_t &rhs) const { @@ -916,7 +911,7 @@ private: return (*ll_lhs) < rhs; }; - logfile_sub_source & llss_controller; + const logfile_sub_source & llss_controller; }; /** diff --git a/src/relative_time.cc b/src/relative_time.cc index 499ed6dd..1f0af007 100644 --- a/src/relative_time.cc +++ b/src/relative_time.cc @@ -31,12 +31,14 @@ #include +#include "base/time_util.hh" #include "pcrepp/pcrepp.hh" #include "relative_time.hh" using namespace std; +using namespace std::chrono_literals; -static struct { +static const struct { const char *name; pcrepp pcre; } MATCHERS[relative_time::RTT__MAX] = { @@ -49,6 +51,15 @@ static struct { { "time", pcrepp("\\A(\\d{1,2}):(\\d{2})(?::(\\d{2})(?:\\.(\\d{3,6}))?)?") }, { "num", pcrepp("\\A((?:-|\\+)?\\d+)") }, + + { "sun", pcrepp("\\Asun(days?)?\\b") }, + { "mon", pcrepp("\\Amon(days?)?\\b") }, + { "tue", pcrepp("\\Atue(s(days?)?)?\\b") }, + { "wed", pcrepp("\\Awed(nesdays?)?\\b") }, + { "thu", pcrepp("\\Athu(rsdays?)?\\b") }, + { "fri", pcrepp("\\Afri(days?)?\\b") }, + { "sat", pcrepp("\\Asat(urdays?)?\\b") }, + { "us", pcrepp("\\A(?:micros(?:econds?)?|us(?![a-zA-Z]))") }, { "ms", pcrepp("\\A(?:millis(?:econds?)?|ms(?![a-zA-Z]))") }, { "sec", pcrepp("\\As(?:ec(?:onds?)?)?(?![a-zA-Z])") }, @@ -154,7 +165,7 @@ relative_time::from_str(const char *str, size_t len) gettimeofday(&tv, nullptr); localtime_r(&tv.tv_sec, &tm.et_tm); tm.et_nsec = tv.tv_usec * 1000; - retval.add(tm); + tm = retval.adjust(tm); retval.rt_field[RTF_YEARS] = tm.et_tm.tm_year; retval.rt_field[RTF_MONTHS] = tm.et_tm.tm_mon; @@ -193,6 +204,7 @@ relative_time::from_str(const char *str, size_t len) retval.rt_field[RTF_MINUTES] = 0; retval.rt_field[RTF_SECONDS] = 0; retval.rt_field[RTF_MICROSECONDS] = 0; + retval.rt_duration = 1min; retval.rt_absolute_field_end = RTF__MAX; number_set = false; } @@ -208,6 +220,36 @@ relative_time::from_str(const char *str, size_t len) else if (retval.rt_field[RTF_HOURS].value < 12) { retval.rt_field[RTF_HOURS].value += 12; } + if (base_token == RTT_AFTER) { + std::chrono::microseconds usecs = 0s; + uint64_t carry = 0; + + if (retval.rt_field[RTF_MICROSECONDS].value > 0) { + usecs += std::chrono::microseconds( + 1000000ULL - retval.rt_field[RTF_MICROSECONDS].value); + carry = 1; + } + if (carry || retval.rt_field[RTF_SECONDS].value > 0) { + usecs += std::chrono::seconds( + 60 - carry - retval.rt_field[RTF_SECONDS].value); + carry = 1; + } + if (carry || retval.rt_field[RTF_MINUTES].value > 0) { + usecs += std::chrono::minutes( + 60 - carry - retval.rt_field[RTF_MINUTES].value); + carry = 1; + } + usecs += std::chrono::hours( + 24 - retval.rt_field[RTF_HOURS].value); + retval.rt_duration = usecs; + } + if (base_token == RTT_BEFORE) { + retval.rt_duration = + std::chrono::hours(retval.rt_field[RTF_HOURS].value) + + std::chrono::minutes(retval.rt_field[RTF_MINUTES].value) + + std::chrono::seconds(retval.rt_field[RTF_SECONDS].value) + + std::chrono::microseconds(retval.rt_field[RTF_MICROSECONDS].value); + } break; case RTT_A: case RTT_AN: @@ -237,11 +279,15 @@ relative_time::from_str(const char *str, size_t len) atoi(substr.c_str()); break; } + } else { + retval.rt_field[RTF_MICROSECONDS].clear(); + retval.rt_duration = 1s; } } else { - retval.rt_field[RTF_SECONDS] = 0; - retval.rt_field[RTF_MICROSECONDS] = 0; + retval.rt_field[RTF_SECONDS].clear(); + retval.rt_field[RTF_MICROSECONDS].clear(); + retval.rt_duration = 1min; } retval.rt_absolute_field_end = RTF__MAX; break; @@ -339,7 +385,6 @@ relative_time::from_str(const char *str, size_t len) retval.rt_absolute_field_end = RTF__MAX; } break; - case RTT_BEFORE: case RTT_AGO: if (retval.empty()) { pe_out.pe_msg = "Expecting a time unit"; @@ -357,6 +402,7 @@ relative_time::from_str(const char *str, size_t len) retval.rt_absolute_field_end = last_field_type; } break; + case RTT_BEFORE: case RTT_AFTER: base_token = token; break; @@ -389,6 +435,19 @@ relative_time::from_str(const char *str, size_t len) } break; + case RTT_SUNDAY: + case RTT_MONDAY: + case RTT_TUESDAY: + case RTT_WEDNESDAY: + case RTT_THURSDAY: + case RTT_FRIDAY: + case RTT_SATURDAY: + if (retval.rt_duration == 0s) { + retval.rt_duration = 24h; + } + retval.rt_included_days.insert(token); + break; + case RTT__MAX: assert(false); break; @@ -443,20 +502,50 @@ void relative_time::rollover() } } -void relative_time::from_timeval(const struct timeval& tv) +relative_time relative_time::from_timeval(const struct timeval& tv) +{ + relative_time retval; + + retval.clear(); + retval.rt_field[RTF_SECONDS] = tv.tv_sec; + retval.rt_field[RTF_MICROSECONDS] = tv.tv_usec; + retval.rollover(); + + return retval; +} + +relative_time relative_time::from_usecs(std::chrono::microseconds usecs) { - this->clear(); - this->rt_field[RTF_SECONDS] = tv.tv_sec; - this->rt_field[RTF_MICROSECONDS] = tv.tv_usec; - this->rollover(); + relative_time retval; + + retval.clear(); + retval.rt_field[RTF_MICROSECONDS] = usecs.count(); + retval.rollover(); + + return retval; } -std::string relative_time::to_string() +std::string relative_time::to_string() const { + static const char *DAYS[] = { + "sun", + "mon", + "tue", + "wed", + "thu", + "fri", + "sat", + }; + char dst[128] = ""; char *pos = dst; if (this->is_absolute()) { + for (const auto& day_token : this->rt_included_days) { + pos += snprintf(pos, sizeof(dst) - (pos - dst), + "%s ", DAYS[day_token - RTT_SUNDAY]); + } + pos += snprintf(pos, sizeof(dst) - (pos - dst), "%s", this->rt_next ? "next " : @@ -528,6 +617,210 @@ std::string relative_time::to_string() return dst; } +struct exttm relative_time::adjust(const exttm &tm) const +{ + auto retval = tm; + + if (this->rt_field[RTF_MICROSECONDS].is_set && this->is_absolute(RTF_MICROSECONDS)) { + retval.et_nsec = this->rt_field[RTF_MICROSECONDS].value * 1000; + } + else { + retval.et_nsec += this->rt_field[RTF_MICROSECONDS].value * 1000; + } + if (this->rt_field[RTF_SECONDS].is_set && this->is_absolute(RTF_SECONDS)) { + if (this->rt_next && + this->rt_field[RTF_SECONDS].value <= tm.et_tm.tm_sec) { + retval.et_tm.tm_min += 1; + } + if (this->rt_previous && + this->rt_field[RTF_SECONDS].value >= tm.et_tm.tm_sec) { + retval.et_tm.tm_min -= 1; + } + retval.et_tm.tm_sec = this->rt_field[RTF_SECONDS].value; + } + else { + retval.et_tm.tm_sec += this->rt_field[RTF_SECONDS].value; + } + if (this->rt_field[RTF_MINUTES].is_set && this->is_absolute(RTF_MINUTES)) { + if (this->rt_next && + this->rt_field[RTF_MINUTES].value <= tm.et_tm.tm_min) { + retval.et_tm.tm_hour += 1; + } + if (this->rt_previous && (this->rt_field[RTF_MINUTES].value == 0 || + (this->rt_field[RTF_MINUTES].value >= tm.et_tm.tm_min))) { + retval.et_tm.tm_hour -= 1; + } + retval.et_tm.tm_min = this->rt_field[RTF_MINUTES].value; + } + else { + retval.et_tm.tm_min += this->rt_field[RTF_MINUTES].value; + } + if (this->rt_field[RTF_HOURS].is_set && this->is_absolute(RTF_HOURS)) { + if (this->rt_next && + this->rt_field[RTF_HOURS].value <= tm.et_tm.tm_hour) { + retval.et_tm.tm_mday += 1; + } + if (this->rt_previous && + this->rt_field[RTF_HOURS].value >= tm.et_tm.tm_hour) { + retval.et_tm.tm_mday -= 1; + } + retval.et_tm.tm_hour = this->rt_field[RTF_HOURS].value; + } + else { + retval.et_tm.tm_hour += this->rt_field[RTF_HOURS].value; + } + if (this->rt_field[RTF_DAYS].is_set && this->is_absolute(RTF_DAYS)) { + if (this->rt_next && + this->rt_field[RTF_DAYS].value <= tm.et_tm.tm_mday) { + retval.et_tm.tm_mon += 1; + } + if (this->rt_previous && + this->rt_field[RTF_DAYS].value >= tm.et_tm.tm_mday) { + retval.et_tm.tm_mon -= 1; + } + retval.et_tm.tm_mday = this->rt_field[RTF_DAYS].value; + } + else { + retval.et_tm.tm_mday += this->rt_field[RTF_DAYS].value; + } + if (this->rt_field[RTF_MONTHS].is_set && this->is_absolute(RTF_MONTHS)) { + if (this->rt_next && + this->rt_field[RTF_MONTHS].value <= tm.et_tm.tm_mon) { + retval.et_tm.tm_year += 1; + } + if (this->rt_previous && + this->rt_field[RTF_MONTHS].value >= tm.et_tm.tm_mon) { + retval.et_tm.tm_year -= 1; + } + retval.et_tm.tm_mon = this->rt_field[RTF_MONTHS].value; + } + else { + retval.et_tm.tm_mon += this->rt_field[RTF_MONTHS].value; + } + if (this->rt_field[RTF_YEARS].is_set && this->is_absolute(RTF_YEARS)) { + retval.et_tm.tm_year = this->rt_field[RTF_YEARS].value; + } + else { + retval.et_tm.tm_year += this->rt_field[RTF_YEARS].value; + } + + return retval; +} + +nonstd::optional relative_time::window_start( + const struct exttm &tm) const +{ + auto retval = tm; + + if (this->is_relative()) { + uint64_t us, remainder; + + auto tv = tm.to_timeval(); + us = (uint64_t) tv.tv_sec * 1000000ULL + (uint64_t) tv.tv_usec; + remainder = us % this->to_microseconds(); + us -= remainder; + + tv.tv_sec = us / 1000000ULL; + tv.tv_usec = us % 1000000ULL; + + retval.et_tm = *gmtime(&tv.tv_sec); + retval.et_nsec = tv.tv_usec * 1000ULL; + + return retval; + } + + bool clear = false; + + if (this->rt_field[RTF_YEARS].is_set) { + if (this->rt_field[RTF_YEARS].value > tm.et_tm.tm_year) { + return nonstd::nullopt; + } + retval.et_tm.tm_year = this->rt_field[RTF_YEARS].value - 1900; + clear = true; + } + + if (this->rt_field[RTF_MONTHS].is_set) { + if (this->rt_field[RTF_MONTHS].value - 1 > tm.et_tm.tm_mon) { + return nonstd::nullopt; + } + retval.et_tm.tm_mon = this->rt_field[RTF_MONTHS].value - 1; + clear = true; + } else if (clear) { + retval.et_tm.tm_mon = 0; + } + + if (this->rt_field[RTF_DAYS].is_set) { + if (this->rt_field[RTF_DAYS].value > tm.et_tm.tm_mday) { + return nonstd::nullopt; + } + retval.et_tm.tm_mday = this->rt_field[RTF_DAYS].value; + clear = true; + } else if (clear) { + retval.et_tm.tm_mday = 1; + } + + if (!this->rt_included_days.empty()) { + auto iter = this->rt_included_days.find((token_t) (RTT_SUNDAY + tm.et_tm.tm_wday)); + + if (iter == this->rt_included_days.end()) { + return nonstd::nullopt; + } + clear = true; + } + + if (this->rt_field[RTF_HOURS].is_set && + this->rt_field[RTF_HOURS].value) { + if (this->rt_field[RTF_HOURS].value > tm.et_tm.tm_hour) { + return nonstd::nullopt; + } + retval.et_tm.tm_hour = this->rt_field[RTF_HOURS].value; + clear = true; + } else if (clear) { + retval.et_tm.tm_hour = 0; + } + + if (this->rt_field[RTF_MINUTES].is_set) { + if (this->rt_field[RTF_MINUTES].value > tm.et_tm.tm_min) { + return nonstd::nullopt; + } + retval.et_tm.tm_min = this->rt_field[RTF_MINUTES].value; + clear = true; + } else if (clear) { + retval.et_tm.tm_min = 0; + } + + if (this->rt_field[RTF_SECONDS].is_set) { + if (this->rt_field[RTF_SECONDS].value > tm.et_tm.tm_sec) { + return nonstd::nullopt; + } + retval.et_tm.tm_sec = this->rt_field[RTF_SECONDS].value; + clear = true; + } else if (clear) { + retval.et_tm.tm_sec = 0; + } + + if (this->rt_field[RTF_MICROSECONDS].is_set) { + if (this->rt_field[RTF_MICROSECONDS].value > tm.et_nsec / 1000) { + return nonstd::nullopt; + } + retval.et_nsec = this->rt_field[RTF_MICROSECONDS].value * 1000ULL; + clear = true; + } else if (clear) { + retval.et_nsec = 0; + } + + auto tv = tm.to_timeval(); + auto start_time = retval.to_timeval(); + auto end_time = relative_time::from_usecs(this->rt_duration) + .adjust(retval).to_timeval(); + + if (tv < start_time || end_time < tv) { + return nonstd::nullopt; + } + + return retval; +} + size_t duration2str(int64_t millis, std::string &value_out) { /* 24h22m33s111 */ diff --git a/src/relative_time.hh b/src/relative_time.hh index 7e24b089..ab18a051 100644 --- a/src/relative_time.hh +++ b/src/relative_time.hh @@ -35,7 +35,9 @@ #define __STDC_FORMAT_MACROS #include +#include #include +#include #include #include "ptimec.hh" @@ -54,6 +56,15 @@ public: RTT_AT, RTT_TIME, RTT_NUMBER, + + RTT_SUNDAY, + RTT_MONDAY, + RTT_TUESDAY, + RTT_WEDNESDAY, + RTT_THURSDAY, + RTT_FRIDAY, + RTT_SATURDAY, + RTT_MICROS, RTT_MILLIS, RTT_SECONDS, @@ -106,6 +117,10 @@ public: return from_str(str.c_str(), str.length()); } + static relative_time from_timeval(const struct timeval& tv); + + static relative_time from_usecs(std::chrono::microseconds usecs); + relative_time() { this->clear(); }; @@ -148,7 +163,7 @@ public: }; bool is_absolute() const { - return this->rt_absolute_field_end > 0; + return !this->rt_included_days.empty() || this->rt_absolute_field_end > 0; }; bool is_absolute(rt_field_type rft) const { @@ -160,6 +175,9 @@ public: } bool empty() const { + if (!this->rt_included_days.empty()) { + return false; + } for (auto rtf : this->rt_field) { if (rtf.is_set) { return false; @@ -168,111 +186,28 @@ public: return true; }; - struct exttm add_now() { + struct exttm adjust_now() const { struct exttm tm; time_t now; time(&now); + memset(&tm, 0, sizeof(tm)); tm.et_tm = *gmtime(&now); - this->add(tm); - - return tm; + return this->adjust(tm); }; - struct exttm add(const struct timeval &tv) { + struct exttm adjust(const struct timeval &tv) const { struct exttm tm; + memset(&tm, 0, sizeof(tm)); tm.et_tm = *gmtime(&tv.tv_sec); tm.et_nsec = tv.tv_usec * 1000; - this->add(tm); - - return tm; + return this->adjust(tm); } - void add(struct exttm &tm) { - if (this->rt_field[RTF_MICROSECONDS].is_set && this->is_absolute(RTF_MICROSECONDS)) { - tm.et_nsec = this->rt_field[RTF_MICROSECONDS].value * 1000; - } - else { - tm.et_nsec += this->rt_field[RTF_MICROSECONDS].value * 1000; - } - if (this->rt_field[RTF_SECONDS].is_set && this->is_absolute(RTF_SECONDS)) { - if (this->rt_next && - this->rt_field[RTF_SECONDS].value <= tm.et_tm.tm_sec) { - tm.et_tm.tm_min += 1; - } - if (this->rt_previous && - this->rt_field[RTF_SECONDS].value >= tm.et_tm.tm_sec) { - tm.et_tm.tm_min -= 1; - } - tm.et_tm.tm_sec = this->rt_field[RTF_SECONDS].value; - } - else { - tm.et_tm.tm_sec += this->rt_field[RTF_SECONDS].value; - } - if (this->rt_field[RTF_MINUTES].is_set && this->is_absolute(RTF_MINUTES)) { - if (this->rt_next && - this->rt_field[RTF_MINUTES].value <= tm.et_tm.tm_min) { - tm.et_tm.tm_hour += 1; - } - if (this->rt_previous && (this->rt_field[RTF_MINUTES].value == 0 || - (this->rt_field[RTF_MINUTES].value >= tm.et_tm.tm_min))) { - tm.et_tm.tm_hour -= 1; - } - tm.et_tm.tm_min = this->rt_field[RTF_MINUTES].value; - } - else { - tm.et_tm.tm_min += this->rt_field[RTF_MINUTES].value; - } - if (this->rt_field[RTF_HOURS].is_set && this->is_absolute(RTF_HOURS)) { - if (this->rt_next && - this->rt_field[RTF_HOURS].value <= tm.et_tm.tm_hour) { - tm.et_tm.tm_mday += 1; - } - if (this->rt_previous && - this->rt_field[RTF_HOURS].value >= tm.et_tm.tm_hour) { - tm.et_tm.tm_mday -= 1; - } - tm.et_tm.tm_hour = this->rt_field[RTF_HOURS].value; - } - else { - tm.et_tm.tm_hour += this->rt_field[RTF_HOURS].value; - } - if (this->rt_field[RTF_DAYS].is_set && this->is_absolute(RTF_DAYS)) { - if (this->rt_next && - this->rt_field[RTF_DAYS].value <= tm.et_tm.tm_mday) { - tm.et_tm.tm_mon += 1; - } - if (this->rt_previous && - this->rt_field[RTF_DAYS].value >= tm.et_tm.tm_mday) { - tm.et_tm.tm_mon -= 1; - } - tm.et_tm.tm_mday = this->rt_field[RTF_DAYS].value; - } - else { - tm.et_tm.tm_mday += this->rt_field[RTF_DAYS].value; - } - if (this->rt_field[RTF_MONTHS].is_set && this->is_absolute(RTF_MONTHS)) { - if (this->rt_next && - this->rt_field[RTF_MONTHS].value <= tm.et_tm.tm_mon) { - tm.et_tm.tm_year += 1; - } - if (this->rt_previous && - this->rt_field[RTF_MONTHS].value >= tm.et_tm.tm_mon) { - tm.et_tm.tm_year -= 1; - } - tm.et_tm.tm_mon = this->rt_field[RTF_MONTHS].value; - } - else { - tm.et_tm.tm_mon += this->rt_field[RTF_MONTHS].value; - } - if (this->rt_field[RTF_YEARS].is_set && this->is_absolute(RTF_YEARS)) { - tm.et_tm.tm_year = this->rt_field[RTF_YEARS].value; - } - else { - tm.et_tm.tm_year += this->rt_field[RTF_YEARS].value; - } - }; + struct exttm adjust(const struct exttm &tm) const; + + nonstd::optional window_start(const struct exttm &tm) const; int64_t to_microseconds() const { int64_t retval; @@ -283,20 +218,19 @@ public: retval = (retval + this->rt_field[RTF_HOURS].value) * 60; retval = (retval + this->rt_field[RTF_MINUTES].value) * 60; retval = (retval + this->rt_field[RTF_SECONDS].value) * 1000 * 1000; + retval = (retval + this->rt_field[RTF_MICROSECONDS].value); return retval; }; - void from_timeval(const struct timeval& tv); - - void to_timeval(struct timeval &tv_out) { + void to_timeval(struct timeval &tv_out) const { int64_t us = this->to_microseconds(); tv_out.tv_sec = us / (1000 * 1000); tv_out.tv_usec = us % (1000 * 1000); }; - struct timeval to_timeval() { + struct timeval to_timeval() const { int64_t us = this->to_microseconds(); struct timeval retval; @@ -305,7 +239,7 @@ public: return retval; }; - std::string to_string(); + std::string to_string() const; void rollover(); @@ -318,11 +252,18 @@ public: _rt_field() : value(0), is_set(false) { }; + void clear() { + this->value = 0; + this->is_set = false; + } + int64_t value; bool is_set; }; std::array<_rt_field, RTF__MAX> rt_field; + std::set rt_included_days; + std::chrono::microseconds rt_duration{0}; bool rt_next; bool rt_previous; diff --git a/src/time-extension-functions.cc b/src/time-extension-functions.cc index 11c1a8c3..6acacf45 100644 --- a/src/time-extension-functions.cc +++ b/src/time-extension-functions.cc @@ -47,7 +47,7 @@ using namespace std; -static auto_buffer timeslice(sqlite3_value *time_in, nonstd::optional slice_in_opt) +static nonstd::optional timeslice(sqlite3_value *time_in, nonstd::optional slice_in_opt) { thread_local date_time_scanner dts; thread_local struct { @@ -73,21 +73,17 @@ static auto_buffer timeslice(sqlite3_value *time_in, nonstd::optional(sqlite3_value_text( time_in)); - struct exttm tm; if (dts.scan(time_in_str, strlen(time_in_str), nullptr, &tm, tv, false) == nullptr) { dts.unlock(); @@ -96,26 +92,42 @@ static auto_buffer timeslice(sqlite3_value *time_in, nonstd::optional(msecs) + .count(); + tm.et_tm = *gmtime(&tv.tv_sec); + tm.et_nsec = std::chrono::duration_cast( + msecs % 1000).count(); + break; + } + case SQLITE_FLOAT: { + auto secs = sqlite3_value_double(time_in); + double integ; + auto fract = modf(secs, &integ); + + tv.tv_sec = integ; + tm.et_tm = *gmtime(&tv.tv_sec); + tm.et_nsec = floor(fract * 1000000000.0); break; } + case SQLITE_NULL: { + return nonstd::nullopt; + } } - remainder = us % cache.c_rel_time.to_microseconds(); - us -= remainder; + auto win_start_opt = cache.c_rel_time.window_start(tm); - tv.tv_sec = us / (1000LL * 1000LL); - tv.tv_usec = us % (1000LL * 1000LL); + if (!win_start_opt) { + return nonstd::nullopt; + } + auto win_start = *win_start_opt; auto ts = auto_buffer::alloc(64); - auto actual_length = sql_strftime(ts.in(), ts.size(), tv); + auto actual_length = sql_strftime(ts.in(), ts.size(), win_start.to_timeval()); ts.shrink_to(actual_length); return ts; @@ -129,14 +141,14 @@ nonstd::optional sql_timediff(const char *time1, const char *time2) auto parse_res1 = relative_time::from_str(time1, -1); if (parse_res1.isOk()) { - tv1 = parse_res1.unwrap().add_now().to_timeval(); + tv1 = parse_res1.unwrap().adjust_now().to_timeval(); } else if (!dts1.convert_to_timeval(time1, -1, nullptr, tv1)) { return nonstd::nullopt; } auto parse_res2 = relative_time::from_str(time2, -1); if (parse_res2.isOk()) { - tv2 = parse_res2.unwrap().add_now().to_timeval(); + tv2 = parse_res2.unwrap().adjust_now().to_timeval(); } else if (!dts2.convert_to_timeval(time2, -1, nullptr, tv2)) { return nonstd::nullopt; } @@ -152,7 +164,8 @@ int time_extension_functions(struct FuncDef **basic_funcs, static struct FuncDef time_funcs[] = { sqlite_func_adapter::builder( help_text("timeslice", - "Return the start of the slice of time that the given timestamp falls in.") + "Return the start of the slice of time that the given timestamp falls in. " + "If the time falls outside of the slice, NULL is returned.") .sql_function() .with_parameter({"time", "The timestamp to get the time slice for."}) .with_parameter({"slice", "The size of the time slices"}) @@ -163,7 +176,11 @@ int time_extension_functions(struct FuncDef **basic_funcs, }) .with_example({ "To group log messages into five minute buckets and count them", - "SELECT timeslice(log_time_msecs, '5m') AS slice, count(*) FROM lnav_example_log GROUP BY slice" + "SELECT timeslice(log_time_msecs, '5m') AS slice, count(1) FROM lnav_example_log GROUP BY slice" + }) + .with_example({ + "To group log messages by those before 4:30am and after", + "SELECT timeslice(log_time_msecs, 'before 4:30am') AS slice, count(1) FROM lnav_example_log GROUP BY slice" }) ), diff --git a/src/vtab_module.hh b/src/vtab_module.hh index 076849f5..5a3093e0 100644 --- a/src/vtab_module.hh +++ b/src/vtab_module.hh @@ -214,6 +214,16 @@ inline void to_sqlite(sqlite3_context *ctx, double val) #define JSON_SUBTYPE 74 /* Ascii for "J" */ +template +inline void to_sqlite(sqlite3_context *ctx, nonstd::optional &val) +{ + if (val.has_value()) { + to_sqlite(ctx, val.value()); + } else { + sqlite3_result_null(ctx); + } +} + template inline void to_sqlite(sqlite3_context *ctx, const nonstd::optional &val) { diff --git a/src/yajlpp/yajlpp_def.hh b/src/yajlpp/yajlpp_def.hh index 411e6c95..5f27b11e 100644 --- a/src/yajlpp/yajlpp_def.hh +++ b/src/yajlpp/yajlpp_def.hh @@ -629,10 +629,8 @@ struct json_path_handler : public json_path_handler_base { } yajlpp_generator gen(handle); - relative_time rt; - rt.from_timeval({ field.count(), 0 }); - return gen(rt.to_string()); + return gen(relative_time::from_timeval({field.count(), 0}).to_string()); }; this->jph_field_getter = [args...](void *root, nonstd::optional name) { return (void *) &json_path_handler::get_field(root, args...); diff --git a/test/expected_help.txt b/test/expected_help.txt index 6e5a96fc..e92700fe 100644 --- a/test/expected_help.txt +++ b/test/expected_help.txt @@ -2778,7 +2778,8 @@ Examples Synopsis timeslice(time, slice) -- Return the start of the slice of time that the - given timestamp falls in. + given timestamp falls in. If the time falls outside of the slice, NULL is + returned. Parameters time The timestamp to get the time slice for. slice The size of the time slices @@ -2790,10 +2791,15 @@ Examples #2 To group log messages into five minute buckets and count them: - ;SELECT timeslice(log_time_msecs, '5m') AS slice, count(*) FROM lnav_example_log GROUP + ;SELECT timeslice(log_time_msecs, '5m') AS slice, count(1) FROM lnav_example_log GROUP BY slice +#3 To group log messages by those before 4:30am and after: + ;SELECT timeslice(log_time_msecs, 'before 4:30am') AS slice, count(1) FROM + lnav_example_log GROUP BY slice + + Synopsis total(X) -- Returns the sum of the values in the group as a floating-point. diff --git a/test/test_reltime.cc b/test/test_reltime.cc index 023a46a0..49ff1a60 100644 --- a/test/test_reltime.cc +++ b/test/test_reltime.cc @@ -91,6 +91,100 @@ TEST_CASE("reltime") struct exttm tm, tm2; time_t new_time; + { + auto rt_res = relative_time::from_str("sun after 1pm"); + + CHECK(rt_res.isOk()); + auto rt = rt_res.unwrap(); + + time_t t_in = 1615727900; + memset(&tm, 0, sizeof(tm)); + tm.et_tm = *gmtime(&t_in); + auto win_opt = rt.window_start(tm); + auto win_tm = *win_opt; + CHECK(win_tm.et_tm.tm_year == 121); + CHECK(win_tm.et_tm.tm_mon == 2); + CHECK(win_tm.et_tm.tm_mday == 14); + CHECK(win_tm.et_tm.tm_hour == 13); + CHECK(win_tm.et_tm.tm_min == 0); + CHECK(win_tm.et_tm.tm_sec == 0); + } + + { + auto rt_res = relative_time::from_str("0:05"); + + CHECK(rt_res.isOk()); + auto rt = rt_res.unwrap(); + + time_t t_in = 5 * 60 + 15; + memset(&tm, 0, sizeof(tm)); + tm.et_tm = *gmtime(&t_in); + auto win_opt = rt.window_start(tm); + auto win_tm = *win_opt; + CHECK(win_tm.et_tm.tm_sec == 0); + CHECK(win_tm.et_tm.tm_min == 5); + CHECK(win_tm.et_tm.tm_hour == 0); + + t_in = 4 * 60 + 15; + memset(&tm, 0, sizeof(tm)); + tm.et_tm = *gmtime(&t_in); + win_opt = rt.window_start(tm); + CHECK(!win_opt.has_value()); + } + + { + auto rt_res = relative_time::from_str("mon"); + + CHECK(rt_res.isOk()); + auto rt = rt_res.unwrap(); + + time_t t_in = 1615841352; + memset(&tm, 0, sizeof(tm)); + tm.et_tm = *gmtime(&t_in); + auto win_opt = rt.window_start(tm); + auto win_tm = *win_opt; + CHECK(win_tm.et_tm.tm_year == 121); + CHECK(win_tm.et_tm.tm_mon == 2); + CHECK(win_tm.et_tm.tm_mday == 15); + CHECK(win_tm.et_tm.tm_hour == 0); + CHECK(win_tm.et_tm.tm_min == 0); + CHECK(win_tm.et_tm.tm_sec == 0); + } + + { + auto rt_res = relative_time::from_str("tue"); + + CHECK(rt_res.isOk()); + auto rt = rt_res.unwrap(); + CHECK(rt.rt_included_days == + std::set{relative_time::RTT_TUESDAY}); + } + + { + auto rt_res = relative_time::from_str("1m"); + + CHECK(rt_res.isOk()); + auto rt = rt_res.unwrap(); + + time_t t_in = 30; + memset(&tm, 0, sizeof(tm)); + tm.et_tm = *gmtime(&t_in); + auto win_opt = rt.window_start(tm); + auto win_tm = *win_opt; + CHECK(win_tm.et_tm.tm_sec == 0); + CHECK(win_tm.et_tm.tm_min == 0); + CHECK(win_tm.et_tm.tm_hour == 0); + + t_in = 90; + memset(&tm, 0, sizeof(tm)); + tm.et_tm = *gmtime(&t_in); + win_opt = rt.window_start(tm); + win_tm = *win_opt; + CHECK(win_tm.et_tm.tm_sec == 0); + CHECK(win_tm.et_tm.tm_min == 1); + CHECK(win_tm.et_tm.tm_hour == 0); + } + relative_time rt; for (int lpc = 0; TEST_DATA[lpc].reltime; lpc++) { auto res = relative_time::from_str(TEST_DATA[lpc].reltime); @@ -144,7 +238,7 @@ TEST_CASE("reltime") CHECK(rt.is_absolute()); tm = base_tm; - rt.add(tm); + tm = rt.adjust(tm); new_time = timegm(&tm.et_tm); tm.et_tm = *gmtime(&new_time); @@ -154,7 +248,7 @@ TEST_CASE("reltime") rt = relative_time::from_str("5 minutes ago").unwrap(); tm = base_tm; - rt.add(tm); + tm = rt.adjust(tm); new_time = timegm(&tm.et_tm); @@ -163,20 +257,20 @@ TEST_CASE("reltime") rt = relative_time::from_str("today at 4pm").unwrap(); memset(&tm, 0, sizeof(tm)); memset(&tm2, 0, sizeof(tm2)); - gettimeofday(&tv, NULL); + gettimeofday(&tv, nullptr); localtime_r(&tv.tv_sec, &tm.et_tm); localtime_r(&tv.tv_sec, &tm2.et_tm); tm2.et_tm.tm_hour = 16; tm2.et_tm.tm_min = 0; tm2.et_tm.tm_sec = 0; - rt.add(tm); + tm = rt.adjust(tm); tm.et_tm.tm_yday = 0; tm2.et_tm.tm_yday = 0; tm.et_tm.tm_wday = 0; tm2.et_tm.tm_wday = 0; #ifdef HAVE_STRUCT_TM_TM_ZONE tm2.et_tm.tm_gmtoff = 0; - tm2.et_tm.tm_zone = NULL; + tm2.et_tm.tm_zone = nullptr; #endif CHECK(tm.et_tm.tm_year == tm2.et_tm.tm_year); CHECK(tm.et_tm.tm_mon == tm2.et_tm.tm_mon); @@ -186,14 +280,14 @@ TEST_CASE("reltime") CHECK(tm.et_tm.tm_sec == tm2.et_tm.tm_sec); rt = relative_time::from_str("yesterday at 4pm").unwrap(); - gettimeofday(&tv, NULL); + gettimeofday(&tv, nullptr); localtime_r(&tv.tv_sec, &tm.et_tm); localtime_r(&tv.tv_sec, &tm2.et_tm); tm2.et_tm.tm_mday -= 1; tm2.et_tm.tm_hour = 16; tm2.et_tm.tm_min = 0; tm2.et_tm.tm_sec = 0; - rt.add(tm); + tm = rt.adjust(tm); tm.et_tm.tm_yday = 0; tm2.et_tm.tm_yday = 0; tm.et_tm.tm_wday = 0; @@ -210,21 +304,21 @@ TEST_CASE("reltime") CHECK(tm.et_tm.tm_sec == tm2.et_tm.tm_sec); rt = relative_time::from_str("2 days ago").unwrap(); - gettimeofday(&tv, NULL); + gettimeofday(&tv, nullptr); localtime_r(&tv.tv_sec, &tm.et_tm); localtime_r(&tv.tv_sec, &tm2.et_tm); tm2.et_tm.tm_mday -= 2; tm2.et_tm.tm_hour = 0; tm2.et_tm.tm_min = 0; tm2.et_tm.tm_sec = 0; - rt.add(tm); + tm = rt.adjust(tm); tm.et_tm.tm_yday = 0; tm2.et_tm.tm_yday = 0; tm.et_tm.tm_wday = 0; tm2.et_tm.tm_wday = 0; #ifdef HAVE_STRUCT_TM_TM_ZONE tm2.et_tm.tm_gmtoff = 0; - tm2.et_tm.tm_zone = NULL; + tm2.et_tm.tm_zone = nullptr; #endif CHECK(tm.et_tm.tm_year == tm2.et_tm.tm_year); CHECK(tm.et_tm.tm_mon == tm2.et_tm.tm_mon); diff --git a/test/test_sql_time_func.sh b/test/test_sql_time_func.sh index 42515eaf..e4ce0bcb 100644 --- a/test/test_sql_time_func.sh +++ b/test/test_sql_time_func.sh @@ -1,5 +1,19 @@ #! /bin/bash +run_test ./drive_sql "select timeslice('2015-08-07 12:01:00', 'after 12pm')" + +check_output "after 12pm" <