From fdc2748e3e4f0fd29abeb2101a255d31d9d1f5ba Mon Sep 17 00:00:00 2001 From: Timothy Stack Date: Sun, 20 Mar 2016 15:15:50 -0700 Subject: [PATCH] [spectro] add a spectrogram view that works with known message fields --- NEWS | 11 + docs/source/commands.rst | 11 + docs/source/hotkeys.rst | 6 + release/spectrolog.py | 113 ++++++++++ src/CMakeLists.txt | 1 + src/ansi_scrubber.hh | 2 +- src/chunky_index.hh | 14 +- src/db_sub_source.hh | 4 +- src/default-log-formats.json | 6 +- src/help.txt | 22 ++ src/hist_source.hh | 19 +- src/hotkeys.cc | 105 ++++++--- src/listview_curses.cc | 3 +- src/listview_curses.hh | 22 +- src/lnav.cc | 57 +++-- src/lnav.hh | 9 +- src/lnav_commands.cc | 199 ++++++++++++++++- src/log_format.cc | 65 +++++- src/log_format.hh | 76 ++++++- src/log_format_loader.cc | 11 +- src/log_vtab_impl.cc | 34 +-- src/logfile.cc | 11 +- src/logfile_sub_source.cc | 12 +- src/logfile_sub_source.hh | 2 +- src/plain_text_source.hh | 2 +- src/spectro_source.hh | 368 ++++++++++++++++++++++++++++++++ src/textfile_sub_source.hh | 2 +- src/textview_curses.hh | 13 +- src/time-extension-functions.cc | 4 - src/top_status_source.hh | 3 +- src/view_curses.cc | 6 + src/view_curses.hh | 5 + test/logfile_uwsgi.0 | 19 ++ test/test_cmds.sh | 8 +- test/test_sql.sh | 4 +- 35 files changed, 1117 insertions(+), 132 deletions(-) create mode 100755 release/spectrolog.py create mode 100644 src/spectro_source.hh create mode 100644 test/logfile_uwsgi.0 diff --git a/NEWS b/NEWS index d4feaa84..2c3b31e7 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ lnav v0.8.1: Features: + * Added a spectrogram command and view that displays the values of a + numeric message field over time. * Log formats can now create SQL views and execute other statements by adding '.sql' files to their format directories. The SQL scripts will be executed on startup. @@ -18,6 +20,7 @@ lnav v0.8.1: total number of files, errors, and warnings. * Pressing 'V' in the DB view will now check for a column with a timestamp and move to the corresponding time in the log view. + * Added 'a/A' hotkeys to restore a view previously popped with 'q/Q'. * Added ":hide-lines-before", ":hide-lines-after", and ":show-lines-before-and-after" commands so that you can filter out log lines based on time. @@ -50,6 +53,14 @@ lnav v0.8.1: original message. A "log_actual_time" hidden field has also been added to the SQLite virtual table so you can operate on the original message time from the file. + * The 'A/B' hotkeys for moving forward/backward by 10% line increments + have been reassigned to '[' and ']'. The 'a' and 'A' hotkeys are now + used to return to the previously popped view while trying to preserve + the time range. For example, after leaving the spectrogram view with + 'q', you can press 'A' return to the view with the top time in the + spectrogram matching the top time in the log view. + * The 'Q' hotkey now pops the current view off of the stack while + maintaining the top time between views. Fixes: * Issues with tailing JSON logs have been fixed. diff --git a/docs/source/commands.rst b/docs/source/commands.rst index 20a5d108..b5eae69b 100644 --- a/docs/source/commands.rst +++ b/docs/source/commands.rst @@ -62,6 +62,17 @@ Display * highlight - Colorize text that matches the given regex. * clear-highlight - Clear a previous highlight. +* spectrogram - Generate a spectrogram for the given log + message field. The spectrogram view displays the range of possible values of + the field on the horizontal axis and time on the vertical axis. The + horizontal axis is split into buckets where each bucket counts how many log + messages contained the field with a value in that range. The buckets are + colored based on the count in the bucket: green means low, yellow means + medium, and red means high. The exact ranges for the colors is computed + automatically and displayed in the middle of the top line of the view. The + minimum and maximum values for the field are displayed in the top left and + right sides of the view, respectively. + * switch-to-view - Switch to the given view name (e.g. log, text, ...) * zoom-to - Set the zoom level for the histogram view. diff --git a/docs/source/hotkeys.rst b/docs/source/hotkeys.rst index 8864a429..2f76290f 100644 --- a/docs/source/hotkeys.rst +++ b/docs/source/hotkeys.rst @@ -186,6 +186,12 @@ Display - View/leave builtin help * - |ks| q |ke| - Return to the previous view/quit + * - |ks| Shift |ke| + |ks| q |ke| + - Return to the previous view/quit while matching the top times of the two views + * - |ks| a |ke| + - Restore the view that was previously popped with 'q/Q' + * - |ks| Shift |ke| + |ks| a |ke| + - Restore the view that was previously popped with 'q/Q' and match the top times of the views * - |ks| Shift |ke| + |ks| p |ke| - Switch to/from the pretty-printed view of the displayed log or text files * - |ks| Shift |ke| + |ks| t |ke| diff --git a/release/spectrolog.py b/release/spectrolog.py new file mode 100755 index 00000000..6b052f81 --- /dev/null +++ b/release/spectrolog.py @@ -0,0 +1,113 @@ +#! /usr/bin/env python + +import sys +import time +import datetime +import random + +DATE_FMT = "%a %b %d %H:%M:%S %Y" + +duration = [] + [80] * 10 + [100] * 10 + [40] * 10 + +diter = iter(duration) + +DURATIONS = ( + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 40, + 50, + 50, + 50, + 50, + 75, + 75, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, + 100, +) + +DURATION_FUZZ = ( + 0, + 0, + 0, + 0, + 0, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -2, + -2, + -2 +) + +while True: + print ("[pid: 88186|app: 0|req: 5/19] 127.0.0.1 () {38 vars in 696 bytes} " + "[%s] POST /update_metrics => generated 47 bytes " + "in %s msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 60)" % + (datetime.datetime.utcnow().strftime(DATE_FMT), + diter.next())) + sys.stdout.flush() + + time.sleep(0.25) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 17200ef7..dcb519b3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -113,6 +113,7 @@ set(diag_STAT_SRCS relative_time.hh sequence_sink.hh shlex.hh + spectro_source.hh status_controllers.hh strong_int.hh sysclip.hh diff --git a/src/ansi_scrubber.hh b/src/ansi_scrubber.hh index b205b8b9..53c3a134 100644 --- a/src/ansi_scrubber.hh +++ b/src/ansi_scrubber.hh @@ -41,7 +41,7 @@ #define ANSI_UNDERLINE_START ANSI_CSI "4m" #define ANSI_NORM ANSI_CSI "0m" -#define ANSI_BOLD(msg) ANSI_BOLD_START msg ANSI_NORM +#define ANSI_BOLD(msg) ANSI_BOLD_START msg ANSI_NORM #define ANSI_ROLE(msg) ANSI_CSI "%dO" msg ANSI_NORM diff --git a/src/chunky_index.hh b/src/chunky_index.hh index 6af8e613..43eaeb92 100644 --- a/src/chunky_index.hh +++ b/src/chunky_index.hh @@ -151,9 +151,7 @@ public: this->merge_up_to(&val, comparator); retval = (this->ci_completed_chunks.size() * CHUNK_SIZE); - if (this->ci_merge_chunk != NULL) { - retval += this->ci_merge_chunk->c_used; - } + retval += this->ci_merge_chunk->c_used; this->ci_merge_chunk->push_back(val); this->ci_size += 1; @@ -219,7 +217,9 @@ private: if (!this->ci_pending_chunks.empty()) { struct chunk *next_chunk = this->ci_pending_chunks.front(); - while (((val == NULL) || comparator(next_chunk->front(), *val)) && + while (((val == NULL) || + comparator(next_chunk->front(), *val) || + !comparator(*val, next_chunk->front())) && !this->ci_merge_chunk->full()) { this->ci_merge_chunk->push_back(next_chunk->consume()); if (next_chunk->empty()) { @@ -249,8 +249,10 @@ private: template bool skippable(const T *val, Comparator comparator) const { - return this->c_consumed == 0 && this->full() && ( - val == NULL || (comparator(this->back(), *val) || !comparator(*val, this->back()))); + return this->c_consumed == 0 && this->full() && + (val == NULL || + comparator(this->back(), *val) || + !comparator(*val, this->back())); }; const T &front() const { diff --git a/src/db_sub_source.hh b/src/db_sub_source.hh index bfe4a333..60e4b2a1 100644 --- a/src/db_sub_source.hh +++ b/src/db_sub_source.hh @@ -51,10 +51,10 @@ public: }; size_t text_size_for_line(textview_curses &tc, int line, bool raw) { - return this->text_line_width(); + return this->text_line_width(tc); }; - size_t text_line_width() { + size_t text_line_width(textview_curses &curses) { size_t retval = 0; for (std::vector::iterator iter = this->dls_column_sizes.begin(); diff --git a/src/default-log-formats.json b/src/default-log-formats.json index e36fdc2d..3c079ec2 100644 --- a/src/default-log-formats.json +++ b/src/default-log-formats.json @@ -1082,10 +1082,12 @@ "identifier" : true }, "s_req" : { - "kind" : "integer" + "kind" : "integer", + "foreign-key" : true }, "s_worker_reqs" : { - "kind" : "integer" + "kind" : "integer", + "foreign-key" : true }, "c_ip" : { "kind" : "string", diff --git a/src/help.txt b/src/help.txt index bf0b359a..cdb2a237 100644 --- a/src/help.txt +++ b/src/help.txt @@ -158,6 +158,14 @@ through the file. ? View/leave this help message. q Leave the current view or quit the program when in the log file view. + Q Similar to 'q', except it will try to sync the top time + between the current and former views. For example, when + leaving the spectrogram view with 'Q', the top time in that + view will be matched to the top time in the log view. + + a/A Restore the view that was previously popped with 'q/Q'. + The 'A' hotkey will try to match the top times between the + two views. g/home Move to the top of the file. G/end Move to the end of the file. If the view is already @@ -493,6 +501,20 @@ COMMANDS close Close the current text file or log file. You can also close the current file by pressing 'X'. + spectrogram + Generate a spectrogram for the given log message field. + The spectrogram view displays the range of possible values + of the field on the horizontal axis and time on the + vertical axis. The horizontal axis is split into buckets + where each bucket counts how many log messages contained + the field with a value in that range. The buckets are + colored based on the count in the bucket: green means low, + yellow means medium, and red means high. The exact ranges + for the colors is computed automatically and displayed in + the middle of the top line of the view. The minimum and + maximum values for the field are displayed in the top left + and right sides of the view, respectively. + graph Graph the value of numbers in the file(s) over time. The given regular expression should capture the number to be displayed. For example: diff --git a/src/hist_source.hh b/src/hist_source.hh index 0af8b187..c531374b 100644 --- a/src/hist_source.hh +++ b/src/hist_source.hh @@ -136,7 +136,7 @@ public: return (this->buckets_per_group() + 1) * this->hs_groups.size(); }; - size_t text_line_width() { + size_t text_line_width(textview_curses &curses) { return this->hs_label_source == NULL ? 0 : this->hs_label_source->hist_label_width(); }; @@ -455,7 +455,9 @@ protected: int sbc_ident_to_show; }; -class hist_source2 : public text_sub_source { +class hist_source2 + : public text_sub_source, + public text_time_translator { public: typedef enum { @@ -486,9 +488,6 @@ public: }; void set_time_slice(int64_t slice) { - require(slice >= 60); - require((slice % 60) == 0); - this->hs_time_slice = slice; }; @@ -500,7 +499,7 @@ public: return this->hs_line_count; }; - size_t text_line_width() { + size_t text_line_width(textview_curses &curses) { return strlen(LINE_FORMAT) + 8 * 4; }; @@ -552,7 +551,7 @@ public: if (gmtime_r(&bucket.b_time, &bucket_tm) != NULL) { strftime(tm_buffer, sizeof(tm_buffer), - " %a %b %d %H:%M ", + " %a %b %d %H:%M:%S ", &bucket_tm); } else { @@ -589,7 +588,7 @@ public: return 0; }; - time_t time_for_row(int64_t row) { + time_t time_for_row(int row) { require(row >= 0); require(row < this->hs_line_count); @@ -598,9 +597,9 @@ public: return bucket.b_time; }; - int64_t row_for_time(time_t time_bucket) { + int row_for_time(time_t time_bucket) { std::map::iterator iter; - int64_t retval = 0; + int retval = 0; time_bucket = rounddown(time_bucket, this->hs_time_slice); diff --git a/src/hotkeys.cc b/src/hotkeys.cc index 8f862930..e889ea25 100644 --- a/src/hotkeys.cc +++ b/src/hotkeys.cc @@ -164,12 +164,8 @@ void update_view_name(void) status_field &sf = lnav_data.ld_top_source.statusview_value_for_field( top_status_source::TSF_VIEW_NAME); textview_curses * tc = lnav_data.ld_view_stack.top(); - struct line_range lr(0); - sf.set_value("% 5s ", tc->get_title().c_str()); - sf.get_value().get_attrs().push_back( - string_attr(lr, &view_curses::VC_STYLE, - A_REVERSE | view_colors::ansi_color_pair(COLOR_BLUE, COLOR_WHITE))); + sf.set_value("%s ", tc->get_title().c_str()); } void handle_paging_key(int ch) @@ -187,8 +183,7 @@ void handle_paging_key(int ch) /* process the command keystroke */ switch (ch) { case 'q': - case 'Q': - { + case 'Q': { string msg = ""; if (tc == &lnav_data.ld_views[LNV_DB]) { @@ -200,12 +195,9 @@ void handle_paging_key(int ch) else if (tc == &lnav_data.ld_views[LNV_TEXT]) { msg = HELP_MSG_1(t, "to switch to the text file view"); } - else if (tc == &lnav_data.ld_views[LNV_GRAPH]) { - msg = HELP_MSG_1(g, "to switch to the graph view"); - } lnav_data.ld_rl_view->set_alt_value(msg); - } + lnav_data.ld_last_view = tc; lnav_data.ld_view_stack.pop(); if (lnav_data.ld_view_stack.empty() || (lnav_data.ld_view_stack.size() == 1 && @@ -215,10 +207,54 @@ void handle_paging_key(int ch) else { tc = lnav_data.ld_view_stack.top(); tc->set_needs_update(); + if (ch == 'Q') { + text_time_translator *ttt = dynamic_cast(lnav_data.ld_last_view->get_sub_source()); + textview_curses &log_view = lnav_data.ld_views[LNV_LOG]; + time_t last_time = 0; + + lss = &lnav_data.ld_log_source; + if (ttt != NULL) { + last_time = ttt->time_for_row(lnav_data.ld_last_view->get_top()); + vis_line_t new_log_top = lss->find_from_time(last_time); + + log_view.set_top(new_log_top); + } + } lnav_data.ld_scroll_broadcaster.invoke(tc); update_view_name(); } break; + } + + case 'a': + if (lnav_data.ld_last_view == NULL) { + alerter::singleton().chime(); + } + else { + textview_curses *tc = lnav_data.ld_last_view; + + lnav_data.ld_last_view = NULL; + ensure_view(tc); + } + break; + + case 'A': + if (lnav_data.ld_last_view == NULL) { + alerter::singleton().chime(); + } + else { + textview_curses *tc = lnav_data.ld_last_view; + text_time_translator *ttt = dynamic_cast(tc->get_sub_source()); + + lnav_data.ld_last_view = NULL; + if (ttt != NULL) { + time_t log_top = lnav_data.ld_top_time; + + tc->set_top(vis_line_t(ttt->row_for_time(log_top))); + } + ensure_view(tc); + } + break; case KEY_F(2): if (xterm_mouse::is_available()) { @@ -382,34 +418,20 @@ void handle_paging_key(int ch) break; case 'z': - if (tc == &lnav_data.ld_views[LNV_HISTOGRAM]) { - if ((lnav_data.ld_hist_zoom + 1) >= HIST_ZOOM_LEVELS) { - alerter::singleton().chime(); - } - else { - lnav_data.ld_hist_zoom += 1; - rebuild_hist(0, true); - } - - lnav_data.ld_rl_view->set_alt_value(HELP_MSG_1( - I, - "to switch to the log view at the top displayed time")); + if ((lnav_data.ld_zoom_level - 1) < 0) { + alerter::singleton().chime(); + } + else { + execute_command("zoom-to " + string(lnav_zoom_strings[lnav_data.ld_zoom_level - 1])); } break; case 'Z': - if (tc == &lnav_data.ld_views[LNV_HISTOGRAM]) { - if (lnav_data.ld_hist_zoom == 0) { - alerter::singleton().chime(); - } - else { - lnav_data.ld_hist_zoom -= 1; - rebuild_hist(0, true); - } - - lnav_data.ld_rl_view->set_alt_value(HELP_MSG_1( - I, - "to switch to the log view at the top displayed time")); + if ((lnav_data.ld_zoom_level + 1) >= ZOOM_COUNT) { + alerter::singleton().chime(); + } + else { + execute_command("zoom-to " + string(lnav_zoom_strings[lnav_data.ld_zoom_level + 1])); } break; @@ -852,10 +874,23 @@ void handle_paging_key(int ch) logfile::iterator ll = lf->begin() + cl; log_data_helper ldh(lss); + lnav_data.ld_rl_view->clear_possibilities(LNM_COMMAND, "numeric-colname"); lnav_data.ld_rl_view->clear_possibilities(LNM_COMMAND, "colname"); ldh.parse_line(log_view.get_top(), true); + for (vector::iterator iter = ldh.ldh_line_values.begin(); + iter != ldh.ldh_line_values.end(); + ++iter) { + const logline_value_stats *stats = iter->lv_format->stats_for_value(iter->lv_name); + + if (stats == NULL) { + continue; + } + + lnav_data.ld_rl_view->add_possibility(LNM_COMMAND, "numeric-colname", iter->lv_name.to_string()); + } + for (vector::iterator iter = ldh.ldh_namer->cn_names.begin(); iter != ldh.ldh_namer->cn_names.end(); ++iter) { diff --git a/src/listview_curses.cc b/src/listview_curses.cc index 565e0e73..7c859900 100644 --- a/src/listview_curses.cc +++ b/src/listview_curses.cc @@ -147,7 +147,7 @@ bool listview_curses::handle_key(int ch) } break; - case 'A': + case ']': { double tenth = ((double)this->get_inner_height()) / 10.0; @@ -155,6 +155,7 @@ bool listview_curses::handle_key(int ch) } break; + case '[': case 'B': { double tenth = ((double)this->get_inner_height()) / 10.0; diff --git a/src/listview_curses.hh b/src/listview_curses.hh index f84bf10b..670c8d9d 100644 --- a/src/listview_curses.hh +++ b/src/listview_curses.hh @@ -351,14 +351,24 @@ public: */ void set_left(unsigned int left) { - if (left > this->get_inner_width()) { - alerter::singleton().chime(); + if (this->lv_left == left) { + return; } - else if (this->lv_left != left) { - this->lv_left = left; - this->lv_scroll.invoke(this); - this->lv_needs_update = true; + + if (left > this->lv_left) { + unsigned long width; + vis_line_t height; + + this->get_dimensions(height, width); + if ((this->get_inner_width() - this->lv_left) <= width) { + alerter::singleton().chime(); + return; + } } + + this->lv_left = left; + this->lv_scroll.invoke(this); + this->lv_needs_update = true; }; /** @return The column number that is displayed at the left. */ diff --git a/src/lnav.cc b/src/lnav.cc index 43f1afa4..da6657b9 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -141,19 +141,20 @@ static multimap DEFAULT_FILES; struct _lnav_data lnav_data; -struct hist_level { - int hl_time_slice; +const int ZOOM_LEVELS[] = { + 1, + 30, + 60, + 5 * 60, + 15 * 60, + 60 * 60, + 4 * 60 * 60, + 8 * 60 * 60, + 24 * 60 * 60, + 7 * 24 * 60 * 60, }; -static struct hist_level HIST_ZOOM_VALUES[] = { - { 24 * 60 * 60, }, - { 4 * 60 * 60, }, - { 60 * 60, }, - { 10 * 60, }, - { 60, }, -}; - -const int HIST_ZOOM_LEVELS = sizeof(HIST_ZOOM_VALUES) / sizeof(struct hist_level); +const size_t ZOOM_COUNT = sizeof(ZOOM_LEVELS) / sizeof(int); bookmark_type_t BM_QUERY("query"); @@ -167,18 +168,24 @@ const char *lnav_view_strings[LNV__MAX + 1] = { "example", "schema", "pretty", + "spectro", NULL }; const char *lnav_zoom_strings[] = { - "day", - "4-hour", - "hour", - "10-minute", - "minute", + "1-second", + "30-second", + "1-minute", + "5-minute", + "15-minute", + "1-hour", + "4-hour", + "8-hour", + "1-day", + "1-week", - NULL + NULL }; static const char *view_titles[LNV__MAX] = { @@ -191,6 +198,7 @@ static const char *view_titles[LNV__MAX] = { "EXAMPLE", "SCHEMA", "PRETTY", + "SPECTRO", }; class log_gutter_source : public list_gutter_source { @@ -439,9 +447,9 @@ void rebuild_hist(size_t old_count, bool force) { logfile_sub_source &lss = lnav_data.ld_log_source; hist_source2 &hs = lnav_data.ld_hist_source2; - int zoom = lnav_data.ld_hist_zoom; + int zoom = lnav_data.ld_zoom_level; - hs.set_time_slice(HIST_ZOOM_VALUES[zoom].hl_time_slice); + hs.set_time_slice(ZOOM_LEVELS[zoom]); lss.text_filters_changed(); } @@ -626,6 +634,8 @@ void rebuild_indexes(bool force) queue_request(start_line); lnav_data.ld_search_child[LNV_LOG]->get_grep_proc()->start(); } + + lnav_data.ld_view_stack.top()->reload_data(); } if (!lnav_data.ld_view_stack.empty()) { @@ -784,6 +794,7 @@ bool toggle_view(textview_curses *toggle_tc) require(toggle_tc < &lnav_data.ld_views[LNV__MAX]); if (tc == toggle_tc) { + lnav_data.ld_last_view = tc; lnav_data.ld_view_stack.pop(); } else { @@ -793,6 +804,7 @@ bool toggle_view(textview_curses *toggle_tc) else if (toggle_tc == &lnav_data.ld_views[LNV_PRETTY]) { open_pretty_view(); } + lnav_data.ld_last_view = NULL; lnav_data.ld_view_stack.push(toggle_tc); retval = true; } @@ -2540,6 +2552,9 @@ int main(int argc, char *argv[]) lnav_data.ld_db_overlay.dos_labels = &lnav_data.ld_db_row_source; lnav_data.ld_views[LNV_DB] .set_overlay_source(&lnav_data.ld_db_overlay); + lnav_data.ld_views[LNV_SPECTRO] + .set_sub_source(&lnav_data.ld_spectro_source) + .set_overlay_source(&lnav_data.ld_spectro_source); lnav_data.ld_match_view.set_left(0); @@ -2561,8 +2576,8 @@ int main(int argc, char *argv[]) new hist_index_delegate(lnav_data.ld_hist_source2, lnav_data.ld_views[LNV_HISTOGRAM])); hs.init(); - lnav_data.ld_hist_zoom = 2; - hs.set_time_slice(HIST_ZOOM_VALUES[lnav_data.ld_hist_zoom].hl_time_slice); + lnav_data.ld_zoom_level = 3; + hs.set_time_slice(ZOOM_LEVELS[lnav_data.ld_zoom_level]); } { diff --git a/src/lnav.hh b/src/lnav.hh index 60b27482..9fad86b4 100644 --- a/src/lnav.hh +++ b/src/lnav.hh @@ -63,6 +63,7 @@ #include "papertrail_proc.hh" #include "relative_time.hh" #include "log_format_loader.hh" +#include "spectro_source.hh" /** The command modes that are available while viewing a file. */ typedef enum { @@ -118,6 +119,7 @@ typedef enum { LNV_EXAMPLE, LNV_SCHEMA, LNV_PRETTY, + LNV_SPECTRO, LNV__MAX } lnav_view_t; @@ -215,6 +217,7 @@ struct _lnav_data { textview_curses ld_match_view; std::stack ld_view_stack; + textview_curses *ld_last_view; textview_curses ld_views[LNV__MAX]; std::auto_ptr ld_search_child[LNV__MAX]; vis_line_t ld_search_start_line; @@ -223,7 +226,8 @@ struct _lnav_data { logfile_sub_source ld_log_source; hist_source ld_hist_source; hist_source2 ld_hist_source2; - int ld_hist_zoom; + int ld_zoom_level; + spectrogram_source ld_spectro_source; textfile_sub_source ld_text_source; @@ -267,7 +271,8 @@ extern struct _lnav_data lnav_data; extern readline_context::command_map_t lnav_commands; extern bookmark_type_t BM_QUERY; -extern const int HIST_ZOOM_LEVELS; +extern const int ZOOM_LEVELS[]; +extern const size_t ZOOM_COUNT; #define HELP_MSG_1(x, msg) \ "Press '" ANSI_BOLD(#x) "' " msg diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index 5ff7a3d4..d2a9147d 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -1919,8 +1919,28 @@ static string com_zoom_to(string cmdline, vector &args) for (int lpc = 0; lnav_zoom_strings[lpc] && !found; lpc++) { if (strcasecmp(args[1].c_str(), lnav_zoom_strings[lpc]) == 0) { - lnav_data.ld_hist_zoom = lpc; + spectrogram_source &ss = lnav_data.ld_spectro_source; + time_t old_time; + + lnav_data.ld_zoom_level = lpc; + + old_time = lnav_data.ld_hist_source2.time_for_row( + lnav_data.ld_views[LNV_HISTOGRAM].get_top()); rebuild_hist(0, true); + lnav_data.ld_views[LNV_HISTOGRAM].set_top( + vis_line_t(lnav_data.ld_hist_source2.row_for_time(old_time))); + + old_time = lnav_data.ld_spectro_source.time_for_row( + lnav_data.ld_views[LNV_SPECTRO].get_top()); + ss.ss_granularity = ZOOM_LEVELS[lnav_data.ld_zoom_level]; + ss.invalidate(); + lnav_data.ld_views[LNV_SPECTRO].set_top( + vis_line_t(lnav_data.ld_spectro_source.row_for_time(old_time))); + + if (!lnav_data.ld_view_stack.empty()) { + lnav_data.ld_view_stack.top()->set_needs_update(); + } + found = true; } } @@ -2343,6 +2363,177 @@ static string com_reset_config(string cmdline, vector &args) return retval; } +class log_spectro_value_source : public spectrogram_value_source { +public: + log_spectro_value_source(intern_string_t colname) + : lsvs_colname(colname), + lsvs_begin_time(0), + lsvs_end_time(0), + lsvs_found(false) { + this->update_stats(); + }; + + void update_stats() { + logfile_sub_source &lss = lnav_data.ld_log_source; + logfile_sub_source::iterator iter; + + this->lsvs_begin_time = 0; + this->lsvs_end_time = 0; + this->lsvs_stats.clear(); + for (iter = lss.begin(); iter != lss.end(); iter++) { + logfile *lf = (*iter)->get_file(); + + if (lf == NULL) { + continue; + } + + log_format *format = lf->get_format(); + const logline_value_stats *stats = format->stats_for_value(this->lsvs_colname); + + if (stats == NULL) { + continue; + } + + logfile::iterator ll = lf->begin(); + + if (this->lsvs_begin_time == 0 || ll->get_time() < this->lsvs_begin_time) { + this->lsvs_begin_time = ll->get_time(); + } + ll = lf->end(); + --ll; + if (ll->get_time() > this->lsvs_end_time) { + this->lsvs_end_time = ll->get_time(); + } + + this->lsvs_found = true; + this->lsvs_stats.merge(*stats); + } + + if (this->lsvs_begin_time) { + time_t filtered_begin_time = lss.find_line(lss.at(vis_line_t(0)))->get_time(); + time_t filtered_end_time = lss.find_line(lss.at(vis_line_t(lss.text_line_count() - 1)))->get_time(); + + if (filtered_begin_time > this->lsvs_begin_time) { + this->lsvs_begin_time = filtered_begin_time; + } + if (filtered_end_time < this->lsvs_end_time) { + this->lsvs_end_time = filtered_end_time; + } + } + }; + + void spectro_bounds(spectrogram_bounds &sb_out) { + logfile_sub_source &lss = lnav_data.ld_log_source; + + if (lss.text_line_count() == 0) { + return; + } + + this->update_stats(); + + sb_out.sb_begin_time = this->lsvs_begin_time; + sb_out.sb_end_time = this->lsvs_end_time; + sb_out.sb_min_value_out = this->lsvs_stats.lvs_min_value; + sb_out.sb_max_value_out = this->lsvs_stats.lvs_max_value; + sb_out.sb_count = this->lsvs_stats.lvs_count; + }; + + void spectro_row(spectrogram_request &sr, spectrogram_row &row_out) { + logfile_sub_source &lss = lnav_data.ld_log_source; + vis_line_t begin_line = lss.find_from_time(sr.sr_begin_time); + vis_line_t end_line = lss.find_from_time(sr.sr_end_time); + vector values; + string_attrs_t sa; + + if (begin_line == -1) { + begin_line = vis_line_t(0); + } + if (end_line == -1) { + end_line = vis_line_t(lss.text_line_count()); + } + for (vis_line_t curr_line = begin_line; curr_line < end_line; ++curr_line) { + content_line_t cl = lss.at(curr_line); + logfile *lf = lss.find(cl); + logfile::iterator ll = lf->begin() + cl; + log_format *format = lf->get_format(); + shared_buffer_ref sbr; + + if (ll->is_continued()) { + continue; + } + + lf->read_full_message(ll, sbr); + sa.clear(); + values.clear(); + format->annotate(sbr, sa, values); + + vector::iterator lv_iter; + + lv_iter = find_if(values.begin(), values.end(), + logline_value_cmp(&this->lsvs_colname)); + + if (lv_iter != values.end()) { + switch (lv_iter->lv_kind) { + case logline_value::VALUE_FLOAT: + row_out.add_value(sr, lv_iter->lv_value.d); + break; + case logline_value::VALUE_INTEGER: + row_out.add_value(sr, lv_iter->lv_value.i); + break; + default: + break; + } + } + } + }; + + intern_string_t lsvs_colname; + logline_value_stats lsvs_stats; + time_t lsvs_begin_time; + time_t lsvs_end_time; + bool lsvs_found; +}; + +static string com_spectrogram(string cmdline, vector &args) +{ + string retval = "error: expecting a message field name"; + + if (args.empty()) { + args.push_back("numeric-colname"); + } + else if (lnav_data.ld_view_stack.top() != &lnav_data.ld_views[LNV_LOG] && + lnav_data.ld_view_stack.top() != &lnav_data.ld_views[LNV_SPECTRO]) { + retval = "error: this command can only be run from the log or spectrogram views"; + } + else if (args.size() == 2) { + intern_string_t colname = intern_string::lookup(remaining_args(cmdline, args)); + spectrogram_source &ss = lnav_data.ld_spectro_source; + + ss.ss_granularity = ZOOM_LEVELS[lnav_data.ld_zoom_level]; + if (ss.ss_value_source != NULL) { + delete ss.ss_value_source; + ss.ss_value_source = NULL; + } + ss.invalidate(); + + auto_ptr lsvs(new log_spectro_value_source(colname)); + + if (!lsvs->lsvs_found) { + retval = "error: unknown message field -- " + colname.to_string(); + } + else { + ss.ss_value_source = lsvs.release(); + ensure_view(&lnav_data.ld_views[LNV_SPECTRO]); + + lnav_data.ld_rl_view->set_alt_value(HELP_MSG_2(z, Z, "to zoom in/out")); + + retval = "info: visualizing field -- " + colname.to_string(); + } + } + + return retval; +} + readline_context::command_t STD_COMMANDS[] = { { "adjust-log-time", @@ -2667,6 +2858,12 @@ readline_context::command_t STD_COMMANDS[] = { "Reset the configuration option to its default value", com_reset_config, }, + { + "spectrogram", + "", + "Visualize the given message field using a spectrogram", + com_spectrogram, + }, { NULL }, }; diff --git a/src/log_format.cc b/src/log_format.cc index f0cad0a2..638654e9 100644 --- a/src/log_format.cc +++ b/src/log_format.cc @@ -738,6 +738,47 @@ log_format::scan_result_t external_log_format::scan(std::vector &dst, } } + size_t numeric_def_count = this->elf_numeric_value_defs.size(); + + for (size_t num_def_index = 0; + num_def_index < numeric_def_count; + num_def_index += 1) { + value_def &vd = *this->elf_numeric_value_defs[num_def_index]; + pcre_context::capture_t *num_cap = pc[vd.vd_index]; + + if (num_cap != NULL && num_cap->is_valid()) { + const struct scaling_factor *scaling = NULL; + + if (vd.vd_unit_field_index >= 0) { + pcre_context::iterator unit_cap = pc[vd.vd_unit_field_index]; + + if (unit_cap != NULL && unit_cap->is_valid()) { + intern_string_t unit_val = intern_string::lookup( + pi.get_substr_start(unit_cap), unit_cap->length()); + std::map::const_iterator unit_iter; + + unit_iter = vd.vd_unit_scaling.find(unit_val); + if (unit_iter != vd.vd_unit_scaling.end()) { + const struct scaling_factor &sf = unit_iter->second; + + scaling = &sf; + } + } + } + + char cap_copy[num_cap->length() + 1]; + double dvalue; + + pi.get_substr(num_cap, cap_copy); + if (sscanf(cap_copy, "%lf", &dvalue) == 1) { + if (scaling != NULL) { + scaling->scale(dvalue); + } + this->lf_value_stats[num_def_index].add_value(dvalue); + } + } + } + dst.push_back(logline(offset, log_tv, level, mod_index, opid)); this->lf_fmt_lock = curr_fmt; @@ -860,8 +901,9 @@ void external_log_format::annotate(shared_buffer_ref &line, pcre_context::iterator unit_cap = pc[vd.vd_unit_field_index]; if (unit_cap != NULL && unit_cap->c_begin != -1) { - std::string unit_val = pi.get_substr(unit_cap); - std::map::const_iterator unit_iter; + intern_string_t unit_val = intern_string::lookup( + pi.get_substr_start(unit_cap), unit_cap->length()); + map::const_iterator unit_iter; unit_iter = vd.vd_unit_scaling.find(unit_val); if (unit_iter != vd.vd_unit_scaling.end()) { @@ -1486,6 +1528,25 @@ void external_log_format::build(std::vector &errors) { } } } + + for (std::map::iterator iter = this->elf_value_defs.begin(); + iter != this->elf_value_defs.end(); + ++iter) { + if (iter->second.vd_foreign_key || iter->second.vd_identifier) { + continue; + } + + switch (iter->second.vd_kind) { + case logline_value::VALUE_INTEGER: + case logline_value::VALUE_FLOAT: + this->elf_numeric_value_defs.push_back(&iter->second); + break; + default: + break; + } + } + + this->lf_value_stats.resize(this->elf_numeric_value_defs.size()); } void external_log_format::register_vtabs(log_vtab_manager *vtab_manager, diff --git a/src/log_format.hh b/src/log_format.hh index a4e6bbc1..35b99587 100644 --- a/src/log_format.hh +++ b/src/log_format.hh @@ -538,6 +538,56 @@ public: const log_format *lv_format; }; +struct logline_value_stats { + + logline_value_stats() { + this->clear(); + }; + + void clear() { + this->lvs_count = 0; + this->lvs_total = 0; + this->lvs_min_value = std::numeric_limits::max(); + this->lvs_max_value = -std::numeric_limits::max(); + }; + + void merge(const logline_value_stats &other) { + if (other.lvs_count == 0) { + return; + } + + require(other.lvs_min_value <= other.lvs_max_value); + + if (other.lvs_min_value < this->lvs_min_value) { + this->lvs_min_value = other.lvs_min_value; + } + if (other.lvs_max_value > this->lvs_max_value) { + this->lvs_max_value = other.lvs_max_value; + } + this->lvs_count += other.lvs_count; + this->lvs_total += other.lvs_total; + + ensure(this->lvs_count >= 0); + ensure(this->lvs_min_value <= this->lvs_max_value); + }; + + void add_value(double value) { + if (value < this->lvs_min_value) { + this->lvs_min_value = value; + } + if (value > this->lvs_max_value) { + this->lvs_max_value = value; + } + this->lvs_count += 1; + this->lvs_total += value; + }; + + int64_t lvs_count; + double lvs_total; + double lvs_min_value; + double lvs_max_value; +}; + struct logline_value_cmp { logline_value_cmp(const intern_string_t *name = NULL, int col = -1) : lvc_name(name), lvc_column(col) { @@ -675,6 +725,10 @@ public: bool annotate_module = true) const { }; + virtual const logline_value_stats *stats_for_value(const intern_string_t &name) const { + return NULL; + }; + virtual std::auto_ptr specialized(int fmt_lock = -1) = 0; virtual log_vtab_impl *get_vtab_impl(void) const { @@ -723,6 +777,7 @@ public: intern_string_t lf_timestamp_field; std::vector lf_timestamp_format; std::map lf_action_defs; + std::vector lf_value_stats; protected: static std::vector lf_root_formats; @@ -778,7 +833,7 @@ public: bool vd_foreign_key; intern_string_t vd_unit_field; int vd_unit_field_index; - std::map vd_unit_scaling; + std::map vd_unit_scaling; int vd_column; bool vd_hidden; std::vector vd_action_list; @@ -886,6 +941,24 @@ public: this->jlf_cached_line.reserve(16 * 1024); } + this->lf_value_stats.clear(); + this->lf_value_stats.resize(this->elf_numeric_value_defs.size()); + + return retval; + }; + + const logline_value_stats *stats_for_value(const intern_string_t &name) const { + const logline_value_stats *retval = NULL; + + for (size_t lpc = 0; lpc < this->elf_numeric_value_defs.size(); lpc++) { + value_def &vd = *this->elf_numeric_value_defs[lpc]; + + if (vd.vd_name == name) { + retval = &this->lf_value_stats[lpc]; + break; + } + } + return retval; }; @@ -942,6 +1015,7 @@ public: std::vector elf_pattern_order; std::vector elf_samples; std::map elf_value_defs; + std::vector elf_numeric_value_defs; int elf_column_count; double elf_timestamp_divisor; intern_string_t elf_level_field; diff --git a/src/log_format_loader.cc b/src/log_format_loader.cc index 72f14700..2a424f9c 100644 --- a/src/log_format_loader.cc +++ b/src/log_format_loader.cc @@ -304,9 +304,9 @@ static int read_scaling(yajlpp_parse_context *ypc, double val) { external_log_format *elf = ensure_format(ypc); const intern_string_t value_name = ypc->get_path_fragment_i(2); - string scale_name = ypc->get_path_fragment(5); + string scale_spec = ypc->get_path_fragment(5); - if (scale_name.empty()) { + if (scale_spec.empty()) { fprintf(stderr, "error:%s:%s: scaling factor field cannot be empty\n", ypc->get_path_fragment(0).c_str(), @@ -314,12 +314,13 @@ static int read_scaling(yajlpp_parse_context *ypc, double val) return 0; } - struct scaling_factor &sf = elf->elf_value_defs[value_name].vd_unit_scaling[scale_name.substr(1)]; + const intern_string_t scale_name = intern_string::lookup(scale_spec.substr(1)); + struct scaling_factor &sf = elf->elf_value_defs[value_name].vd_unit_scaling[scale_name]; - if (scale_name[0] == '/') { + if (scale_spec[0] == '/') { sf.sf_op = SO_DIVIDE; } - else if (scale_name[0] == '*') { + else if (scale_spec[0] == '*') { sf.sf_op = SO_MULTIPLY; } else { diff --git a/src/log_vtab_impl.cc b/src/log_vtab_impl.cc index e70560eb..b5be6877 100644 --- a/src/log_vtab_impl.cc +++ b/src/log_vtab_impl.cc @@ -302,20 +302,26 @@ static int vt_column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int col) char buffer[64]; if (ll->is_time_skewed()) { - log_format *format = lf->get_format(); - shared_buffer_ref line; - vector dst; - - lf->read_line(ll, line); - switch (format->scan(dst, 0, line)) { - case log_format::SCAN_MATCH: - sql_strftime(buffer, sizeof(buffer), - dst.back().get_time(), - dst.back().get_millis()); - break; - default: - buffer[0] = '\0'; - break; + if (vc->line_values.empty()) { + lf->read_full_message(ll, vc->log_msg); + vt->vi->extract(lf, vc->log_msg, vc->line_values); + } + + struct line_range time_range; + + time_range = find_string_attr_range( + vt->vi->vi_attrs, &logline::L_TIMESTAMP); + + const char *time_src = vc->log_msg.get_data() + time_range.lr_start; + struct timeval actual_tv; + struct exttm tm; + + if (lf->get_format()->lf_date_time.scan( + time_src, time_range.length(), + lf->get_format()->get_timestamp_formats(), + &tm, actual_tv, + false)) { + sql_strftime(buffer, sizeof(buffer), actual_tv); } } else { diff --git a/src/logfile.cc b/src/logfile.cc index 42d9971e..ee289900 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -113,7 +113,8 @@ throw (error) } logfile::~logfile() -{ } +{ +} bool logfile::exists(void) const { @@ -272,7 +273,13 @@ throw (line_buffer::error, logfile::error) } /* Check for new data based on the file size. */ - if (this->lf_index_size < st.st_size) { + if (this->lf_index_size > st.st_size) { + log_info("truncated file detected, closing -- %s", + this->lf_filename.c_str()); + this->close(); + return false; + } + else if (this->lf_index_size < st.st_size) { bool has_format = this->lf_format.get() != NULL; shared_buffer_ref sbr; off_t last_off, off; diff --git a/src/logfile_sub_source.cc b/src/logfile_sub_source.cc index 7879a054..cbcfdd46 100644 --- a/src/logfile_sub_source.cc +++ b/src/logfile_sub_source.cc @@ -452,8 +452,9 @@ bool logfile_sub_source::rebuild_index(bool force) iter++) { logfile_data *ld = *iter; logfile *lf = ld->get_file(); - if (lf == NULL) + if (lf == NULL) { continue; + } merge.add(ld, lf->begin() + ld->ld_lines_indexed, @@ -519,8 +520,7 @@ bool logfile_sub_source::rebuild_index(bool force) if (!ld->ld_filter_state.excluded(filter_in_mask, filter_out_mask, line_number) && - line_iter->get_msg_level() >= - this->lss_min_log_level && + line_iter->get_msg_level() >= this->lss_min_log_level && !(*line_iter < this->lss_min_log_time) && *line_iter <= this->lss_max_log_time) { this->lss_filtered_index.push_back(index_index); @@ -651,11 +651,13 @@ void logfile_sub_source::text_filters_changed() content_line_t cl = (content_line_t) this->lss_index[index_index]; uint64_t line_number; logfile_data *ld = this->find_data(cl, line_number); + logfile::iterator line_iter = ld->get_file()->begin() + line_number; if (!ld->ld_filter_state.excluded(filtered_in_mask, filtered_out_mask, line_number) && - (*(ld->get_file()->begin() + line_number)).get_msg_level() >= - this->lss_min_log_level) { + line_iter->get_msg_level() >= this->lss_min_log_level && + !(*line_iter < this->lss_min_log_time) && + *line_iter <= this->lss_max_log_time) { this->lss_filtered_index.push_back(index_index); if (this->lss_index_delegate != NULL) { logfile *lf = ld->get_file(); diff --git a/src/logfile_sub_source.hh b/src/logfile_sub_source.hh index b1ea7456..4a266999 100644 --- a/src/logfile_sub_source.hh +++ b/src/logfile_sub_source.hh @@ -150,7 +150,7 @@ public: return this->lss_filtered_index.size(); }; - size_t text_line_width() { + size_t text_line_width(textview_curses &curses) { return this->lss_longest_line; }; diff --git a/src/plain_text_source.hh b/src/plain_text_source.hh index 24accca6..c1adfe29 100644 --- a/src/plain_text_source.hh +++ b/src/plain_text_source.hh @@ -61,7 +61,7 @@ public: return this->tds_lines.size(); }; - size_t text_line_width() { + size_t text_line_width(textview_curses &curses) { return this->tds_longest_line; }; diff --git a/src/spectro_source.hh b/src/spectro_source.hh new file mode 100644 index 00000000..8032513a --- /dev/null +++ b/src/spectro_source.hh @@ -0,0 +1,368 @@ +/** + * Copyright (c) 2016, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * @file spectroview_curses.hh + */ + +#ifndef __spectro_source_hh +#define __spectro_source_hh + +#include +#include + +#include +#include + +#include "textview_curses.hh" + +struct spectrogram_bounds { + spectrogram_bounds() + : sb_begin_time(0), + sb_end_time(0), + sb_min_value_out(0.0), + sb_max_value_out(0.0), + sb_count(0) { + + }; + + time_t sb_begin_time; + time_t sb_end_time; + double sb_min_value_out; + double sb_max_value_out; + int64_t sb_count; +}; + +struct spectrogram_thresholds { + int st_green_threshold; + int st_yellow_threshold; +}; + +struct spectrogram_request { + spectrogram_request(spectrogram_bounds &sb) + : sr_bounds(sb), sr_width(0), sr_column_size(0) { + }; + + spectrogram_bounds &sr_bounds; + unsigned long sr_width; + time_t sr_begin_time; + time_t sr_end_time; + double sr_column_size; +}; + +struct spectrogram_row { + spectrogram_row() : sr_values(NULL), sr_width(0) { + + }; + + ~spectrogram_row() { + delete this->sr_values; + } + + int *sr_values; + unsigned long sr_width; + double sr_column_size; + + void add_value(spectrogram_request &sr, double value) { + long index = lrint((value - sr.sr_bounds.sb_min_value_out) / sr.sr_column_size); + + this->sr_values[index] += 1; + }; +}; + +class spectrogram_value_source { +public: + virtual ~spectrogram_value_source() { }; + + virtual void spectro_bounds(spectrogram_bounds &sb_out) = 0; + + virtual void spectro_row(spectrogram_request &sr, + spectrogram_row &row_out) = 0; +}; + +class spectrogram_source + : public text_sub_source, + public text_time_translator, + public list_overlay_source { +public: + + spectrogram_source() + : ss_granularity(60), + ss_value_source(NULL) { + + }; + + void invalidate() { + this->ss_cached_bounds.sb_count = 0; + this->ss_row_cache.clear(); + }; + + size_t list_overlay_count(const listview_curses &lv) { + return 1; + }; + + bool list_value_for_overlay(const listview_curses &lv, + vis_line_t y, + attr_line_t &value_out) { + if (y != 0) { + return false; + } + + std::string &line = value_out.get_string(); + char buf[128]; + vis_line_t height; + unsigned long width; + + lv.get_dimensions(height, width); + + this->cache_bounds(); + + if (this->ss_cached_line_count == 0) { + value_out.with_ansi_string( + ANSI_ROLE("error: no log data"), + view_colors::VCR_ERROR); + return true; + } + + spectrogram_bounds &sb = this->ss_cached_bounds; + spectrogram_thresholds &st = this->ss_cached_thresholds; + + snprintf(buf, sizeof(buf), "Min: %'.10lg", sb.sb_min_value_out); + line = buf; + + snprintf(buf, sizeof(buf), + ANSI_ROLE(" ") " 1-%'d " + ANSI_ROLE(" ") " %'d-%'d " + ANSI_ROLE(" ") " %'d+", + view_colors::VCR_LOW_THRESHOLD, + st.st_green_threshold - 1, + view_colors::VCR_MED_THRESHOLD, + st.st_green_threshold, + st.st_yellow_threshold - 1, + view_colors::VCR_HIGH_THRESHOLD, + st.st_yellow_threshold); + line.append(width / 2 - strlen(buf) / 3 - line.length(), ' '); + line.append(buf); + scrub_ansi_string(line, value_out.get_attrs()); + + snprintf(buf, sizeof(buf), "Max: %'.10lg", sb.sb_max_value_out); + line.append(width - strlen(buf) - line.length() - 2, ' '); + line.append(buf); + + value_out.with_attr(string_attr( + line_range(0, -1), + &view_curses::VC_STYLE, + A_UNDERLINE)); + + return true; + }; + + size_t text_line_count() { + if (this->ss_value_source == NULL) { + return 0; + } + + this->cache_bounds(); + + return this->ss_cached_line_count; + }; + + size_t text_line_width(textview_curses &tc) { + unsigned long width; + vis_line_t height; + + tc.get_dimensions(height, width); + return width; + }; + + size_t text_size_for_line(textview_curses &tc, int row, bool raw) { + return 0; + }; + + time_t time_for_row(int row) { + time_t retval; + + this->cache_bounds(); + retval = rounddown(this->ss_cached_bounds.sb_begin_time, this->ss_granularity) + + row * this->ss_granularity; + + return retval; + } + + int row_for_time(time_t time_bucket) { + if (this->ss_value_source == NULL) { + return 0; + } + + time_t diff; + int retval; + + this->cache_bounds(); + if (time_bucket < this->ss_cached_bounds.sb_begin_time) { + return 0; + } + + diff = time_bucket - this->ss_cached_bounds.sb_begin_time; + retval = diff / this->ss_granularity; + + return retval; + } + + void text_value_for_line(textview_curses &tc, + int row, + std::string &value_out, + bool no_scrub) { + time_t row_time; + char tm_buffer[128]; + struct tm tm; + + row_time = this->time_for_row(row); + + gmtime_r(&row_time, &tm); + strftime(tm_buffer, sizeof(tm_buffer), " %a %b %d %H:%M:%S", &tm); + + value_out = tm_buffer; + }; + + void text_attrs_for_line(textview_curses &tc, + int row, + string_attrs_t &value_out) { + if (this->ss_value_source == NULL) { + return; + } + + this->cache_bounds(); + + view_colors &vc = view_colors::singleton(); + unsigned long width; + vis_line_t height; + + tc.get_dimensions(height, width); + width -= 2; + + spectrogram_bounds &sb = this->ss_cached_bounds; + spectrogram_thresholds &st = this->ss_cached_thresholds; + spectrogram_request sr(sb); + time_t row_time; + + sr.sr_width = width; + row_time = rounddown(sb.sb_begin_time, this->ss_granularity) + + row * this->ss_granularity; + sr.sr_begin_time = row_time; + sr.sr_end_time = row_time + this->ss_granularity; + + sr.sr_column_size = (sb.sb_max_value_out - sb.sb_min_value_out) / + (double) (width - 1); + + spectrogram_row &s_row = this->ss_row_cache[row_time]; + + if (s_row.sr_values == NULL || + s_row.sr_width != width || + s_row.sr_column_size != sr.sr_column_size) { + s_row.sr_width = width; + s_row.sr_column_size = sr.sr_column_size; + delete s_row.sr_values; + s_row.sr_values = new int[width + 1]; + memset(s_row.sr_values, 0, sizeof(int) * (width + 1)); + this->ss_value_source->spectro_row(sr, s_row); + } + + for (int lpc = 0; lpc <= width; lpc++) { + int col_value = s_row.sr_values[lpc]; + + if (col_value == 0) { + continue; + } + + int color; + + if (col_value < st.st_green_threshold) { + color = COLOR_GREEN; + } + else if (col_value < st.st_yellow_threshold) { + color = COLOR_YELLOW; + } + else { + color = COLOR_RED; + } + value_out.push_back(string_attr( + line_range(lpc, lpc + 1), + &view_curses::VC_STYLE, + vc.ansi_color_pair(COLOR_BLACK, color) + )); + } + }; + + void cache_bounds() { + if (this->ss_value_source == NULL) { + this->ss_cached_bounds.sb_count = 0; + this->ss_cached_bounds.sb_begin_time = 0; + return; + } + + spectrogram_bounds sb; + + this->ss_value_source->spectro_bounds(sb); + + if (sb.sb_count == this->ss_cached_bounds.sb_count) { + return; + } + + this->ss_cached_bounds = sb; + + if (sb.sb_count == 0) { + this->ss_cached_line_count = 0; + return; + } + + time_t diff = std::max((time_t) 1, sb.sb_end_time - sb.sb_begin_time + 1); + this->ss_cached_line_count = + (diff + this->ss_granularity - 1) / this->ss_granularity; + + int64_t samples_per_row = sb.sb_count / this->ss_cached_line_count; + spectrogram_thresholds &st = this->ss_cached_thresholds; + + st.st_yellow_threshold = samples_per_row / 2; + st.st_green_threshold = st.st_yellow_threshold / 2; + + if (st.st_green_threshold <= 1) { + st.st_green_threshold = 2; + } + if (st.st_yellow_threshold <= st.st_green_threshold) { + st.st_yellow_threshold = st.st_green_threshold + 1; + } + }; + + int ss_granularity; + spectrogram_value_source *ss_value_source; + spectrogram_bounds ss_cached_bounds; + spectrogram_thresholds ss_cached_thresholds; + size_t ss_cached_line_count; + std::map ss_row_cache; +}; + +#endif diff --git a/src/textfile_sub_source.hh b/src/textfile_sub_source.hh index faf9ea08..4af017c7 100644 --- a/src/textfile_sub_source.hh +++ b/src/textfile_sub_source.hh @@ -63,7 +63,7 @@ public: return retval; }; - size_t text_line_width() { + size_t text_line_width(textview_curses &curses) { return this->tss_files.empty() ? 0 : this->current_file()->get_longest_line_length(); }; diff --git a/src/textview_curses.hh b/src/textview_curses.hh index ca756b99..cb7e6ecb 100644 --- a/src/textview_curses.hh +++ b/src/textview_curses.hh @@ -299,6 +299,15 @@ private: std::vector fs_filters; }; +class text_time_translator { +public: + virtual ~text_time_translator() { }; + + virtual int row_for_time(time_t time_bucket) = 0; + + virtual time_t time_for_row(int row) = 0; +}; + /** * Source for the text to be shown in a textview_curses view. */ @@ -313,7 +322,7 @@ public: */ virtual size_t text_line_count() = 0; - virtual size_t text_line_width() { + virtual size_t text_line_width(textview_curses &curses) { return INT_MAX; }; @@ -656,7 +665,7 @@ public: size_t listview_width(const listview_curses &lv) { return this->tc_sub_source == NULL ? 0 : - this->tc_sub_source->text_line_width(); + this->tc_sub_source->text_line_width(*this); }; void listview_value_for_row(const listview_curses &lv, diff --git a/src/time-extension-functions.cc b/src/time-extension-functions.cc index d3de1921..ff4f6a26 100644 --- a/src/time-extension-functions.cc +++ b/src/time-extension-functions.cc @@ -110,10 +110,6 @@ int time_extension_functions(const struct FuncDef **basic_funcs, static const struct FuncDef time_funcs[] = { { "timeslice", 2, 0, SQLITE_UTF8, 0, timeslice }, - /* - * TODO: add other functions like readlink, normpath, ... - */ - { NULL } }; diff --git a/src/top_status_source.hh b/src/top_status_source.hh index c3afc10f..24bf24ee 100644 --- a/src/top_status_source.hh +++ b/src/top_status_source.hh @@ -60,7 +60,8 @@ public: { this->tss_fields[TSF_TIME].set_width(24); this->tss_fields[TSF_PARTITION_NAME].set_width(34); - this->tss_fields[TSF_VIEW_NAME].set_width(6); + this->tss_fields[TSF_VIEW_NAME].set_width(8); + this->tss_fields[TSF_VIEW_NAME].set_role(view_colors::VCR_VIEW_STATUS); this->tss_fields[TSF_VIEW_NAME].right_justify(true); this->tss_fields[TSF_STITCH_VIEW_FORMAT].set_width(2); this->tss_fields[TSF_STITCH_VIEW_FORMAT].set_stitch_value( diff --git a/src/view_curses.cc b/src/view_curses.cc index 6a8b04a6..cb7a7eab 100644 --- a/src/view_curses.cc +++ b/src/view_curses.cc @@ -342,6 +342,8 @@ void view_colors::init_roles(void) ansi_color_pair(COLOR_GREEN, COLOR_WHITE) | A_BOLD; this->vc_role_colors[VCR_BOLD_STATUS] = ansi_color_pair(COLOR_BLACK, COLOR_WHITE) | A_BOLD; + this->vc_role_colors[VCR_VIEW_STATUS] = + ansi_color_pair(COLOR_WHITE, COLOR_BLUE); this->vc_role_colors[VCR_KEYWORD] = ansi_color_pair(COLOR_BLUE, COLOR_BLACK); this->vc_role_colors[VCR_STRING] = ansi_color_pair(COLOR_GREEN, COLOR_BLACK) | A_BOLD; @@ -352,6 +354,10 @@ void view_colors::init_roles(void) this->vc_role_colors[VCR_DIFF_ADD] = ansi_color_pair(COLOR_GREEN, COLOR_BLACK); this->vc_role_colors[VCR_DIFF_SECTION] = ansi_color_pair(COLOR_MAGENTA, COLOR_BLACK); + this->vc_role_colors[VCR_LOW_THRESHOLD] = ansi_color_pair(COLOR_BLACK, COLOR_GREEN); + this->vc_role_colors[VCR_MED_THRESHOLD] = ansi_color_pair(COLOR_BLACK, COLOR_YELLOW); + this->vc_role_colors[VCR_HIGH_THRESHOLD] = ansi_color_pair(COLOR_BLACK, COLOR_RED); + for (lpc = 0; lpc < VCR_HIGHLIGHT_START; lpc++) { this->vc_role_reverse_colors[lpc] = this->vc_role_colors[lpc] | A_REVERSE; diff --git a/src/view_curses.hh b/src/view_curses.hh index c8a14c9d..2aabfb9a 100644 --- a/src/view_curses.hh +++ b/src/view_curses.hh @@ -544,6 +544,7 @@ public: VCR_ACTIVE_STATUS, /*< */ VCR_ACTIVE_STATUS2, /*< */ VCR_BOLD_STATUS, + VCR_VIEW_STATUS, VCR_KEYWORD, VCR_STRING, @@ -554,6 +555,10 @@ public: VCR_DIFF_ADD, /*< Added line in a diff. */ VCR_DIFF_SECTION, /*< Section marker in a diff. */ + VCR_LOW_THRESHOLD, + VCR_MED_THRESHOLD, + VCR_HIGH_THRESHOLD, + VCR_HIGHLIGHT_START, VCR_HIGHLIGHT_END = VCR_HIGHLIGHT_START + HL_COLOR_COUNT, diff --git a/test/logfile_uwsgi.0 b/test/logfile_uwsgi.0 new file mode 100644 index 00000000..005802bc --- /dev/null +++ b/test/logfile_uwsgi.0 @@ -0,0 +1,19 @@ +[pid: 88185|app: 0|req: 1/1] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:49:12 2016] POST /update_metrics => generated 47 bytes in 129 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 3) +[pid: 88185|app: 0|req: 3/2] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:49:15 2016] POST /update_metrics => generated 47 bytes in 35 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 30) +[pid: 88185|app: 0|req: 3/3] 127.0.0.1 () {34 vars in 617 bytes} [Sun Mar 13 22:49:15 2016] POST /endpoint2 => generated 215 bytes in 68 msecs (HTTP/1.1 200) 9 headers in 373 bytes (1 switches on core 8) +[pid: 88185|app: 0|req: 4/4] 127.0.0.1 () {34 vars in 617 bytes} [Sun Mar 13 22:49:15 2016] POST /endpoint2 => generated 215 bytes in 16 msecs (HTTP/1.1 200) 9 headers in 373 bytes (1 switches on core 22) +[pid: 88185|app: 0|req: 5/5] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:50:12 2016] POST /update_metrics => generated 47 bytes in 10 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 0) +[pid: 88186|app: 0|req: 1/6] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:50:15 2016] POST /update_metrics => generated 47 bytes in 65 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 16) +[pid: 88186|app: 0|req: 2/7] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:51:12 2016] POST /update_metrics => generated 47 bytes in 11 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 30) +[pid: 88188|app: 0|req: 1/8] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:51:15 2016] POST /update_metrics => generated 47 bytes in 66 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 31) +[pid: 88186|app: 0|req: 3/9] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:52:12 2016] POST /update_metrics => generated 47 bytes in 11 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 81) +[pid: 88188|app: 0|req: 2/10] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:52:15 2016] POST /update_metrics => generated 47 bytes in 18 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 38) +[pid: 88187|app: 0|req: 1/11] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:53:12 2016] POST /update_metrics => generated 47 bytes in 107 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 7) +[pid: 88187|app: 0|req: 2/12] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:53:15 2016] POST /update_metrics => generated 47 bytes in 17 msecs (HTTP/1.1 200) 9 headers in 378 bytes (2 switches on core 8) +[pid: 88187|app: 0|req: 3/13] 127.0.0.1 () {38 vars in 695 bytes} [Sun Mar 13 22:54:12 2016] POST /update_metrics => generated 47 bytes in 16 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 9) +[pid: 88188|app: 0|req: 3/14] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:54:15 2016] POST /update_metrics => generated 47 bytes in 52 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 1) +[pid: 88186|app: 0|req: 4/15] 127.0.0.1 () {34 vars in 617 bytes} [Sun Mar 13 22:54:15 2016] POST /endpoint2 => generated 215 bytes in 35 msecs (HTTP/1.1 200) 9 headers in 373 bytes (1 switches on core 43) +[pid: 88187|app: 0|req: 4/16] 127.0.0.1 () {38 vars in 695 bytes} [Sun Mar 13 22:55:12 2016] POST /update_metrics => generated 47 bytes in 11 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 14) +[pid: 88188|app: 0|req: 4/17] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:55:15 2016] POST /update_metrics => generated 47 bytes in 14 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 57) +[pid: 88187|app: 0|req: 5/18] 127.0.0.1 () {38 vars in 695 bytes} [Sun Mar 13 22:56:12 2016] POST /update_metrics => generated 47 bytes in 11 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 35) +[pid: 88186|app: 0|req: 5/19] 127.0.0.1 () {38 vars in 696 bytes} [Sun Mar 13 22:56:15 2016] POST /update_metrics => generated 47 bytes in 40 msecs (HTTP/1.1 200) 9 headers in 378 bytes (1 switches on core 60) diff --git a/test/test_cmds.sh b/test/test_cmds.sh index 20b5150e..8c98fcf1 100644 --- a/test/test_cmds.sh +++ b/test/test_cmds.sh @@ -658,16 +658,16 @@ run_test ${lnav_test} -n \ ${test_dir}/logfile_syslog.0 check_output "histogram is not working?" <