[filters] add :toggle-filtering command

NEWS

@@ -9,6 +9,12 @@ lnav v0.9.1:
      * To jump to the first search hit above the top line in a view, you can
        press CTRL+J instead of ENTER in the search prompt.  Pressing ENTER
        will jump to the first hit below the current window.
+     * Filtering, as a whole, can be now disabled/enabled without affecting
+       the state of individual filters.  This includes text and time-filters
+       (i.e. :hide-lines-before).  You can enable/disable filtering by:
+       pressing 'f' in the filter editor UI; executing the ':toggle-filtering'
+       command; or by doing an UPDATE on the "filtering" column of the
+       "lnav_views" SQLite table.
 
 lnav v0.9.0:
      Features:

src/ansi_scrubber.hh

@@ -43,6 +43,8 @@
 #define ANSI_BOLD_START       ANSI_CSI ANSI_BOLD_PARAM ANSI_CHAR_ATTR
 #define ANSI_UNDERLINE_START  ANSI_CSI "4m"
 #define ANSI_NORM             ANSI_CSI "0m"
+#define ANSI_STRIKE_PARAM     "9"
+#define ANSI_STRIKE_START     ANSI_CSI ANSI_STRIKE_PARAM ANSI_CHAR_ATTR
 
 #define ANSI_BOLD(msg)  ANSI_BOLD_START msg ANSI_NORM
 #define ANSI_UNDERLINE(msg)  ANSI_UNDERLINE_START msg ANSI_NORM

src/filter_status_source.cc

@@ -35,12 +35,12 @@
 static auto TOGGLE_MSG = "Press " ANSI_BOLD("TAB") " to edit ";
 static auto EXIT_MSG = "Press " ANSI_BOLD("TAB") " to exit ";
 
-static auto HOTKEY_HELP = " "
-    ANSI_BOLD("SPC") ": Enable/Disable  "
-    ANSI_BOLD("i") "/" ANSI_BOLD("o") ": Create in/out  "
-    ANSI_BOLD("ENTER") ": Edit  "
-    ANSI_BOLD("t") ": Toggle type  "
-    ANSI_BOLD("D") ": Delete ";
+static auto CREATE_HELP = ANSI_BOLD("i") "/" ANSI_BOLD("o") ": Create in/out";
+static auto ENABLE_HELP = ANSI_BOLD("SPC") ": ";
+static auto EDIT_HELP = ANSI_BOLD("ENTER") ": Edit";
+static auto TOGGLE_HELP = ANSI_BOLD("t") ": To ";
+static auto DELETE_HELP = ANSI_BOLD("D") ": Delete";
+static auto FILTERING_HELP = ANSI_BOLD("f") ": ";
 
 filter_status_source::filter_status_source()
 {
@@ -104,7 +104,9 @@ size_t filter_status_source::statusview_fields()
                 this->tss_fields[TSF_COUNT].set_value(
                     " " ANSI_BOLD("%d")
                     " of " ANSI_BOLD("%d")
-                    " enabled ", enabled_count, filter_count);
+                    " enabled ",
+                    enabled_count,
+                    filter_count);
             }
         };
 
@@ -136,7 +138,13 @@ void filter_status_source::update_filtered(text_sub_source *tss)
     status_field &sf = this->tss_fields[TSF_FILTERED];
 
     if (tss == nullptr || tss->get_filtered_count() == 0) {
-        sf.clear();
+        if (tss->tss_apply_filters) {
+            sf.clear();
+        } else {
+            sf.set_value(
+                " \u2718 Filtering disabled, re-enable with "
+                ANSI_BOLD_START ":toggle-filtering" ANSI_NORM);
+        }
     }
     else {
         ui_periodic_timer &timer = ui_periodic_timer::singleton();
@@ -163,11 +171,57 @@ filter_help_status_source::filter_help_status_source()
 {
     this->fss_help.set_min_width(10);
     this->fss_help.set_share(1);
-    this->fss_help.set_value(HOTKEY_HELP);
 }
 
 size_t filter_help_status_source::statusview_fields()
 {
+    lnav_data.ld_view_stack.top() | [this] (auto tc) {
+        text_sub_source *tss = tc->get_sub_source();
+        if (tss == nullptr) {
+            return;
+        }
+
+        if (lnav_data.ld_mode != LNM_FILTER) {
+            return;
+        }
+
+        auto &editor = lnav_data.ld_filter_source;
+        auto &lv = lnav_data.ld_filter_view;
+        auto &fs = tss->get_filters();
+
+        if (editor.fss_editing) {
+            auto tf = *(fs.begin() + lv.get_selection());
+
+            if (tf->get_type() == text_filter::type_t::INCLUDE) {
+                this->fss_help.set_value(
+                    "                     "
+                    "Enter a regular expression to match lines to filter in:");
+            } else {
+                this->fss_help.set_value(
+                    "                     "
+                    "Enter a regular expression to match lines to filter out:");
+            }
+        } else if (fs.empty()) {
+            this->fss_help.set_value("  %s", CREATE_HELP);
+        } else {
+            auto tf = *(fs.begin() + lv.get_selection());
+
+            this->fss_help.set_value("  %s  %s%s  %s  %s%s  %s  %s%s",
+                                     CREATE_HELP,
+                                     ENABLE_HELP,
+                                     tf->is_enabled() ? "Disable" : "Enable ",
+                                     EDIT_HELP,
+                                     TOGGLE_HELP,
+                                     tf->get_type() == text_filter::type_t::INCLUDE ?
+                                     "OUT" : "IN ",
+                                     DELETE_HELP,
+                                     FILTERING_HELP,
+                                     tss->tss_apply_filters ?
+                                     "Disable Filtering" :
+                                     "Enable Filtering");
+        }
+    };
+
     return 1;
 }
 

src/filter_sub_source.cc

@@ -78,6 +78,13 @@ bool filter_sub_source::list_input_handle_key(listview_curses &lv, int ch)
             lnav_data.ld_mode = LNM_PAGING;
             lnav_data.ld_filter_view.reload_data();
             return true;
+        case 'f': {
+            auto top_view = *lnav_data.ld_view_stack.top();
+            auto tss = top_view->get_sub_source();
+
+            tss->toggle_apply_filters();
+            break;
+        }
         case ' ': {
             textview_curses *top_view = *lnav_data.ld_view_stack.top();
             text_sub_source *tss = top_view->get_sub_source();
@@ -148,8 +155,6 @@ bool filter_sub_source::list_input_handle_key(listview_curses &lv, int ch)
             this->fss_editing = true;
 
             add_view_text_possibilities(&this->fss_editor, LNM_FILTER, "*", top_view);
-            lnav_data.ld_filter_status_source.tss_prompt.set_value(
-                "Enter a regular expression to match lines to filter in:");
             this->fss_editor.set_window(lv.get_window());
             this->fss_editor.set_visible(true);
             this->fss_editor.set_y(lv.get_y() + (int) (lv.get_selection() - lv.get_top()));
@@ -177,8 +182,6 @@ bool filter_sub_source::list_input_handle_key(listview_curses &lv, int ch)
             this->fss_editing = true;
 
             add_view_text_possibilities(&this->fss_editor, LNM_FILTER, "*", top_view);
-            lnav_data.ld_filter_status_source.tss_prompt.set_value(
-                "Enter a regular expression to match lines to filter out:");
             this->fss_editor.set_window(lv.get_window());
             this->fss_editor.set_visible(true);
             this->fss_editor.set_y(lv.get_y() + (int) (lv.get_selection() - lv.get_top()));

src/filter_sub_source.hh

@@ -80,7 +80,7 @@ public:
     fss_functor_t fss_abort_wire;
     fss_functor_t fss_display_match_wire;
     fss_functor_t fss_display_next_wire;
-    bool fss_filter_state;
+    bool fss_filter_state{false};
 };
 
 #endif

src/internals/cmd-ref.rst

@@ -1324,6 +1324,17 @@
 ----
 
 
+.. _toggle_filtering:
+
+:toggle-filtering
+^^^^^^^^^^^^^^^^^
+
+  Toggle the filtering flag for the current view
+
+
+----
+
+
 .. _toggle_view:
 
 :toggle-view *view-name*

src/lnav_commands.cc

@@ -2733,6 +2733,22 @@ static Result<string, string> com_switch_to_view(exec_context &ec, string cmdlin
     return Ok(retval);
 }
 
+static Result<string, string> com_toggle_filtering(exec_context &ec, string cmdline, vector<string> &args)
+{
+    string retval;
+
+    if (args.empty()) {
+    }
+    else if (!ec.ec_dry_run) {
+        auto tc = *lnav_data.ld_view_stack.top();
+        auto tss = tc->get_sub_source();
+
+        tss->toggle_apply_filters();
+    }
+
+    return Ok(retval);
+}
+
 static Result<string, string> com_zoom_to(exec_context &ec, string cmdline, vector<string> &args)
 {
     string retval;
@@ -4827,6 +4843,13 @@ readline_context::command_t STD_COMMANDS[] = {
                 "schema"
             })
     },
+    {
+        "toggle-filtering",
+        com_toggle_filtering,
+
+        help_text(":toggle-filtering")
+            .with_summary("Toggle the filtering flag for the current view")
+    },
     {
         "reset-session",
         com_reset_session,

src/logfile_sub_source.cc

@@ -113,9 +113,6 @@ logfile_sub_source::logfile_sub_source()
     this->clear_min_max_log_times();
 }
 
-logfile_sub_source::~logfile_sub_source()
-{ }
-
 shared_ptr<logfile> logfile_sub_source::find(const char *fn,
                                   content_line_t &line_base)
 {
@@ -740,8 +737,10 @@ logfile_sub_source::rebuild_result logfile_sub_source::rebuild_index()
             logfile_data *ld = this->find_data(cl, line_number);
             auto line_iter = ld->get_file()->begin() + line_number;
 
-            if (!ld->ld_filter_state.excluded(filter_in_mask, filter_out_mask,
-                    line_number) && this->check_extra_filters(*line_iter)) {
+            if (!this->tss_apply_filters ||
+                (!ld->ld_filter_state.excluded(filter_in_mask, filter_out_mask,
+                                               line_number) &&
+                 this->check_extra_filters(*line_iter))) {
                 this->lss_filtered_index.push_back(index_index);
                 if (this->lss_index_delegate != NULL) {
                     shared_ptr<logfile> lf = ld->get_file();
@@ -879,8 +878,10 @@ void logfile_sub_source::text_filters_changed()
         logfile_data *ld = this->find_data(cl, line_number);
         auto line_iter = ld->get_file()->begin() + line_number;
 
-        if (!ld->ld_filter_state.excluded(filtered_in_mask, filtered_out_mask,
-                line_number) && this->check_extra_filters(*line_iter)) {
+        if (!this->tss_apply_filters ||
+            (!ld->ld_filter_state.excluded(filtered_in_mask, filtered_out_mask,
+                                           line_number) &&
+             this->check_extra_filters(*line_iter))) {
             this->lss_filtered_index.push_back(index_index);
             if (this->lss_index_delegate != nullptr) {
                 shared_ptr<logfile> lf = ld->get_file();

src/logfile_sub_source.hh

@@ -79,7 +79,7 @@ public:
         : text_filter(type, id, index),
           pf_pcre(code) { };
 
-    ~pcre_filter() override { };
+    ~pcre_filter() override = default;
 
     bool matches(const logfile &lf, const logline &ll, shared_buffer_ref &line) override {
         pcre_context_static<30> pc;
@@ -139,7 +139,7 @@ public:
     virtual void text_filters_changed();
 
     logfile_sub_source();
-    virtual ~logfile_sub_source();
+    virtual ~logfile_sub_source() = default;
 
     void toggle_time_offset() {
         this->lss_flags ^= F_TIME_OFFSET;

src/session_data.cc

@@ -790,6 +790,27 @@ static int read_word_wrap(yajlpp_parse_context *ypc, int value)
     return 1;
 }
 
+static int read_filtering(yajlpp_parse_context *ypc, int value)
+{
+    const char **view_name;
+    int view_index;
+
+    view_name = find(lnav_view_strings,
+                     lnav_view_strings + LNV__MAX,
+                     ypc->get_path_fragment(-2));
+    view_index = view_name - lnav_view_strings;
+    if (view_index == LNV_HELP) {
+
+    }
+    else if (view_index < LNV__MAX) {
+        textview_curses &tc = lnav_data.ld_views[view_index];
+
+        tc.get_sub_source()->tss_apply_filters = value;
+    }
+
+    return 1;
+}
+
 static int read_commands(yajlpp_parse_context *ypc, const unsigned char *str, size_t len)
 {
     std::string cmdline = std::string((const char *)str, len);
@@ -813,6 +834,7 @@ static struct json_path_container view_def_handlers = {
     json_path_handler("top_line",  read_top_line),
     json_path_handler("search",    read_last_search),
     json_path_handler("word_wrap", read_word_wrap),
+    json_path_handler("filtering", read_filtering),
     json_path_handler("commands#", read_commands)
 };
 
@@ -1287,11 +1309,14 @@ static void save_session_with_id(const std::string session_id)
                     view_map.gen("word_wrap");
                     view_map.gen(tc.get_word_wrap());
 
-                    text_sub_source *tss = tc.get_sub_source();
+                    auto tss = tc.get_sub_source();
                     if (tss == nullptr) {
                         continue;
                     }
 
+                    view_map.gen("filtering");
+                    view_map.gen(tss->tss_apply_filters);
+
                     filter_stack &fs = tss->get_filters();
 
                     view_map.gen("commands");
@@ -1434,6 +1459,7 @@ void reset_session()
             continue;
         }
         tss->get_filters().clear_filters();
+        tss->tss_apply_filters = true;
         tss->text_filters_changed();
         tss->text_clear_marks(&textview_curses::BM_USER);
         tc.get_bookmarks()[&textview_curses::BM_USER].clear();

src/textfile_sub_source.hh

@@ -215,7 +215,8 @@ public:
                 this->get_filters().get_enabled_mask(filter_in_mask, filter_out_mask);
                 line_filter_observer *lfo = (line_filter_observer *) lf->get_logline_observer();
                 for (uint32_t lpc = old_size; lpc < lf->size(); lpc++) {
-                    if (lfo->excluded(filter_in_mask, filter_out_mask, lpc)) {
+                    if (this->tss_apply_filters &&
+                        lfo->excluded(filter_in_mask, filter_out_mask, lpc)) {
                         continue;
                     }
                     lfo->lfo_filter_state.tfs_index.push_back(lpc);
@@ -255,7 +256,8 @@ public:
         this->get_filters().get_enabled_mask(filter_in_mask, filter_out_mask);
         lfo->lfo_filter_state.tfs_index.clear();
         for (uint32_t lpc = 0; lpc < lf->size(); lpc++) {
-            if (lfo->excluded(filter_in_mask, filter_out_mask, lpc)) {
+            if (this->tss_apply_filters &&
+                lfo->excluded(filter_in_mask, filter_out_mask, lpc)) {
                 continue;
             }
             lfo->lfo_filter_state.tfs_index.push_back(lpc);

src/textview_curses.hh

@@ -486,7 +486,13 @@ public:
         return nonstd::nullopt;
     }
 
+    void toggle_apply_filters() {
+        this->tss_apply_filters = !this->tss_apply_filters;
+        this->text_filters_changed();
+    }
+
     bool tss_supports_filtering{false};
+    bool tss_apply_filters{true};
 protected:
     textview_curses *tss_view{nullptr};
     filter_stack tss_filters;

src/views_vtab.cc

@@ -130,7 +130,8 @@ CREATE TABLE lnav_views (
     inner_height INTEGER,   -- The number of lines in the view.
     top_time DATETIME,      -- The time of the top line in the view, if the content is time-based.
     paused INTEGER,         -- Indicates if the view is paused and will not load new data.
-    search TEXT             -- The text to search for in the view.
+    search TEXT,            -- The text to search for in the view.
+    filtering INTEGER       -- Indicates if the view is applying filters.
 );
 )";
 
@@ -192,6 +193,16 @@ CREATE TABLE lnav_views (
                 sqlite3_result_text(ctx, str.c_str(), str.length(), SQLITE_TRANSIENT);
                 break;
             }
+            case 8: {
+                auto tss = tc.get_sub_source();
+
+                if (tss != nullptr && tss->tss_supports_filtering) {
+                    sqlite3_result_int(ctx, tss->tss_apply_filters);
+                } else {
+                    sqlite3_result_int(ctx, 0);
+                }
+                break;
+            }
         }
 
         return SQLITE_OK;
@@ -218,7 +229,8 @@ CREATE TABLE lnav_views (
                    int64_t inner_height,
                    const char *top_time,
                    bool is_paused,
-                   const char *search) {
+                   const char *search,
+                   bool do_filtering) {
         textview_curses &tc = lnav_data.ld_views[index];
         text_time_translator *time_source = dynamic_cast<text_time_translator *>(tc.get_sub_source());
 
@@ -244,6 +256,13 @@ CREATE TABLE lnav_views (
         tc.set_left(left);
         tc.set_paused(is_paused);
         tc.execute_search(search);
+        auto tss = tc.get_sub_source();
+        if (tss != nullptr &&
+            tss->tss_supports_filtering &&
+            tss->tss_apply_filters != do_filtering) {
+            tss->tss_apply_filters = do_filtering;
+            tss->text_filters_changed();
+        }
 
         return SQLITE_OK;
     };

test/test_cmds.sh

@@ -36,6 +36,18 @@ check_output "filter-in vmk is not working" <<EOF
 192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkernel.gz HTTP/1.0" 200 78929 "-" "gPXE/0.9.7"
 EOF
 
+run_test ${lnav_test} -n -d /tmp/lnav.err \
+    -c ":goto 0" \
+    -c ":filter-out vmk" \
+    -c ":toggle-filtering" \
+    ${test_dir}/logfile_access_log.0
+
+check_output "toggle-filtering is not working" <<EOF
+192.168.202.254 - - [20/Jul/2009:22:59:26 +0000] "GET /vmw/cgi/tramp HTTP/1.0" 200 134 "-" "gPXE/0.9.7"
+192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkboot.gz HTTP/1.0" 404 46210 "-" "gPXE/0.9.7"
+192.168.202.254 - - [20/Jul/2009:22:59:29 +0000] "GET /vmw/vSphere/default/vmkernel.gz HTTP/1.0" 200 78929 "-" "gPXE/0.9.7"
+EOF
+
 run_test ${lnav_test} -n \
     -c ":switch-to-view help" \
     ${test_dir}/logfile_access_log.0