[indexing] index more of demuxed file

src/file_collection.cc

@@ -80,7 +80,8 @@ child_poller::poll(file_collection& fc)
         });
 }
 
-file_collection::limits_t::limits_t()
+file_collection::limits_t::
+limits_t()
 {
     static constexpr rlim_t RESERVED_FDS = 32;
 
@@ -812,6 +813,22 @@ file_collection::request_close(const std::shared_ptr<logfile>& lf)
     this->fc_files_generation += 1;
 }
 
+size_t
+file_collection::initial_indexing_pipers() const
+{
+    size_t retval = 0;
+
+    for (const auto& pair : this->fc_file_names) {
+        if (pair.second.loo_piper
+            && pair.second.loo_piper->get_loop_count() == 0)
+        {
+            retval += 1;
+        }
+    }
+
+    return retval;
+}
+
 size_t
 file_collection::active_pipers() const
 {

src/file_collection.hh

@@ -223,6 +223,8 @@ struct file_collection {
 
     void regenerate_unique_file_names();
 
+    size_t initial_indexing_pipers() const;
+
     size_t active_pipers() const;
 
     size_t finished_pipers();

src/lnav.cc

@@ -1547,7 +1547,11 @@ VALUES ('org.lnav.mouse-support', -1, DATETIME('now', '+1 minute'),
             {
                 auto ui_now = ui_clock::now();
                 auto new_files = rescan_future.get();
-                if (!initial_rescan_completed && new_files.empty()) {
+                auto indexing_pipers
+                    = lnav_data.ld_active_files.initial_indexing_pipers();
+                if (!initial_rescan_completed && new_files.empty()
+                    && indexing_pipers == 0)
+                {
                     initial_rescan_completed = true;
 
                     log_debug("initial rescan rebuild");
@@ -2308,27 +2312,19 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
 
         std::vector<std::string> tables_to_drop;
         {
-            auto_mem<sqlite3_stmt> stmt(sqlite3_finalize);
-            bool done = false;
-
-            sqlite3_prepare_v2(
-                lnav_data.ld_db.in(), VIRT_TABLES, -1, stmt.out(), nullptr);
-            do {
-                auto ret = sqlite3_step(stmt.in());
+            auto prep_res = prepare_stmt(lnav_data.ld_db.in(), VIRT_TABLES);
+            if (prep_res.isErr()) {
+                log_error("unable to prepare VIRT_TABLES: %s",
+                          prep_res.unwrapErr().c_str());
+            } else {
+                auto stmt = prep_res.unwrap();
 
-                switch (ret) {
-                    case SQLITE_OK:
-                    case SQLITE_DONE:
-                        done = true;
-                        break;
-                    case SQLITE_ROW:
-                        tables_to_drop.emplace_back(fmt::format(
-                            FMT_STRING("DROP TABLE {}"),
-                            reinterpret_cast<const char*>(
-                                sqlite3_column_text(stmt.in(), 0))));
-                        break;
-                }
-            } while (!done);
+                stmt.for_each_row<std::string>(
+                    [&tables_to_drop](auto table_name) {
+                        tables_to_drop.emplace_back(table_name);
+                        return false;
+                    });
+            }
         }
 
         // XXX
@@ -2352,11 +2348,16 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
         }
 
         for (auto& drop_stmt : tables_to_drop) {
-            sqlite3_exec(lnav_data.ld_db.in(),
-                         drop_stmt.c_str(),
-                         nullptr,
-                         nullptr,
-                         nullptr);
+            auto prep_res
+                = prepare_stmt(lnav_data.ld_db.in(), drop_stmt.c_str());
+            if (prep_res.isErr()) {
+                log_error("unable to prepare DROP statement: %s",
+                          prep_res.unwrapErr().c_str());
+                continue;
+            }
+
+            auto stmt = prep_res.unwrap();
+            stmt.execute();
         }
 #if defined(HAVE_SQLITE3_DROP_MODULES)
         sqlite3_drop_modules(lnav_data.ld_db.in(), nullptr);

src/lnav.indexing.cc

@@ -202,9 +202,9 @@ public:
 rebuild_indexes_result_t
 rebuild_indexes(std::optional<ui_clock::time_point> deadline)
 {
-    logfile_sub_source& lss = lnav_data.ld_log_source;
-    textview_curses& log_view = lnav_data.ld_views[LNV_LOG];
-    textview_curses& text_view = lnav_data.ld_views[LNV_TEXT];
+    auto& lss = lnav_data.ld_log_source;
+    auto& log_view = lnav_data.ld_views[LNV_LOG];
+    auto& text_view = lnav_data.ld_views[LNV_TEXT];
     bool scroll_downs[LNV__MAX];
     rebuild_indexes_result_t retval;
 

src/piper.looper.cc

@@ -187,10 +187,8 @@ environ_to_map()
     return retval;
 }
 
-looper::looper(std::string name,
-               auto_fd stdout_fd,
-               auto_fd stderr_fd,
-               options opts)
+looper::
+looper(std::string name, auto_fd stdout_fd, auto_fd stderr_fd, options opts)
     : l_name(std::move(name)), l_cwd(ghc::filesystem::current_path().string()),
       l_env(environ_to_map()), l_stdout(std::move(stdout_fd)),
       l_stderr(std::move(stderr_fd)), l_options(opts)
@@ -209,7 +207,8 @@ looper::looper(std::string name,
     this->l_future = std::async(std::launch::async, [this]() { this->loop(); });
 }
 
-looper::~looper()
+looper::~
+looper()
 {
     log_info("piper destructed, shutting down: %s", this->l_name.c_str());
     this->l_looping = false;
@@ -224,10 +223,12 @@ enum class read_mode_t {
 void
 looper::loop()
 {
-    static const auto FORCE_MTIME_UPDATE_DURATION = 8h;
+    static constexpr auto FORCE_MTIME_UPDATE_DURATION = 8h;
     static const auto DEFAULT_ID = string_fragment{};
     static const auto OUT_OF_FRAME_ID
         = string_fragment::from_const("_out_of_frame_");
+    static constexpr auto FILE_TIMEOUT_BACKOFF = 30ms;
+    static constexpr auto FILE_TIMEOUT_MAX = 1000ms;
 
     const auto& cfg = injector::get<const config&>();
     struct pollfd pfd[2];
@@ -265,6 +266,7 @@ looper::loop()
     date_time_scanner dts;
     struct timeval line_tv;
     struct exttm line_tm;
+    auto file_timeout = 0ms;
 
     log_info("starting loop to capture: %s (%d %d)",
              this->l_name.c_str(),
@@ -279,9 +281,8 @@ looper::loop()
     captured_fds[1].cf_level = LEVEL_ERROR;
     auto last_write = std::chrono::system_clock::now();
     do {
-        static const auto TIMEOUT
+        static constexpr auto TIMEOUT
             = std::chrono::duration_cast<std::chrono::milliseconds>(1s).count();
-        static const auto FILE_TIMEOUT = (30ms).count();
 
         auto poll_timeout = TIMEOUT;
         size_t used_pfds = 0;
@@ -294,7 +295,7 @@ looper::loop()
 
             if (!cap.lb.is_pipe()) {
                 file_count += 1;
-                poll_timeout = FILE_TIMEOUT;
+                poll_timeout = file_timeout.count();
             } else if (!cap.lb.is_pipe_closed()) {
                 cap.pfd = &pfd[used_pfds];
                 used_pfds += 1;
@@ -435,8 +436,14 @@ looper::loop()
                                  this->l_name.c_str());
                         this->l_looping = false;
                     }
+                    if (file_count > 0 && file_timeout < FILE_TIMEOUT_MAX) {
+                        file_timeout += FILE_TIMEOUT_BACKOFF;
+                    }
                     break;
                 }
+                if (file_count > 0) {
+                    file_timeout = 0ms;
+                }
 
                 if (li.li_partial && !cap.lb.is_pipe_closed()) {
                     break;
@@ -679,6 +686,7 @@ looper::loop()
                 }
             }
         }
+        this->l_loop_count += 1;
     } while (this->l_looping);
 
     log_info("exiting loop to capture: %s", this->l_name.c_str());

src/piper.looper.hh

@@ -94,6 +94,11 @@ public:
                            this->l_out_dir.filename().string());
     }
 
+    int get_loop_count() const
+    {
+        return this->l_loop_count.load();
+    }
+
     bool is_finished() const
     {
         return this->l_future.wait_for(std::chrono::seconds(0))
@@ -125,6 +130,7 @@ private:
     options l_options;
     std::future<void> l_future;
     std::atomic<int> l_finished{0};
+    std::atomic<int> l_loop_count{0};
     safe_demux_id l_demux_id;
 };
 
@@ -152,6 +158,8 @@ public:
 
     std::string get_url() const { return this->h_looper->get_url(); }
 
+    int get_loop_count() const { return this->h_looper->get_loop_count(); }
+
     bool is_finished() const { return this->h_looper->is_finished(); }
 
     size_t consume_finished() { return this->h_looper->consume_finished(); }

src/session_data.cc

@@ -424,7 +424,7 @@ load_time_bookmarks()
        ORDER BY same_session DESC, session_time DESC
 )";
 
-    logfile_sub_source& lss = lnav_data.ld_log_source;
+    auto& lss = lnav_data.ld_log_source;
     auto_sqlite3 db;
     auto db_path = lnav::paths::dotlnav() / LOG_METADATA_NAME;
     auto_mem<sqlite3_stmt> stmt(sqlite3_finalize);
@@ -532,6 +532,7 @@ load_time_bookmarks()
                         = (const char*) sqlite3_column_text(stmt.in(), 0);
                     const char* log_hash
                         = (const char*) sqlite3_column_text(stmt.in(), 2);
+                    int64_t mark_time = sqlite3_column_int64(stmt.in(), 3);
                     const char* part_name
                         = (const char*) sqlite3_column_text(stmt.in(), 4);
                     const char* comment
@@ -540,7 +541,6 @@ load_time_bookmarks()
                         = (const char*) sqlite3_column_text(stmt.in(), 7);
                     const auto annotations = sqlite3_column_text(stmt.in(), 8);
                     const auto log_opid = sqlite3_column_text(stmt.in(), 9);
-                    int64_t mark_time = sqlite3_column_int64(stmt.in(), 3);
                     struct timeval log_tv;
                     struct exttm log_tm;