[remote] clean up the cache

NEWS

@@ -25,6 +25,8 @@ lnav v0.10.1:
        is an issue opening a file.
      * Overwritten files should be reloaded again.
      * The "jget()" SQL function now returns numbers with the correct type.
+     * The local copies of remote files are now cleaned up after a couple
+       days of the host not being accessed.
 
 lnav v0.10.0:
      Features:

docs/schemas/config-v1.schema.json

@@ -68,6 +68,15 @@
                     "title": "/tuning/remote",
                     "type": "object",
                     "properties": {
+                        "cache-ttl": {
+                            "title": "/tuning/remote/cache-ttl",
+                            "description": "The time-to-live for files copied from remote hosts, expressed as a duration (e.g. '3d' for three days)",
+                            "type": "string",
+                            "examples": [
+                                "3d",
+                                "12h"
+                            ]
+                        },
                         "ssh": {
                             "description": "Settings related to the ssh command used to contact remote machines",
                             "title": "/tuning/remote/ssh",

src/lnav.cc

@@ -1857,6 +1857,7 @@ static void looper()
 
                     if (!ran_cleanup) {
                         archive_manager::cleanup_cache();
+                        tailer::cleanup_cache();
                         ran_cleanup = true;
                     }
                 }
@@ -2868,6 +2869,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
                 log_info("Executing initial commands");
                 execute_init_commands(lnav_data.ld_exec_context, cmd_results);
                 archive_manager::cleanup_cache();
+                tailer::cleanup_cache();
                 wait_for_pipers();
                 isc::to<curl_looper&, services::curl_streamer_t>()
                     .send_and_wait([](auto& clooper) {

src/lnav_config.cc

@@ -957,6 +957,15 @@ static struct json_path_container ssh_handlers = {
 };
 
 static struct json_path_container remote_handlers = {
+    yajlpp::property_handler("cache-ttl")
+        .with_synopsis("<duration>")
+        .with_description(
+            "The time-to-live for files copied from remote hosts, expressed as a duration "
+            "(e.g. '3d' for three days)")
+        .with_example("3d")
+        .with_example("12h")
+        .for_field(&_lnav_config::lc_tailer,
+                   &tailer::config::c_cache_ttl),
     yajlpp::property_handler("ssh")
         .with_description(
             "Settings related to the ssh command used to contact remote "

src/tailer/tailer.looper.cc

@@ -410,9 +410,15 @@ tailer::looper::host_tailer::for_host(const std::string& netloc)
     ));
 }
 
+static
+ghc::filesystem::path remote_cache_path()
+{
+    return lnav::paths::workdir() / "remotes";
+}
+
 ghc::filesystem::path tailer::looper::host_tailer::tmp_path()
 {
-    auto local_path = lnav::paths::workdir() / "remotes";
+    auto local_path = remote_cache_path();
 
     ghc::filesystem::create_directories(local_path);
     auto_mem<char> resolved_path;
@@ -527,10 +533,19 @@ void tailer::looper::host_tailer::complete_path(const std::string &path)
 
 void tailer::looper::host_tailer::loop_body()
 {
+    const static uint64_t TOUCH_FREQ = 10000;
+
     if (!this->ht_state.is<connected>()) {
         return;
     }
 
+    this->ht_cycle_count += 1;
+    if (this->ht_cycle_count % TOUCH_FREQ == 0) {
+        auto now = ghc::filesystem::file_time_type{
+            std::chrono::system_clock::now()};
+        ghc::filesystem::last_write_time(this->ht_local_path, now);
+    }
+
     auto& conn = this->ht_state.get<connected>();
 
     pollfd pfds[1];
@@ -993,3 +1008,29 @@ void tailer::looper::report_error(std::string path, std::string msg)
                 sp_tailers.erase(path);
         });
 }
+
+void tailer::cleanup_cache()
+{
+    (void) std::async(std::launch::async, []() {
+        auto now = std::chrono::system_clock::now();
+        auto cache_path = remote_cache_path();
+        auto& cfg = injector::get<const config&>();
+        std::vector<ghc::filesystem::path> to_remove;
+
+        log_debug("cache-ttl %d", cfg.c_cache_ttl.count());
+        for (const auto& entry : ghc::filesystem::directory_iterator(cache_path)) {
+            auto mtime = ghc::filesystem::last_write_time(entry.path());
+            auto exp_time = mtime + cfg.c_cache_ttl;
+            if (now < exp_time) {
+                continue;
+            }
+
+            to_remove.emplace_back(entry.path());
+        }
+
+        for (auto& entry : to_remove) {
+            log_debug("removing cached remote: %s", entry.c_str());
+            ghc::filesystem::remove_all(entry);
+        }
+    });
+}

src/tailer/tailer.looper.cfg.hh

@@ -30,9 +30,13 @@
 #ifndef lnav_tailer_looper_cfg_hh
 #define lnav_tailer_looper_cfg_hh
 
+#include <chrono>
+
 namespace tailer {
 
 struct config {
+    int64_t c_min_free_space{32 * 1024 * 1024};
+    std::chrono::seconds c_cache_ttl{std::chrono::hours(48)};
     std::string c_transfer_cmd{"cat > {0:} && chmod ugo+rx ./{0:}"};
     std::string c_start_cmd{"bash -c ./{0:}"};
     std::string c_ssh_cmd{"ssh"};

src/tailer/tailer.looper.hh

@@ -128,6 +128,7 @@ private:
         std::vector<std::string> ht_error_queue;
         std::thread ht_error_reader;
         state_v ht_state{disconnected()};
+        uint64_t ht_cycle_count{0};
     };
 
     static void report_error(std::string path, std::string msg);
@@ -146,6 +147,8 @@ private:
     std::map<std::string, std::shared_ptr<host_tailer>> l_remotes;
 };
 
+void cleanup_cache();
+
 }
 
 #endif