From 107199cb7c59d8c3d35c5285dfe40bee3be52942 Mon Sep 17 00:00:00 2001 From: Timothy Stack Date: Fri, 10 Sep 2021 21:01:25 -0700 Subject: [PATCH] [remote] clean up the cache --- NEWS | 2 ++ docs/schemas/config-v1.schema.json | 9 +++++++ src/lnav.cc | 2 ++ src/lnav_config.cc | 9 +++++++ src/tailer/tailer.looper.cc | 43 +++++++++++++++++++++++++++++- src/tailer/tailer.looper.cfg.hh | 4 +++ src/tailer/tailer.looper.hh | 3 +++ 7 files changed, 71 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index c0b610e5..6ff0691e 100644 --- a/NEWS +++ b/NEWS @@ -25,6 +25,8 @@ lnav v0.10.1: is an issue opening a file. * Overwritten files should be reloaded again. * The "jget()" SQL function now returns numbers with the correct type. + * The local copies of remote files are now cleaned up after a couple + days of the host not being accessed. lnav v0.10.0: Features: diff --git a/docs/schemas/config-v1.schema.json b/docs/schemas/config-v1.schema.json index 03b38258..43fa2a6c 100644 --- a/docs/schemas/config-v1.schema.json +++ b/docs/schemas/config-v1.schema.json @@ -68,6 +68,15 @@ "title": "/tuning/remote", "type": "object", "properties": { + "cache-ttl": { + "title": "/tuning/remote/cache-ttl", + "description": "The time-to-live for files copied from remote hosts, expressed as a duration (e.g. '3d' for three days)", + "type": "string", + "examples": [ + "3d", + "12h" + ] + }, "ssh": { "description": "Settings related to the ssh command used to contact remote machines", "title": "/tuning/remote/ssh", diff --git a/src/lnav.cc b/src/lnav.cc index cafab238..2860740e 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -1857,6 +1857,7 @@ static void looper() if (!ran_cleanup) { archive_manager::cleanup_cache(); + tailer::cleanup_cache(); ran_cleanup = true; } } @@ -2868,6 +2869,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' log_info("Executing initial commands"); execute_init_commands(lnav_data.ld_exec_context, cmd_results); archive_manager::cleanup_cache(); + tailer::cleanup_cache(); wait_for_pipers(); isc::to() .send_and_wait([](auto& clooper) { diff --git a/src/lnav_config.cc b/src/lnav_config.cc index 7771384d..0a3a11a3 100644 --- a/src/lnav_config.cc +++ b/src/lnav_config.cc @@ -957,6 +957,15 @@ static struct json_path_container ssh_handlers = { }; static struct json_path_container remote_handlers = { + yajlpp::property_handler("cache-ttl") + .with_synopsis("") + .with_description( + "The time-to-live for files copied from remote hosts, expressed as a duration " + "(e.g. '3d' for three days)") + .with_example("3d") + .with_example("12h") + .for_field(&_lnav_config::lc_tailer, + &tailer::config::c_cache_ttl), yajlpp::property_handler("ssh") .with_description( "Settings related to the ssh command used to contact remote " diff --git a/src/tailer/tailer.looper.cc b/src/tailer/tailer.looper.cc index e0ef07d6..7a69cf2c 100644 --- a/src/tailer/tailer.looper.cc +++ b/src/tailer/tailer.looper.cc @@ -410,9 +410,15 @@ tailer::looper::host_tailer::for_host(const std::string& netloc) )); } +static +ghc::filesystem::path remote_cache_path() +{ + return lnav::paths::workdir() / "remotes"; +} + ghc::filesystem::path tailer::looper::host_tailer::tmp_path() { - auto local_path = lnav::paths::workdir() / "remotes"; + auto local_path = remote_cache_path(); ghc::filesystem::create_directories(local_path); auto_mem resolved_path; @@ -527,10 +533,19 @@ void tailer::looper::host_tailer::complete_path(const std::string &path) void tailer::looper::host_tailer::loop_body() { + const static uint64_t TOUCH_FREQ = 10000; + if (!this->ht_state.is()) { return; } + this->ht_cycle_count += 1; + if (this->ht_cycle_count % TOUCH_FREQ == 0) { + auto now = ghc::filesystem::file_time_type{ + std::chrono::system_clock::now()}; + ghc::filesystem::last_write_time(this->ht_local_path, now); + } + auto& conn = this->ht_state.get(); pollfd pfds[1]; @@ -993,3 +1008,29 @@ void tailer::looper::report_error(std::string path, std::string msg) sp_tailers.erase(path); }); } + +void tailer::cleanup_cache() +{ + (void) std::async(std::launch::async, []() { + auto now = std::chrono::system_clock::now(); + auto cache_path = remote_cache_path(); + auto& cfg = injector::get(); + std::vector to_remove; + + log_debug("cache-ttl %d", cfg.c_cache_ttl.count()); + for (const auto& entry : ghc::filesystem::directory_iterator(cache_path)) { + auto mtime = ghc::filesystem::last_write_time(entry.path()); + auto exp_time = mtime + cfg.c_cache_ttl; + if (now < exp_time) { + continue; + } + + to_remove.emplace_back(entry.path()); + } + + for (auto& entry : to_remove) { + log_debug("removing cached remote: %s", entry.c_str()); + ghc::filesystem::remove_all(entry); + } + }); +} diff --git a/src/tailer/tailer.looper.cfg.hh b/src/tailer/tailer.looper.cfg.hh index fd8631fd..24c314e9 100644 --- a/src/tailer/tailer.looper.cfg.hh +++ b/src/tailer/tailer.looper.cfg.hh @@ -30,9 +30,13 @@ #ifndef lnav_tailer_looper_cfg_hh #define lnav_tailer_looper_cfg_hh +#include + namespace tailer { struct config { + int64_t c_min_free_space{32 * 1024 * 1024}; + std::chrono::seconds c_cache_ttl{std::chrono::hours(48)}; std::string c_transfer_cmd{"cat > {0:} && chmod ugo+rx ./{0:}"}; std::string c_start_cmd{"bash -c ./{0:}"}; std::string c_ssh_cmd{"ssh"}; diff --git a/src/tailer/tailer.looper.hh b/src/tailer/tailer.looper.hh index 3c12873a..2faa68e1 100644 --- a/src/tailer/tailer.looper.hh +++ b/src/tailer/tailer.looper.hh @@ -128,6 +128,7 @@ private: std::vector ht_error_queue; std::thread ht_error_reader; state_v ht_state{disconnected()}; + uint64_t ht_cycle_count{0}; }; static void report_error(std::string path, std::string msg); @@ -146,6 +147,8 @@ private: std::map> l_remotes; }; +void cleanup_cache(); + } #endif