Archives
/
lnav
mirror of https://github.com/tstack/lnav
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[lss] missed a full_sort and improve error handling when too many files are opened

Browse Source
pull/1205/head
Tim Stack 10 months ago
parent 80a7332fc8
commit fbb89a73ab
17 changed files with 391 additions and 175 deletions
Show Stats Download Patch File Download Diff File

8
src/archive_manager.cc
Unescape Escape View File

@ -355,13 +355,19 @@ walk_archive_files(
return result; return result;
} }
for (const auto& entry : fs::recursive_directory_iterator(tmp_path)) { std::error_code ec;
for (const auto& entry : fs::recursive_directory_iterator(tmp_path, ec)) {
if (!entry.is_regular_file()) { if (!entry.is_regular_file()) {
continue; continue;
} }
callback(tmp_path, entry); callback(tmp_path, entry);
} }
if (ec) {
return Err(fmt::format(FMT_STRING("failed to walk temp dir: {} -- {}"),
tmp_path.string(),
ec.message()));
}
return Ok(); return Ok();
#else #else

43
src/base/future_util.hh
Unescape Escape View File

@ -57,22 +57,32 @@ make_ready_future(T&& t)
* A queue used to limit the number of futures that are running concurrently. * A queue used to limit the number of futures that are running concurrently.
* *
* @tparam T The result of the futures. * @tparam T The result of the futures.
* @tparam MAX_QUEUE_SIZE The maximum number of futures that can be in flight.
*/ */
template<typename T, int MAX_QUEUE_SIZE = 8> template<typename T>
class future_queue { class future_queue {
public: public:
enum class processor_result_t {
ok,
interrupt,
};
/** /**
* @param processor The function to execute with the result of a future. * @param processor The function to execute with the result of a future.
* @param max_queue_size The maximum number of futures that can be in
* flight.
*/ */
explicit future_queue(std::function<void(T&)> processor) explicit future_queue(
: fq_processor(processor){}; std::function<processor_result_t(std::future<T>&)> processor,
size_t max_queue_size = 8)
~future_queue() : fq_processor(processor), fq_max_queue_size(max_queue_size)
{ {
this->pop_to();
} }
future_queue(const future_queue&) = delete;
future_queue& operator=(const future_queue&) = delete;
~future_queue() { this->pop_to(); }
/** /**
* Add a future to the queue. If the size of the queue is greater than the * Add a future to the queue. If the size of the queue is greater than the
* MAX_QUEUE_SIZE, this call will block waiting for the first queued * MAX_QUEUE_SIZE, this call will block waiting for the first queued
@ -80,10 +90,10 @@ public:
* *
* @param f The future to add to the queue. * @param f The future to add to the queue.
*/ */
void push_back(std::future<T>&& f) processor_result_t push_back(std::future<T>&& f)
{ {
this->fq_deque.emplace_back(std::move(f)); this->fq_deque.emplace_back(std::move(f));
this->pop_to(MAX_QUEUE_SIZE); return this->pop_to(this->fq_max_queue_size);
} }
/** /**
@ -92,17 +102,24 @@ public:
* *
* @param size The new desired size of the queue. * @param size The new desired size of the queue.
*/ */
void pop_to(size_t size = 0) processor_result_t pop_to(size_t size = 0)
{ {
processor_result_t retval = processor_result_t::ok;
while (this->fq_deque.size() > size) { while (this->fq_deque.size() > size) {
auto v = this->fq_deque.front().get(); if (this->fq_processor(this->fq_deque.front())
this->fq_processor(v); == processor_result_t::interrupt)
{
retval = processor_result_t::interrupt;
}
this->fq_deque.pop_front(); this->fq_deque.pop_front();
} }
return retval;
} }
std::function<void(T&)> fq_processor; std::function<processor_result_t(std::future<T>&)> fq_processor;
std::deque<std::future<T>> fq_deque; std::deque<std::future<T>> fq_deque;
size_t fq_max_queue_size;
}; };
} // namespace futures } // namespace futures

211
src/file_collection.cc
Unescape Escape View File

@ -79,6 +79,38 @@ child_poller::poll(file_collection& fc)
}); });
} }
file_collection::limits_t::limits_t()
{
static constexpr rlim_t RESERVED_FDS = 32;
struct rlimit rl;
if (getrlimit(RLIMIT_NOFILE, &rl) == 0) {
this->l_fds = rl.rlim_cur;
} else {
log_error("getrlimit() failed -- %s", strerror(errno));
this->l_fds = 8192;
}
if (this->l_fds < RESERVED_FDS) {
this->l_open_files = this->l_fds;
} else {
this->l_open_files = this->l_fds - RESERVED_FDS;
}
log_info(
"fd limit: %zu; open file limit: %zu", this->l_fds, this->l_open_files);
}
const file_collection::limits_t&
file_collection::get_limits()
{
static const limits_t INSTANCE;
return INSTANCE;
}
void void
file_collection::close_files(const std::vector<std::shared_ptr<logfile>>& files) file_collection::close_files(const std::vector<std::shared_ptr<logfile>>& files)
{ {
@ -123,11 +155,15 @@ file_collection::regenerate_unique_file_names()
upg.generate(); upg.generate();
this->fc_largest_path_length = 0; this->fc_largest_path_length = 0;
for (const auto& pair : this->fc_name_to_errors) { {
auto path = ghc::filesystem::path(pair.first).filename().string(); safe::ReadAccess<safe_name_to_errors> errs(*this->fc_name_to_errors);
if (path.length() > this->fc_largest_path_length) { for (const auto& pair : *errs) {
this->fc_largest_path_length = path.length(); auto path = ghc::filesystem::path(pair.first).filename().string();
if (path.length() > this->fc_largest_path_length) {
this->fc_largest_path_length = path.length();
}
} }
} }
for (const auto& lf : this->fc_files) { for (const auto& lf : this->fc_files) {
@ -166,13 +202,23 @@ file_collection::merge(file_collection& other)
this->fc_synced_files.insert(other.fc_synced_files.begin(), this->fc_synced_files.insert(other.fc_synced_files.begin(),
other.fc_synced_files.end()); other.fc_synced_files.end());
this->fc_name_to_errors.insert(other.fc_name_to_errors.begin(),
other.fc_name_to_errors.end()); std::map<std::string, file_error_info> new_errors;
{
safe::WriteAccess<safe_name_to_errors> errs(*other.fc_name_to_errors);
new_errors = std::move(*errs);
}
{
safe::WriteAccess<safe_name_to_errors> errs(*this->fc_name_to_errors);
errs->insert(new_errors.begin(), new_errors.end());
}
this->fc_file_names.insert(other.fc_file_names.begin(), this->fc_file_names.insert(other.fc_file_names.begin(),
other.fc_file_names.end()); other.fc_file_names.end());
if (!other.fc_files.empty()) { if (!other.fc_files.empty()) {
for (const auto& lf : other.fc_files) { for (const auto& lf : other.fc_files) {
this->fc_name_to_errors.erase(lf->get_filename()); this->fc_name_to_errors->writeAccess()->erase(lf->get_filename());
} }
this->fc_files.insert( this->fc_files.insert(
this->fc_files.end(), other.fc_files.begin(), other.fc_files.end()); this->fc_files.end(), other.fc_files.begin(), other.fc_files.end());
@ -267,7 +313,11 @@ file_collection::watch_logfile(const std::string& filename,
if (this->fc_file_names.find(wilddir) == this->fc_file_names.end()) if (this->fc_file_names.find(wilddir) == this->fc_file_names.end())
{ {
retval.fc_file_names.emplace(wilddir, logfile_open_options()); retval.fc_file_names.emplace(
wilddir,
logfile_open_options()
.with_non_utf_visibility(false)
.with_visible_size_limit(256 * 1024));
} }
return lnav::futures::make_ready_future(std::move(retval)); return lnav::futures::make_ready_future(std::move(retval));
} }
@ -279,10 +329,15 @@ file_collection::watch_logfile(const std::string& filename,
return lnav::futures::make_ready_future(std::move(retval)); return lnav::futures::make_ready_future(std::move(retval));
} }
} }
auto err_iter = this->fc_name_to_errors.find(filename_key); {
if (err_iter != this->fc_name_to_errors.end()) { safe::WriteAccess<safe_name_to_errors> errs(
if (err_iter->second.fei_mtime != st.st_mtime) { *this->fc_name_to_errors);
this->fc_name_to_errors.erase(err_iter);
auto err_iter = errs->find(filename_key);
if (err_iter != errs->end()) {
if (err_iter->second.fei_mtime != st.st_mtime) {
errs->erase(err_iter);
}
} }
} }
} }
@ -291,11 +346,12 @@ file_collection::watch_logfile(const std::string& filename,
log_error("failed to open required file: %s -- %s", log_error("failed to open required file: %s -- %s",
filename.c_str(), filename.c_str(),
strerror(errno)); strerror(errno));
retval.fc_name_to_errors.emplace(filename, retval.fc_name_to_errors->writeAccess()->emplace(
file_error_info{ filename,
time(nullptr), file_error_info{
std::string(strerror(errno)), time(nullptr),
}); std::string(strerror(errno)),
});
} }
return lnav::futures::make_ready_future(std::move(retval)); return lnav::futures::make_ready_future(std::move(retval));
} }
@ -328,9 +384,13 @@ file_collection::watch_logfile(const std::string& filename,
errs = this->fc_name_to_errors]() mutable { errs = this->fc_name_to_errors]() mutable {
file_collection retval; file_collection retval;
if (errs.find(filename) != errs.end()) { {
// The file is broken, no reason to try and reopen safe::ReadAccess<safe_name_to_errors> errs_inner(*errs);
return retval;
if (errs_inner->find(filename) != errs_inner->end()) {
// The file is broken, no reason to try and reopen
return retval;
}
} }
auto ff = loo.loo_temp_file ? file_format_t::UNKNOWN auto ff = loo.loo_temp_file ? file_format_t::UNKNOWN
@ -394,11 +454,12 @@ file_collection::watch_logfile(const std::string& filename,
log_error("archive extraction failed: %s", log_error("archive extraction failed: %s",
res.unwrapErr().c_str()); res.unwrapErr().c_str());
retval.clear(); retval.clear();
retval.fc_name_to_errors.emplace(filename, retval.fc_name_to_errors->writeAccess()->emplace(
file_error_info{ filename,
st.st_mtime, file_error_info{
res.unwrapErr(), st.st_mtime,
}); res.unwrapErr(),
});
} else { } else {
retval.fc_other_files[filename] = ff; retval.fc_other_files[filename] = ff;
} }
@ -422,12 +483,12 @@ file_collection::watch_logfile(const std::string& filename,
eff.value(), filename); eff.value(), filename);
if (cr.isErr()) { if (cr.isErr()) {
retval.fc_name_to_errors.emplace( retval.fc_name_to_errors->writeAccess()
filename, ->emplace(filename,
file_error_info{ file_error_info{
st.st_mtime, st.st_mtime,
cr.unwrapErr(), cr.unwrapErr(),
}); });
break; break;
} }
@ -450,14 +511,16 @@ file_collection::watch_logfile(const std::string& filename,
log_error("converter[%d] exited with %d", log_error("converter[%d] exited with %d",
child.in(), child.in(),
child.status()); child.status());
fc.fc_name_to_errors.emplace( fc.fc_name_to_errors->writeAccess()
filename, ->emplace(
file_error_info{ filename,
st.st_mtime, file_error_info{
fmt::format( st.st_mtime,
FMT_STRING("{}"), fmt::format(
fmt::join(*error_queue, "\n")), FMT_STRING("{}"),
}); fmt::join(*error_queue,
"\n")),
});
}, },
}); });
loo.with_filename(filename); loo.with_filename(filename);
@ -473,7 +536,7 @@ file_collection::watch_logfile(const std::string& filename,
if (open_res.isOk()) { if (open_res.isOk()) {
retval.fc_files.push_back(open_res.unwrap()); retval.fc_files.push_back(open_res.unwrap());
} else { } else {
retval.fc_name_to_errors.emplace( retval.fc_name_to_errors->writeAccess()->emplace(
filename, filename,
file_error_info{ file_error_info{
st.st_mtime, st.st_mtime,
@ -525,7 +588,7 @@ file_collection::expand_filename(
} }
} }
if (is_url(path.c_str())) { if (is_url(path)) {
return; return;
} }
@ -602,20 +665,22 @@ file_collection::expand_filename(
log_error("failed to find path: %s -- %s", log_error("failed to find path: %s -- %s",
path.c_str(), path.c_str(),
errmsg); errmsg);
retval.fc_name_to_errors.emplace(filename_key, retval.fc_name_to_errors->writeAccess()->emplace(
file_error_info{ filename_key,
time(nullptr), file_error_info{
errmsg, time(nullptr),
}); errmsg,
});
} else { } else {
log_error("failed to find path: %s -- %s", log_error("failed to find path: %s -- %s",
path_str.c_str(), path_str.c_str(),
errmsg); errmsg);
retval.fc_name_to_errors.emplace(path_str, retval.fc_name_to_errors->writeAccess()->emplace(
file_error_info{ path_str,
time(nullptr), file_error_info{
errmsg, time(nullptr),
}); errmsg,
});
} }
fq.push_back(lnav::futures::make_ready_future( fq.push_back(lnav::futures::make_ready_future(
std::move(retval))); std::move(retval)));
@ -629,7 +694,12 @@ file_collection::expand_filename(
} }
if (required || access(iter->second.c_str(), R_OK) == 0) { if (required || access(iter->second.c_str(), R_OK) == 0) {
fq.push_back(watch_logfile(iter->second, loo, required)); if (fq.push_back(watch_logfile(iter->second, loo, required))
== lnav::futures::future_queue<
file_collection>::processor_result_t::interrupt)
{
break;
}
} }
} }
} }
@ -640,9 +710,34 @@ file_collection::rescan_files(bool required)
{ {
file_collection retval; file_collection retval;
lnav::futures::future_queue<file_collection> fq( lnav::futures::future_queue<file_collection> fq(
[&retval](auto& fc) { retval.merge(fc); }); [this, &retval](std::future<file_collection>& fc) {
try {
auto v = fc.get();
retval.merge(v);
} catch (const std::exception& e) {
log_error("rescan future exception: %s", e.what());
} catch (...) {
log_error("unknown exception thrown by rescan future");
}
if (this->fc_files.size() + retval.fc_files.size()
< get_limits().l_open_files)
{
return lnav::futures::future_queue<
file_collection>::processor_result_t::ok;
}
return lnav::futures::future_queue<
file_collection>::processor_result_t::interrupt;
});
for (auto& pair : this->fc_file_names) { for (auto& pair : this->fc_file_names) {
if (this->fc_files.size() + retval.fc_files.size()
>= get_limits().l_open_files)
{
log_debug("too many files open, breaking...");
break;
}
if (pair.second.loo_piper) { if (pair.second.loo_piper) {
this->expand_filename( this->expand_filename(
fq, fq,
@ -690,3 +785,13 @@ file_collection::active_pipers() const
return retval; return retval;
} }
file_collection
file_collection::copy()
{
file_collection retval;
retval.merge(*this);
retval.fc_progress = this->fc_progress;
return retval;
}

34
src/file_collection.hh
Unescape Escape View File

@ -73,6 +73,8 @@ struct file_error_info {
const std::string fei_description; const std::string fei_description;
}; };
using safe_name_to_errors = safe::Safe<std::map<std::string, file_error_info>>;
struct file_collection; struct file_collection;
enum class child_poll_result_t { enum class child_poll_result_t {
@ -139,7 +141,8 @@ struct file_collection {
bool fc_recursive{false}; bool fc_recursive{false};
bool fc_rotated{false}; bool fc_rotated{false};
std::map<std::string, file_error_info> fc_name_to_errors; std::shared_ptr<safe_name_to_errors> fc_name_to_errors{
std::make_shared<safe_name_to_errors>()};
std::map<std::string, logfile_open_options> fc_file_names; std::map<std::string, logfile_open_options> fc_file_names;
std::vector<std::shared_ptr<logfile>> fc_files; std::vector<std::shared_ptr<logfile>> fc_files;
int fc_files_generation{0}; int fc_files_generation{0};
@ -148,19 +151,31 @@ struct file_collection {
std::set<std::string> fc_closed_files; std::set<std::string> fc_closed_files;
std::map<std::string, other_file_descriptor> fc_other_files; std::map<std::string, other_file_descriptor> fc_other_files;
std::set<std::string> fc_synced_files; std::set<std::string> fc_synced_files;
std::shared_ptr<safe_scan_progress> fc_progress; std::shared_ptr<safe_scan_progress> fc_progress{
std::make_shared<safe_scan_progress>()};
std::vector<struct stat> fc_new_stats; std::vector<struct stat> fc_new_stats;
std::list<child_poller> fc_child_pollers; std::list<child_poller> fc_child_pollers;
size_t fc_largest_path_length{0}; size_t fc_largest_path_length{0};
file_collection() struct limits_t {
: fc_progress(std::make_shared<safe::Safe<scan_progress>>()) limits_t();
{
} rlim_t l_fds;
rlim_t l_open_files;
};
static const limits_t& get_limits();
file_collection() = default;
file_collection(const file_collection&) = delete;
file_collection& operator=(const file_collection&) = delete;
file_collection(file_collection&&) = default;
file_collection copy();
void clear() void clear()
{ {
this->fc_name_to_errors.clear(); this->fc_name_to_errors->writeAccess()->clear();
this->fc_file_names.clear(); this->fc_file_names.clear();
this->fc_files.clear(); this->fc_files.clear();
this->fc_closed_files.clear(); this->fc_closed_files.clear();
@ -168,6 +183,11 @@ struct file_collection {
this->fc_new_stats.clear(); this->fc_new_stats.clear();
} }
bool is_below_open_file_limit() const
{
return this->fc_files.size() < get_limits().l_open_files;
}
file_collection rescan_files(bool required = false); file_collection rescan_files(bool required = false);
void expand_filename(lnav::futures::future_queue<file_collection>& fq, void expand_filename(lnav::futures::future_queue<file_collection>& fq,

80
src/files_sub_source.cc
Unescape Escape View File

@ -46,14 +46,18 @@ from_selection(vis_line_t sel_vis)
auto& fc = lnav_data.ld_active_files; auto& fc = lnav_data.ld_active_files;
int sel = (int) sel_vis; int sel = (int) sel_vis;
if (sel < fc.fc_name_to_errors.size()) { {
auto iter = fc.fc_name_to_errors.begin(); safe::ReadAccess<safe_name_to_errors> errs(*fc.fc_name_to_errors);
std::advance(iter, sel); if (sel < errs->size()) {
return error_selection::build(sel, iter); auto iter = errs->begin();
}
sel -= fc.fc_name_to_errors.size(); std::advance(iter, sel);
return error_selection::build(sel, iter->first);
}
sel -= errs->size();
}
if (sel < fc.fc_other_files.size()) { if (sel < fc.fc_other_files.size()) {
auto iter = fc.fc_other_files.begin(); auto iter = fc.fc_other_files.begin();
@ -171,11 +175,11 @@ files_sub_source::list_input_handle_key(listview_curses& lv, int ch)
[&](files_model::error_selection& es) { [&](files_model::error_selection& es) {
auto& fc = lnav_data.ld_active_files; auto& fc = lnav_data.ld_active_files;
fc.fc_file_names.erase(es.sb_iter->first); fc.fc_file_names.erase(es.sb_iter);
auto name_iter = fc.fc_file_names.begin(); auto name_iter = fc.fc_file_names.begin();
while (name_iter != fc.fc_file_names.end()) { while (name_iter != fc.fc_file_names.end()) {
if (name_iter->first == es.sb_iter->first) { if (name_iter->first == es.sb_iter) {
name_iter = fc.fc_file_names.erase(name_iter); name_iter = fc.fc_file_names.erase(name_iter);
continue; continue;
} }
@ -186,8 +190,7 @@ files_sub_source::list_input_handle_key(listview_curses& lv, int ch)
if (rp_opt) { if (rp_opt) {
auto rp = *rp_opt; auto rp = *rp_opt;
if (fmt::to_string(rp.home()) == es.sb_iter->first) if (fmt::to_string(rp.home()) == es.sb_iter) {
{
fc.fc_other_files.erase(name_iter->first); fc.fc_other_files.erase(name_iter->first);
name_iter = fc.fc_file_names.erase(name_iter); name_iter = fc.fc_file_names.erase(name_iter);
continue; continue;
@ -196,7 +199,7 @@ files_sub_source::list_input_handle_key(listview_curses& lv, int ch)
++name_iter; ++name_iter;
} }
fc.fc_name_to_errors.erase(es.sb_iter); fc.fc_name_to_errors->writeAccess()->erase(es.sb_iter);
fc.fc_invalidate_merge = true; fc.fc_invalidate_merge = true;
lv.reload_data(); lv.reload_data();
}, },
@ -220,7 +223,7 @@ files_sub_source::text_line_count()
{ {
const auto& fc = lnav_data.ld_active_files; const auto& fc = lnav_data.ld_active_files;
return fc.fc_name_to_errors.size() + fc.fc_other_files.size() return fc.fc_name_to_errors->readAccess()->size() + fc.fc_other_files.size()
+ fc.fc_files.size(); + fc.fc_files.size();
} }
@ -242,22 +245,26 @@ files_sub_source::text_value_for_line(textview_curses& tc,
= std::min(fc.fc_largest_path_length, = std::min(fc.fc_largest_path_length,
std::max((size_t) 40, (size_t) dim.second - 30)); std::max((size_t) 40, (size_t) dim.second - 30));
if (line < fc.fc_name_to_errors.size()) { {
auto iter = fc.fc_name_to_errors.begin(); safe::ReadAccess<safe_name_to_errors> errs(*fc.fc_name_to_errors);
std::advance(iter, line);
auto path = ghc::filesystem::path(iter->first); if (line < errs->size()) {
auto fn = path.filename().string(); auto iter = errs->begin();
std::advance(iter, line);
auto path = ghc::filesystem::path(iter->first);
auto fn = path.filename().string();
truncate_to(fn, filename_width);
value_out = fmt::format(FMT_STRING(" {:<{}} {}"),
fn,
filename_width,
iter->second.fei_description);
return;
}
truncate_to(fn, filename_width); line -= errs->size();
value_out = fmt::format(FMT_STRING(" {:<{}} {}"),
fn,
filename_width,
iter->second.fei_description);
return;
} }
line -= fc.fc_name_to_errors.size();
if (line < fc.fc_other_files.size()) { if (line < fc.fc_other_files.size()) {
auto iter = fc.fc_other_files.begin(); auto iter = fc.fc_other_files.begin();
std::advance(iter, line); std::advance(iter, line);
@ -317,17 +324,22 @@ files_sub_source::text_attrs_for_line(textview_curses& tc,
value_out.emplace_back(line_range{0, 1}, VC_GRAPHIC.value(ACS_RARROW)); value_out.emplace_back(line_range{0, 1}, VC_GRAPHIC.value(ACS_RARROW));
} }
if (line < fc.fc_name_to_errors.size()) { {
if (selected) { safe::ReadAccess<safe_name_to_errors> errs(*fc.fc_name_to_errors);
value_out.emplace_back(line_range{0, -1},
VC_ROLE.value(role_t::VCR_DISABLED_FOCUSED));
}
value_out.emplace_back(line_range{4 + (int) filename_width, -1}, if (line < errs->size()) {
VC_ROLE_FG.value(role_t::VCR_ERROR)); if (selected) {
return; value_out.emplace_back(
line_range{0, -1},
VC_ROLE.value(role_t::VCR_DISABLED_FOCUSED));
}
value_out.emplace_back(line_range{4 + (int) filename_width, -1},
VC_ROLE_FG.value(role_t::VCR_ERROR));
return;
}
line -= errs->size();
} }
line -= fc.fc_name_to_errors.size();
if (line < fc.fc_other_files.size()) { if (line < fc.fc_other_files.size()) {
if (selected) { if (selected) {

5
src/files_sub_source.hh
Unescape Escape View File

@ -89,10 +89,7 @@ struct selection_base {
} }
}; };
struct error_selection struct error_selection : public selection_base<error_selection, std::string> {};
: public selection_base<error_selection,
std::map<std::string, file_error_info>::iterator> {
};
struct other_selection struct other_selection
: public selection_base< : public selection_base<

10
src/filter_status_source.cc
Unescape Escape View File

@ -122,7 +122,8 @@ filter_status_source::statusview_fields()
} else { } else {
this->tss_fields[TSF_FILES_TITLE].set_value(" " ANSI_ROLE("F") "iles ", this->tss_fields[TSF_FILES_TITLE].set_value(" " ANSI_ROLE("F") "iles ",
role_t::VCR_STATUS_HOTKEY); role_t::VCR_STATUS_HOTKEY);
if (lnav_data.ld_active_files.fc_name_to_errors.empty()) { if (lnav_data.ld_active_files.fc_name_to_errors->readAccess()->empty())
{
this->tss_fields[TSF_FILES_TITLE].set_role( this->tss_fields[TSF_FILES_TITLE].set_role(
role_t::VCR_STATUS_DISABLED_TITLE); role_t::VCR_STATUS_DISABLED_TITLE);
} else { } else {
@ -130,12 +131,13 @@ filter_status_source::statusview_fields()
role_t::VCR_ALERT_STATUS); role_t::VCR_ALERT_STATUS);
auto& fc = lnav_data.ld_active_files; auto& fc = lnav_data.ld_active_files;
if (fc.fc_name_to_errors.size() == 1) { if (fc.fc_name_to_errors->readAccess()->size() == 1) {
this->tss_error.set_value(" error: a file cannot be opened "); this->tss_error.set_value(" error: a file cannot be opened ");
} else { } else {
this->tss_error.set_value( this->tss_error.set_value(
" error: %u files cannot be opened ", " error: %u files cannot be opened ",
lnav_data.ld_active_files.fc_name_to_errors.size()); lnav_data.ld_active_files.fc_name_to_errors->readAccess()
->size());
} }
} }
this->tss_fields[TSF_FILES_RIGHT_STITCH].set_stitch_value( this->tss_fields[TSF_FILES_RIGHT_STITCH].set_stitch_value(
@ -181,7 +183,7 @@ status_field&
filter_status_source::statusview_value_for_field(int field) filter_status_source::statusview_value_for_field(int field)
{ {
if (field == TSF_FILTERED if (field == TSF_FILTERED
&& !lnav_data.ld_active_files.fc_name_to_errors.empty()) && !lnav_data.ld_active_files.fc_name_to_errors->readAccess()->empty())
{ {
return this->tss_error; return this->tss_error;
} }

8
src/formats/vmw_log.json
Unescape Escape View File

@ -90,6 +90,14 @@
} }
] ]
}, },
"esxtokend": {
"format": [
{
"field": "body",
"extractor": "Invoke-MethodId: (.*)"
}
]
},
"hostd": { "hostd": {
"format": [ "format": [
{ {

2
src/gantt_source.hh
Unescape Escape View File

@ -175,7 +175,7 @@ public:
} }
private: private:
bool gho_show_details{true}; bool gho_show_details{false};
std::shared_ptr<gantt_source> gho_src; std::shared_ptr<gantt_source> gho_src;
}; };

7
src/line_buffer.cc
Unescape Escape View File

@ -1461,12 +1461,13 @@ line_buffer::cleanup_cache()
auto now = std::chrono::system_clock::now(); auto now = std::chrono::system_clock::now();
auto cache_path = line_buffer_cache_path(); auto cache_path = line_buffer_cache_path();
std::vector<ghc::filesystem::path> to_remove; std::vector<ghc::filesystem::path> to_remove;
std::error_code ec;
for (const auto& cache_subdir : for (const auto& cache_subdir :
ghc::filesystem::directory_iterator(cache_path)) ghc::filesystem::directory_iterator(cache_path, ec))
{ {
for (const auto& entry : for (const auto& entry :
ghc::filesystem::directory_iterator(cache_subdir)) ghc::filesystem::directory_iterator(cache_subdir, ec))
{ {
auto mtime = ghc::filesystem::last_write_time(entry.path()); auto mtime = ghc::filesystem::last_write_time(entry.path());
auto exp_time = mtime + 1h; auto exp_time = mtime + 1h;
@ -1480,7 +1481,7 @@ line_buffer::cleanup_cache()
for (auto& entry : to_remove) { for (auto& entry : to_remove) {
log_debug("removing compressed file cache: %s", entry.c_str()); log_debug("removing compressed file cache: %s", entry.c_str());
ghc::filesystem::remove_all(entry); ghc::filesystem::remove_all(entry, ec);
} }
}); });
} }

80
src/lnav.cc
Unescape Escape View File

@ -1523,15 +1523,13 @@ looper()
timer.start_fade(index_counter, 1); timer.start_fade(index_counter, 1);
file_collection active_copy; std::future<file_collection> rescan_future;
log_debug("rescan started %p", &active_copy);
active_copy.merge(lnav_data.ld_active_files); log_debug("rescan started");
active_copy.fc_progress = lnav_data.ld_active_files.fc_progress; rescan_future = std::async(std::launch::async,
std::future<file_collection> rescan_future &file_collection::rescan_files,
= std::async(std::launch::async, lnav_data.ld_active_files.copy(),
&file_collection::rescan_files, false);
std::move(active_copy),
false);
bool initial_rescan_completed = false; bool initial_rescan_completed = false;
int session_stage = 0; int session_stage = 0;
@ -1604,20 +1602,18 @@ looper()
} }
} }
active_copy.clear();
rescan_future = std::future<file_collection>{}; rescan_future = std::future<file_collection>{};
next_rescan_time = ui_clock::now() + 333ms; next_rescan_time = ui_clock::now() + 333ms;
} }
if (!rescan_future.valid() if (!rescan_future.valid()
&& (session_stage < 2 || ui_clock::now() >= next_rescan_time)) && (session_stage < 2
|| (lnav_data.ld_active_files.is_below_open_file_limit()
&& ui_clock::now() >= next_rescan_time)))
{ {
active_copy.clear();
active_copy.merge(lnav_data.ld_active_files);
active_copy.fc_progress = lnav_data.ld_active_files.fc_progress;
rescan_future = std::async(std::launch::async, rescan_future = std::async(std::launch::async,
&file_collection::rescan_files, &file_collection::rescan_files,
std::move(active_copy), lnav_data.ld_active_files.copy(),
false); false);
} }
@ -1954,7 +1950,9 @@ looper()
{ {
lnav::session::restore_view_states(); lnav::session::restore_view_states();
if (lnav_data.ld_mode == ln_mode_t::FILES) { if (lnav_data.ld_mode == ln_mode_t::FILES) {
if (lnav_data.ld_active_files.fc_name_to_errors.empty()) if (lnav_data.ld_active_files.fc_name_to_errors
->readAccess()
->empty())
{ {
log_info("switching to paging!"); log_info("switching to paging!");
lnav_data.ld_mode = ln_mode_t::PAGING; lnav_data.ld_mode = ln_mode_t::PAGING;
@ -3240,19 +3238,21 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
wait_for_pipers(); wait_for_pipers();
rescan_files(true); rescan_files(true);
rebuild_indexes_repeatedly(); rebuild_indexes_repeatedly();
if (!lnav_data.ld_active_files.fc_name_to_errors.empty()) { {
for (const auto& pair : safe::WriteAccess<safe_name_to_errors> errs(
lnav_data.ld_active_files.fc_name_to_errors) *lnav_data.ld_active_files.fc_name_to_errors);
{ if (!errs->empty()) {
lnav::console::print( for (const auto& pair : *errs) {
stderr, lnav::console::print(
lnav::console::user_message::error( stderr,
attr_line_t("unable to open file: ") lnav::console::user_message::error(
.append(lnav::roles::file(pair.first))) attr_line_t("unable to open file: ")
.with_reason(pair.second.fei_description)); .append(lnav::roles::file(pair.first)))
} .with_reason(pair.second.fei_description));
}
return EXIT_FAILURE; return EXIT_FAILURE;
}
} }
init_session(); init_session();
lnav_data.ld_exec_context.set_output("stdout", stdout, nullptr); lnav_data.ld_exec_context.set_output("stdout", stdout, nullptr);
@ -3291,17 +3291,21 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
[](auto& clooper) { clooper.process_all(); }); [](auto& clooper) { clooper.process_all(); });
rebuild_indexes_repeatedly(); rebuild_indexes_repeatedly();
wait_for_children(); wait_for_children();
if (!lnav_data.ld_active_files.fc_name_to_errors.empty()) { {
for (const auto& pair : safe::WriteAccess<safe_name_to_errors> errs(
lnav_data.ld_active_files.fc_name_to_errors) *lnav_data.ld_active_files.fc_name_to_errors);
{ if (!errs->empty()) {
fprintf(stderr, for (const auto& pair : *errs) {
"error: unable to open file: %s -- %s\n", lnav::console::print(
pair.first.c_str(), stderr,
pair.second.fei_description.c_str()); lnav::console::user_message::error(
} attr_line_t("unable to open file: ")
.append(lnav::roles::file(pair.first)))
.with_reason(pair.second.fei_description));
}
return EXIT_FAILURE; return EXIT_FAILURE;
}
} }
for (const auto& lf : lnav_data.ld_active_files.fc_files) { for (const auto& lf : lnav_data.ld_active_files.fc_files) {

32
src/lnav.indexing.cc
Unescape Escape View File

@ -37,6 +37,7 @@
#include "sql_util.hh" #include "sql_util.hh"
using namespace std::chrono_literals; using namespace std::chrono_literals;
using namespace lnav::roles::literals;
/** /**
* Observer for loading progress that updates the bottom status bar. * Observer for loading progress that updates the bottom status bar.
@ -390,6 +391,9 @@ update_active_files(file_collection& new_files)
return true; return true;
} }
bool was_below_open_file_limit
= lnav_data.ld_active_files.is_below_open_file_limit();
for (const auto& lf : new_files.fc_files) { for (const auto& lf : new_files.fc_files) {
lf->set_logfile_observer(&obs); lf->set_logfile_observer(&obs);
lnav_data.ld_text_source.push_back(lf); lnav_data.ld_text_source.push_back(lf);
@ -405,7 +409,7 @@ update_active_files(file_collection& new_files)
} }
lnav_data.ld_active_files.merge(new_files); lnav_data.ld_active_files.merge(new_files);
if (!new_files.fc_files.empty() || !new_files.fc_other_files.empty() if (!new_files.fc_files.empty() || !new_files.fc_other_files.empty()
|| !new_files.fc_name_to_errors.empty()) || !new_files.fc_name_to_errors->readAccess()->empty())
{ {
lnav_data.ld_active_files.regenerate_unique_file_names(); lnav_data.ld_active_files.regenerate_unique_file_names();
} }
@ -424,6 +428,25 @@ update_active_files(file_collection& new_files)
}; };
}); });
if (was_below_open_file_limit
&& !lnav_data.ld_active_files.is_below_open_file_limit())
{
auto um
= lnav::console::user_message::error("Unable to open more files")
.with_reason(
attr_line_t("The file-descriptor limit of ")
.append(lnav::roles::number(fmt::to_string(
file_collection::get_limits().l_fds)))
.append(" is too low to support opening more files"))
.with_help(
attr_line_t("Use ")
.append("ulimit -n"_quoted_code)
.append(
" to increase the limit before running lnav"));
lnav_data.ld_exec_context.ec_error_callback_stack.back()(um);
}
return true; return true;
} }
@ -445,7 +468,9 @@ rescan_files(bool req)
continue; continue;
} }
if (lnav_data.ld_active_files.fc_name_to_errors.count(pair.first)) { if (lnav_data.ld_active_files.fc_name_to_errors->readAccess()
->count(pair.first))
{
continue; continue;
} }
@ -455,7 +480,8 @@ rescan_files(bool req)
all_synced = false; all_synced = false;
} }
} }
if (!lnav_data.ld_active_files.fc_name_to_errors.empty()) { if (!lnav_data.ld_active_files.fc_name_to_errors->readAccess()->empty())
{
return false; return false;
} }
if (!all_synced) { if (!all_synced) {

11
src/lnav_commands.cc
Unescape Escape View File

@ -2914,13 +2914,18 @@ com_close(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
if (tss.empty()) { if (tss.empty()) {
return ec.make_error("no text files are opened"); return ec.make_error("no text files are opened");
} else { } else if (!ec.ec_dry_run) {
fn_v.emplace_back(tss.current_file()->get_filename()); auto lf = tss.current_file();
lnav_data.ld_active_files.request_close(tss.current_file()); actual_path_v.emplace_back(lf->get_actual_path());
fn_v.emplace_back(lf->get_filename());
lnav_data.ld_active_files.request_close(lf);
if (tss.size() == 1) { if (tss.size() == 1) {
lnav_data.ld_view_stack.pop_back(); lnav_data.ld_view_stack.pop_back();
} }
} else {
retval = fmt::format(FMT_STRING("closing -- {}"),
tss.current_file()->get_filename());
} }
} else if (tc == &lnav_data.ld_views[LNV_LOG]) { } else if (tc == &lnav_data.ld_views[LNV_LOG]) {
if (tc->get_inner_height() == 0) { if (tc->get_inner_height() == 0) {

12
src/log_format.cc
Unescape Escape View File

@ -2376,10 +2376,16 @@ detect_mime_type(const ghc::filesystem::path& filename)
continue; continue;
} }
if (elf->elf_converter.c_header.h_exprs.he_exprs.empty()) {
continue;
}
if (buffer_size < elf->elf_converter.c_header.h_size) { if (buffer_size < elf->elf_converter.c_header.h_size) {
log_debug("file content too small (%d) for header detection: %s", log_debug(
buffer_size, "%s: file content too small (%d) for header detection: %s",
elf->get_name().get()); filename.c_str(),
buffer_size,
elf->get_name().get());
continue; continue;
} }
for (const auto& hpair : elf->elf_converter.c_header.h_exprs.he_exprs) { for (const auto& hpair : elf->elf_converter.c_header.h_exprs.he_exprs) {

6
src/logfile.cc
Unescape Escape View File

@ -79,7 +79,6 @@ logfile::open(std::string filename, const logfile_open_options& loo, auto_fd fd)
char resolved_path[PATH_MAX] = ""; char resolved_path[PATH_MAX] = "";
if (!fd.has_value()) { if (!fd.has_value()) {
errno = 0;
if (realpath(lf->lf_filename.c_str(), resolved_path) == nullptr) { if (realpath(lf->lf_filename.c_str(), resolved_path) == nullptr) {
return Err(fmt::format(FMT_STRING("realpath({}) failed with: {}"), return Err(fmt::format(FMT_STRING("realpath({}) failed with: {}"),
lf->lf_filename, lf->lf_filename,
@ -168,7 +167,10 @@ logfile::logfile(std::string filename, const logfile_open_options& loo)
this->lf_opids.writeAccess()->los_opid_ranges.reserve(64); this->lf_opids.writeAccess()->los_opid_ranges.reserve(64);
} }
logfile::~logfile() {} logfile::~logfile()
{
log_info("destructing logfile: %s", this->lf_filename.c_str());
}
bool bool
logfile::exists() const logfile::exists() const

5
src/logfile_sub_source.cc
Unescape Escape View File

@ -820,6 +820,11 @@ logfile_sub_source::rebuild_index(
{ {
lowest_tv = new_file_line.get_timeval(); lowest_tv = new_file_line.get_timeval();
} }
} else {
log_debug(
"already doing full rebuild, doing "
"full_sort as well");
full_sort = true;
} }
} }
} }

12
src/tailer/tailer.looper.cc
Unescape Escape View File

@ -132,7 +132,7 @@ update_tailer_description(
iter->second.ofd_description = remote_uname; iter->second.ofd_description = remote_uname;
} }
fc.fc_name_to_errors.erase(netloc); fc.fc_name_to_errors->writeAccess()->erase(netloc);
}); });
} }
@ -1149,11 +1149,11 @@ tailer::looper::report_error(std::string path, std::string msg)
isc::to<main_looper&, services::main_t>().send([=](auto& mlooper) { isc::to<main_looper&, services::main_t>().send([=](auto& mlooper) {
file_collection fc; file_collection fc;
fc.fc_name_to_errors.emplace(path, fc.fc_name_to_errors->writeAccess()->emplace(path,
file_error_info{ file_error_info{
{}, {},
msg, msg,
}); });
update_active_files(fc); update_active_files(fc);
lnav_data.ld_active_files.fc_progress->writeAccess()->sp_tailers.erase( lnav_data.ld_active_files.fc_progress->writeAccess()->sp_tailers.erase(
path); path);

Write Preview
Loading…
Cancel
Save