|
|
|
/**
|
|
|
|
* Copyright (c) 2022, Timothy Stack
|
|
|
|
*
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions are met:
|
|
|
|
*
|
|
|
|
* * Redistributions of source code must retain the above copyright notice, this
|
|
|
|
* list of conditions and the following disclaimer.
|
|
|
|
* * Redistributions in binary form must reproduce the above copyright notice,
|
|
|
|
* this list of conditions and the following disclaimer in the documentation
|
|
|
|
* and/or other materials provided with the distribution.
|
|
|
|
* * Neither the name of Timothy Stack nor the names of its contributors
|
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
* without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
|
|
|
|
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
|
|
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
|
|
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
|
|
|
|
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
|
|
|
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
|
|
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
|
|
|
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
|
|
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "lnav.indexing.hh"
|
|
|
|
|
|
|
|
#include "lnav.events.hh"
|
|
|
|
#include "lnav.hh"
|
|
|
|
#include "service_tags.hh"
|
|
|
|
#include "session_data.hh"
|
|
|
|
|
|
|
|
using namespace std::chrono_literals;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Observer for loading progress that updates the bottom status bar.
|
|
|
|
*/
|
|
|
|
class loading_observer : public logfile_observer {
|
|
|
|
public:
|
|
|
|
loading_observer() : lo_last_offset(0){};
|
|
|
|
|
|
|
|
indexing_result logfile_indexing(const std::shared_ptr<logfile>& lf,
|
|
|
|
file_off_t off,
|
|
|
|
file_size_t total) override
|
|
|
|
{
|
|
|
|
static sig_atomic_t index_counter = 0;
|
|
|
|
|
|
|
|
if (lnav_data.ld_window == nullptr) {
|
|
|
|
return indexing_result::CONTINUE;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* XXX require(off <= total); */
|
|
|
|
if (off > (off_t) total) {
|
|
|
|
off = total;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((((size_t) off == total) && (this->lo_last_offset != off))
|
|
|
|
|| ui_periodic_timer::singleton().time_to_update(index_counter))
|
|
|
|
{
|
|
|
|
lnav_data.ld_bottom_source.update_loading(off, total);
|
|
|
|
do_observer_update(lf);
|
|
|
|
this->lo_last_offset = off;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!lnav_data.ld_looping) {
|
|
|
|
return indexing_result::BREAK;
|
|
|
|
}
|
|
|
|
return indexing_result::CONTINUE;
|
|
|
|
}
|
|
|
|
|
|
|
|
off_t lo_last_offset;
|
|
|
|
};
|
|
|
|
|
|
|
|
void
|
|
|
|
do_observer_update(const std::shared_ptr<logfile>& lf)
|
|
|
|
{
|
|
|
|
if (isendwin()) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
for (auto& sc : lnav_data.ld_status) {
|
|
|
|
sc.do_update();
|
|
|
|
}
|
|
|
|
if (lf && lnav_data.ld_mode == ln_mode_t::FILES
|
|
|
|
&& !lnav_data.ld_initial_build) {
|
|
|
|
auto& fc = lnav_data.ld_active_files;
|
|
|
|
auto iter = std::find(fc.fc_files.begin(), fc.fc_files.end(), lf);
|
|
|
|
|
|
|
|
if (iter != fc.fc_files.end()) {
|
|
|
|
auto index = std::distance(fc.fc_files.begin(), iter);
|
|
|
|
lnav_data.ld_files_view.set_selection(
|
|
|
|
vis_line_t(fc.fc_other_files.size() + index));
|
|
|
|
lnav_data.ld_files_view.reload_data();
|
|
|
|
lnav_data.ld_files_view.do_update();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
refresh();
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
rebuild_hist()
|
|
|
|
{
|
|
|
|
logfile_sub_source& lss = lnav_data.ld_log_source;
|
|
|
|
hist_source2& hs = lnav_data.ld_hist_source2;
|
|
|
|
int zoom = lnav_data.ld_zoom_level;
|
|
|
|
|
|
|
|
hs.set_time_slice(ZOOM_LEVELS[zoom]);
|
|
|
|
lss.reload_index_delegate();
|
|
|
|
}
|
|
|
|
|
|
|
|
class textfile_callback : public textfile_sub_source::scan_callback {
|
|
|
|
public:
|
|
|
|
void closed_files(
|
|
|
|
const std::vector<std::shared_ptr<logfile>>& files) override
|
|
|
|
{
|
|
|
|
for (const auto& lf : files) {
|
|
|
|
log_info("closed text files: %s", lf->get_filename().c_str());
|
|
|
|
}
|
|
|
|
lnav_data.ld_active_files.close_files(files);
|
|
|
|
}
|
|
|
|
|
|
|
|
void promote_file(const std::shared_ptr<logfile>& lf) override
|
|
|
|
{
|
|
|
|
if (lnav_data.ld_log_source.insert_file(lf)) {
|
|
|
|
this->did_promotion = true;
|
|
|
|
log_info("promoting text file to log file: %s (%s)",
|
|
|
|
lf->get_filename().c_str(),
|
|
|
|
lf->get_content_id().c_str());
|
|
|
|
auto format = lf->get_format();
|
|
|
|
if (format->lf_is_self_describing) {
|
|
|
|
auto vt = format->get_vtab_impl();
|
|
|
|
|
|
|
|
if (vt != nullptr) {
|
|
|
|
lnav_data.ld_vtab_manager->register_vtab(vt);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
auto iter = session_data.sd_file_states.find(lf->get_filename());
|
|
|
|
if (iter != session_data.sd_file_states.end()) {
|
|
|
|
log_debug("found state for log file %d",
|
|
|
|
iter->second.fs_is_visible);
|
|
|
|
|
|
|
|
lnav_data.ld_log_source.find_data(lf) | [&iter](auto ld) {
|
|
|
|
ld->set_visibility(iter->second.fs_is_visible);
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
lnav::events::publish(lnav_data.ld_db.in(),
|
|
|
|
lnav::events::file::format_detected{
|
|
|
|
lf->get_filename(),
|
|
|
|
lf->get_format_name().to_string(),
|
|
|
|
});
|
|
|
|
} else {
|
|
|
|
this->closed_files({lf});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void scanned_file(const std::shared_ptr<logfile>& lf) override
|
|
|
|
{
|
|
|
|
if (!lnav_data.ld_files_to_front.empty()
|
|
|
|
&& lnav_data.ld_files_to_front.front().first == lf->get_filename())
|
|
|
|
{
|
|
|
|
this->front_file = lf;
|
|
|
|
this->front_top = lnav_data.ld_files_to_front.front().second;
|
|
|
|
|
|
|
|
lnav_data.ld_files_to_front.pop_front();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
std::shared_ptr<logfile> front_file;
|
|
|
|
int front_top{-1};
|
|
|
|
bool did_promotion{false};
|
|
|
|
};
|
|
|
|
|
|
|
|
size_t
|
|
|
|
rebuild_indexes(nonstd::optional<ui_clock::time_point> deadline)
|
|
|
|
{
|
|
|
|
logfile_sub_source& lss = lnav_data.ld_log_source;
|
|
|
|
textview_curses& log_view = lnav_data.ld_views[LNV_LOG];
|
|
|
|
textview_curses& text_view = lnav_data.ld_views[LNV_TEXT];
|
|
|
|
vis_line_t old_bottoms[LNV__MAX];
|
|
|
|
bool scroll_downs[LNV__MAX];
|
|
|
|
size_t retval = 0;
|
|
|
|
|
|
|
|
for (int lpc = 0; lpc < LNV__MAX; lpc++) {
|
|
|
|
old_bottoms[lpc] = lnav_data.ld_views[lpc].get_top_for_last_row();
|
|
|
|
scroll_downs[lpc]
|
|
|
|
= (lnav_data.ld_views[lpc].get_top() >= old_bottoms[lpc])
|
|
|
|
&& !(lnav_data.ld_flags & LNF_HEADLESS);
|
|
|
|
}
|
|
|
|
|
|
|
|
{
|
|
|
|
auto* tss = &lnav_data.ld_text_source;
|
|
|
|
textfile_callback cb;
|
|
|
|
|
|
|
|
if (tss->rescan_files(cb, deadline)) {
|
|
|
|
text_view.reload_data();
|
|
|
|
retval += 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (cb.front_file != nullptr) {
|
|
|
|
ensure_view(&text_view);
|
|
|
|
|
|
|
|
if (tss->current_file() != cb.front_file) {
|
|
|
|
tss->to_front(cb.front_file);
|
|
|
|
old_bottoms[LNV_TEXT] = -1_vl;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (cb.front_top < 0) {
|
|
|
|
cb.front_top += text_view.get_inner_height();
|
|
|
|
}
|
|
|
|
if (cb.front_top < text_view.get_inner_height()) {
|
|
|
|
text_view.set_top(vis_line_t(cb.front_top));
|
|
|
|
scroll_downs[LNV_TEXT] = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (cb.did_promotion && deadline) {
|
|
|
|
// If there's a new log file, extend the deadline so it can be
|
|
|
|
// indexed quickly.
|
|
|
|
deadline = deadline.value() + 500ms;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<std::shared_ptr<logfile>> closed_files;
|
|
|
|
for (auto& lf : lnav_data.ld_active_files.fc_files) {
|
|
|
|
if ((!lf->exists() || lf->is_closed())) {
|
|
|
|
log_info("closed log file: %s", lf->get_filename().c_str());
|
|
|
|
lnav_data.ld_text_source.remove(lf);
|
|
|
|
lnav_data.ld_log_source.remove_file(lf);
|
|
|
|
closed_files.emplace_back(lf);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!closed_files.empty()) {
|
|
|
|
lnav_data.ld_active_files.close_files(closed_files);
|
|
|
|
}
|
|
|
|
|
|
|
|
auto result = lss.rebuild_index(deadline);
|
|
|
|
if (result != logfile_sub_source::rebuild_result::rr_no_change) {
|
|
|
|
size_t new_count = lss.text_line_count();
|
|
|
|
bool force
|
|
|
|
= result == logfile_sub_source::rebuild_result::rr_full_rebuild;
|
|
|
|
|
|
|
|
if ((!scroll_downs[LNV_LOG]
|
|
|
|
|| log_view.get_top() > vis_line_t(new_count))
|
|
|
|
&& force)
|
|
|
|
{
|
|
|
|
scroll_downs[LNV_LOG] = false;
|
|
|
|
}
|
|
|
|
|
|
|
|
log_view.reload_data();
|
|
|
|
|
|
|
|
{
|
|
|
|
std::unordered_map<std::string, std::list<std::shared_ptr<logfile>>>
|
|
|
|
id_to_files;
|
|
|
|
bool reload = false;
|
|
|
|
|
|
|
|
for (const auto& lf : lnav_data.ld_active_files.fc_files) {
|
|
|
|
id_to_files[lf->get_content_id()].push_back(lf);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (auto& pair : id_to_files) {
|
|
|
|
if (pair.second.size() == 1) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
pair.second.sort([](const auto& left, const auto& right) {
|
|
|
|
return right->get_stat().st_size < left->get_stat().st_size;
|
|
|
|
});
|
|
|
|
|
|
|
|
auto dupe_name = pair.second.front()->get_unique_path();
|
|
|
|
pair.second.pop_front();
|
|
|
|
for_each(pair.second.begin(),
|
|
|
|
pair.second.end(),
|
|
|
|
[&dupe_name](auto& lf) {
|
|
|
|
log_info("Hiding duplicate file: %s",
|
|
|
|
lf->get_filename().c_str());
|
|
|
|
lf->mark_as_duplicate(dupe_name);
|
|
|
|
lnav_data.ld_log_source.find_data(lf) |
|
|
|
|
[](auto ld) { ld->set_visibility(false); };
|
|
|
|
});
|
|
|
|
reload = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (reload) {
|
|
|
|
lss.text_filters_changed();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
retval += 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (int lpc = 0; lpc < LNV__MAX; lpc++) {
|
|
|
|
textview_curses& scroll_view = lnav_data.ld_views[lpc];
|
|
|
|
|
|
|
|
if (scroll_downs[lpc]
|
|
|
|
&& scroll_view.get_top_for_last_row() > scroll_view.get_top())
|
|
|
|
{
|
|
|
|
scroll_view.set_top(scroll_view.get_top_for_last_row());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
lnav_data.ld_view_stack.top() | [](auto tc) {
|
|
|
|
lnav_data.ld_filter_status_source.update_filtered(tc->get_sub_source());
|
|
|
|
lnav_data.ld_scroll_broadcaster(tc);
|
|
|
|
};
|
|
|
|
|
|
|
|
return retval;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
rebuild_indexes_repeatedly()
|
|
|
|
{
|
|
|
|
for (size_t attempt = 0; attempt < 10 && rebuild_indexes() > 0; attempt++) {
|
|
|
|
log_info("continuing to rebuild indexes...");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
update_active_files(file_collection& new_files)
|
|
|
|
{
|
|
|
|
static loading_observer obs;
|
|
|
|
|
|
|
|
if (lnav_data.ld_active_files.fc_invalidate_merge) {
|
|
|
|
lnav_data.ld_active_files.fc_invalidate_merge = false;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (const auto& lf : new_files.fc_files) {
|
|
|
|
lf->set_logfile_observer(&obs);
|
|
|
|
lnav_data.ld_text_source.push_back(lf);
|
|
|
|
}
|
|
|
|
for (const auto& other_pair : new_files.fc_other_files) {
|
|
|
|
switch (other_pair.second.ofd_format) {
|
|
|
|
case file_format_t::SQLITE_DB:
|
|
|
|
attach_sqlite_db(lnav_data.ld_db.in(), other_pair.first);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
lnav_data.ld_active_files.merge(new_files);
|
|
|
|
if (!new_files.fc_files.empty() || !new_files.fc_other_files.empty()
|
|
|
|
|| !new_files.fc_name_to_errors.empty())
|
|
|
|
{
|
|
|
|
lnav_data.ld_active_files.regenerate_unique_file_names();
|
|
|
|
}
|
|
|
|
lnav_data.ld_child_pollers.insert(
|
|
|
|
lnav_data.ld_child_pollers.begin(),
|
|
|
|
std::make_move_iterator(
|
|
|
|
lnav_data.ld_active_files.fc_child_pollers.begin()),
|
|
|
|
std::make_move_iterator(
|
|
|
|
lnav_data.ld_active_files.fc_child_pollers.end()));
|
|
|
|
|
|
|
|
lnav::events::publish(
|
|
|
|
lnav_data.ld_db.in(), new_files.fc_files, [](const auto& lf) {
|
|
|
|
return lnav::events::file::open{
|
|
|
|
lf->get_filename(),
|
|
|
|
};
|
|
|
|
});
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
rescan_files(bool req)
|
|
|
|
{
|
|
|
|
auto& mlooper = injector::get<main_looper&, services::main_t>();
|
|
|
|
bool done = false;
|
|
|
|
auto delay = 0ms;
|
|
|
|
|
|
|
|
do {
|
|
|
|
auto fc = lnav_data.ld_active_files.rescan_files(req);
|
|
|
|
bool all_synced = true;
|
|
|
|
|
|
|
|
update_active_files(fc);
|
|
|
|
mlooper.get_port().process_for(delay);
|
|
|
|
if (lnav_data.ld_flags & LNF_HEADLESS) {
|
|
|
|
for (const auto& pair : lnav_data.ld_active_files.fc_other_files) {
|
|
|
|
if (pair.second.ofd_format != file_format_t::REMOTE) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (lnav_data.ld_active_files.fc_synced_files.count(pair.first)
|
|
|
|
== 0) {
|
|
|
|
all_synced = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!all_synced) {
|
|
|
|
delay = 30ms;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
done = fc.fc_file_names.empty() && all_synced;
|
|
|
|
} while (!done);
|
|
|
|
return true;
|
|
|
|
}
|