|
|
|
/**
|
|
|
|
* Copyright (c) 2021, Timothy Stack
|
|
|
|
*
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions are met:
|
|
|
|
*
|
|
|
|
* * Redistributions of source code must retain the above copyright notice, this
|
|
|
|
* list of conditions and the following disclaimer.
|
|
|
|
* * Redistributions in binary form must reproduce the above copyright notice,
|
|
|
|
* this list of conditions and the following disclaimer in the documentation
|
|
|
|
* and/or other materials provided with the distribution.
|
|
|
|
* * Neither the name of Timothy Stack nor the names of its contributors
|
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
* without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
|
|
|
|
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
|
|
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
|
|
* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
|
|
|
|
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
|
|
|
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
|
|
|
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
|
|
|
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
|
|
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <memory>
|
|
|
|
#include <thread>
|
|
|
|
#include <vector>
|
|
|
|
|
|
|
|
#include "file_converter_manager.hh"
|
|
|
|
|
|
|
|
#include <unistd.h>
|
|
|
|
|
|
|
|
#include "base/fs_util.hh"
|
|
|
|
#include "base/injector.hh"
|
|
|
|
#include "base/paths.hh"
|
|
|
|
#include "config.h"
|
|
|
|
#include "line_buffer.hh"
|
|
|
|
#include "piper.looper.cfg.hh"
|
|
|
|
|
|
|
|
namespace file_converter_manager {
|
|
|
|
|
|
|
|
static const ghc::filesystem::path&
|
|
|
|
cache_dir()
|
|
|
|
{
|
|
|
|
static auto INSTANCE = lnav::paths::workdir() / "conversion";
|
|
|
|
|
|
|
|
return INSTANCE;
|
|
|
|
}
|
|
|
|
|
|
|
|
Result<convert_result, std::string>
|
|
|
|
convert(const external_file_format& eff, const std::string& filename)
|
|
|
|
{
|
|
|
|
log_info("attempting to convert file -- %s", filename.c_str());
|
|
|
|
|
|
|
|
ghc::filesystem::create_directories(cache_dir());
|
|
|
|
auto outfile = TRY(lnav::filesystem::open_temp_file(
|
|
|
|
cache_dir()
|
|
|
|
/ fmt::format(FMT_STRING("{}.XXXXXX"), eff.eff_format_name)));
|
|
|
|
auto err_pipe = TRY(auto_pipe::for_child_fd(STDERR_FILENO));
|
|
|
|
auto child = TRY(lnav::pid::from_fork());
|
|
|
|
|
|
|
|
err_pipe.after_fork(child.in());
|
|
|
|
if (child.in_child()) {
|
|
|
|
auto dev_null = open("/dev/null", O_RDONLY | O_CLOEXEC);
|
|
|
|
|
|
|
|
dup2(dev_null, STDIN_FILENO);
|
|
|
|
dup2(outfile.second.get(), STDOUT_FILENO);
|
|
|
|
outfile.second.reset();
|
|
|
|
|
|
|
|
auto new_path = lnav::filesystem::build_path({
|
|
|
|
eff.eff_source_path.parent_path(),
|
|
|
|
lnav::paths::dotlnav() / "formats/default",
|
|
|
|
});
|
|
|
|
setenv("PATH", new_path.c_str(), 1);
|
|
|
|
log_info("invoking converter: %s (PATH=%s)",
|
|
|
|
eff.eff_converter.c_str(),
|
|
|
|
new_path.c_str());
|
|
|
|
auto format_str = eff.eff_format_name;
|
|
|
|
|
|
|
|
const char* args[] = {
|
|
|
|
eff.eff_converter.c_str(),
|
|
|
|
format_str.c_str(),
|
|
|
|
filename.c_str(),
|
|
|
|
nullptr,
|
|
|
|
};
|
|
|
|
|
|
|
|
execvp(eff.eff_converter.c_str(), (char**) args);
|
|
|
|
if (errno == ENOENT) {
|
|
|
|
fprintf(stderr,
|
|
|
|
"cannot find converter: %s\n",
|
|
|
|
eff.eff_converter.c_str());
|
|
|
|
} else {
|
|
|
|
fprintf(stderr,
|
|
|
|
"failed to execute converter: %s -- %s\n",
|
|
|
|
eff.eff_converter.c_str(),
|
|
|
|
strerror(errno));
|
|
|
|
}
|
|
|
|
_exit(EXIT_FAILURE);
|
|
|
|
}
|
|
|
|
|
|
|
|
auto error_queue = std::make_shared<std::vector<std::string>>();
|
|
|
|
std::thread err_reader([err = std::move(err_pipe.read_end()),
|
|
|
|
converter = eff.eff_converter,
|
|
|
|
error_queue,
|
|
|
|
child_pid = child.in()]() mutable {
|
|
|
|
line_buffer lb;
|
|
|
|
file_range pipe_range;
|
|
|
|
bool done = false;
|
|
|
|
|
|
|
|
lb.set_fd(err);
|
|
|
|
while (!done) {
|
|
|
|
auto load_res = lb.load_next_line(pipe_range);
|
|
|
|
|
|
|
|
if (load_res.isErr()) {
|
|
|
|
done = true;
|
|
|
|
} else {
|
|
|
|
auto li = load_res.unwrap();
|
|
|
|
|
|
|
|
pipe_range = li.li_file_range;
|
|
|
|
if (li.li_file_range.empty()) {
|
|
|
|
done = true;
|
|
|
|
} else {
|
|
|
|
lb.read_range(li.li_file_range)
|
|
|
|
.then([converter, error_queue, child_pid](auto sbr) {
|
|
|
|
auto line_str = string_fragment(
|
|
|
|
sbr.get_data(), 0, sbr.length())
|
|
|
|
.trim("\n");
|
|
|
|
if (error_queue->size() < 5) {
|
|
|
|
error_queue->emplace_back(line_str.to_string());
|
|
|
|
}
|
|
|
|
|
|
|
|
log_debug("%s[%d]: %.*s",
|
|
|
|
converter.c_str(),
|
|
|
|
child_pid,
|
|
|
|
line_str.length(),
|
|
|
|
line_str.data());
|
|
|
|
});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
|
|
|
err_reader.detach();
|
|
|
|
|
|
|
|
log_info("started tshark %d to process file", child.in());
|
|
|
|
|
|
|
|
return Ok(convert_result{
|
|
|
|
std::move(child),
|
|
|
|
outfile.first,
|
|
|
|
error_queue,
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
cleanup()
|
|
|
|
{
|
|
|
|
(void) std::async(std::launch::async, []() {
|
|
|
|
const auto& cfg = injector::get<const lnav::piper::config&>();
|
|
|
|
auto now = std::chrono::system_clock::now();
|
|
|
|
auto cache_path = cache_dir();
|
|
|
|
std::vector<ghc::filesystem::path> to_remove;
|
|
|
|
|
|
|
|
for (const auto& entry :
|
|
|
|
ghc::filesystem::directory_iterator(cache_path))
|
|
|
|
{
|
|
|
|
auto mtime = ghc::filesystem::last_write_time(entry.path());
|
|
|
|
auto exp_time = mtime + cfg.c_ttl;
|
|
|
|
if (now < exp_time) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
to_remove.emplace_back(entry);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (auto& entry : to_remove) {
|
|
|
|
log_debug("removing conversion: %s", entry.c_str());
|
|
|
|
ghc::filesystem::remove_all(entry);
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace file_converter_manager
|