diff --git a/NEWS.md b/NEWS.md index 71edb63c..f275a52e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -20,6 +20,16 @@ Features: to specify how a file type can be detected and converted. The built-in PCAP support in **lnav** is implemented using this mechanism. +* Added a `shell_exec()` SQLite function that executes a + command-line with the user's `$SHELL` and returns the + output. +* Added support for custom URL schemes that are handled by an + lnav script. Schemes can be defined under + `/tuning/url-schemes`. See the main docs for more details. +* Added a `docker://` URL scheme that can be used to tail + the logs for a container (e.g. `docker://my-container`) or + files within a container (e.g. + `docker://my-serv/var/log/dpkg.log`). Bug Fixes: * When piping data into **lnav**'s stdin, the input used to diff --git a/docs/schemas/config-v1.schema.json b/docs/schemas/config-v1.schema.json index 1019772d..705ae0a8 100644 --- a/docs/schemas/config-v1.schema.json +++ b/docs/schemas/config-v1.schema.json @@ -199,6 +199,28 @@ } }, "additionalProperties": false + }, + "url-scheme": { + "description": "Settings related to custom URL handling", + "title": "/tuning/url-scheme", + "type": "object", + "patternProperties": { + "(\\w+)": { + "description": "Definition of a custom URL scheme", + "title": "/tuning/url-scheme/", + "type": "object", + "properties": { + "handler": { + "title": "/tuning/url-scheme//handler", + "description": "The name of the lnav script that can handle URLs with of this scheme. This should not include the '.lnav' suffix.", + "type": "string", + "pattern": "^[\\w\\-]+(?!\\.lnav)$" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false } }, "additionalProperties": false diff --git a/docs/source/config.rst b/docs/source/config.rst index 771c5f36..c1e7bde8 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -269,3 +269,7 @@ command. .. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/logfile .. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/remote/properties/ssh + +.. _url_scheme: + +.. jsonschema:: ../schemas/config-v1.schema.json#/properties/tuning/properties/url-scheme diff --git a/docs/source/usage.rst b/docs/source/usage.rst index d43ed527..20b305f8 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -116,6 +116,52 @@ file. The binary file is named ``tailer.bin.XXXXXX`` where *XXXXXX* is 6 random digits. The file is, under normal circumstancies, deleted immediately. +Command Output +^^^^^^^^^^^^^^ + +The output of commands can be captured and displayed in **lnav** using +the :ref:`:sh` command or by passing the :option:`-e` option on the +command-line. The captured output will be displayed in the TEXT view. +The lines from stdout and stderr are recorded separately so that the +lines from stderr can be shown in the theme's "error" highlight. The +time that the lines were received are also recorded internally so that +the "time-offset" display (enabled by pressing :kbd:`Shift` + :kbd:`T`) +can be shown and the "jump to slow-down" hotkeys (:kbd:`s` / +:kbd:`Shift` + :kbd:`S`) work. Since the line-by-line timestamps are +recorded internally, they will not interfere with timestamps that are +in the commands output. + +Docker Logs +^^^^^^^^^^^ + +To make it easier to view +`docker logs `_ +within **lnav**, a :code:`docker://` URL scheme is available. Passing +the container name in the authority field will run the :code:`docker logs` +command. If a path is added to the URL, then **lnav** will execute +:code:`docker exec tail -F -n +0 /path/to/file` to try and +tail the file in the container. + +Custom URL Schemes +^^^^^^^^^^^^^^^^^^ + +Custom URL schemes can be defined using the :ref:`/tuning/url-schemes` +configuration. By adding a scheme name to the tuning configuration along +with the name of an **lnav** handler script, you can control how the URL is +interpreted and turned into **lnav** commands. This feature is how the +`Docker Logs`_ functionality is implemented. + +Custom URLs can be passed on the command-line or to the :ref:`:open` +command. When passed on the command-line, an :code:`:open` command with the +URL is added to the list of initial commands. When the :code:`:open` command +detects a custom URL, it checks for the definition in the configuration. +If found, it will call the associated handler script with the URL as the +first parameter. The script can parse the URL using the :ref:`parse_url` +SQL function, if needed. The script should then execute whatever commands +it needs to open the destination for viewing in **lnav**. For example, +the docker URL handler uses the :ref:`:sh` command to run +:code:`docker logs` with the container. + Searching --------- diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 84b64154..8f192a6b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -190,7 +190,9 @@ add_custom_command( list(APPEND GEN_SRCS default-config.h default-config.cc) set(BUILTIN_LNAV_SCRIPTS - scripts/dhclient-summary.lnav scripts/lnav-pop-view.lnav + scripts/dhclient-summary.lnav + scripts/docker-url-handler.lnav + scripts/lnav-pop-view.lnav scripts/partition-by-boot.lnav scripts/rename-stdin.lnav scripts/search-for.lnav) @@ -550,6 +552,7 @@ add_library( time_T.hh timer.hh top_status_source.hh + url_handler.cfg.hh url_loader.hh view_helpers.hh view_helpers.crumbs.hh diff --git a/src/Makefile.am b/src/Makefile.am index 942d7c05..783f1a6f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -314,6 +314,7 @@ noinst_HEADERS = \ top_status_source.hh \ top_status_source.cfg.hh \ unique_path.hh \ + url_handler.cfg.hh \ url_loader.hh \ view_curses.hh \ view_helpers.hh \ diff --git a/src/base/auto_fd.cc b/src/base/auto_fd.cc index 47a17eb5..d5e2c489 100644 --- a/src/base/auto_fd.cc +++ b/src/base/auto_fd.cc @@ -121,6 +121,17 @@ auto_fd::close_on_exec() const log_perror(fcntl(this->af_fd, F_SETFD, FD_CLOEXEC)); } +void +auto_fd::non_blocking() const +{ + auto fl = fcntl(this->af_fd, F_GETFL, 0); + if (fl < 0) { + return; + } + + log_perror(fcntl(this->af_fd, F_SETFL, fl | O_NONBLOCK)); +} + auto_fd& auto_fd::operator=(int fd) { diff --git a/src/base/auto_fd.hh b/src/base/auto_fd.hh index b53edc75..5a2c334c 100644 --- a/src/base/auto_fd.hh +++ b/src/base/auto_fd.hh @@ -160,6 +160,8 @@ public: void close_on_exec() const; + void non_blocking() const; + private: int af_fd; /*< The managed file descriptor. */ }; diff --git a/src/base/auto_mem.hh b/src/base/auto_mem.hh index e6b456c4..25bea2e2 100644 --- a/src/base/auto_mem.hh +++ b/src/base/auto_mem.hh @@ -241,6 +241,8 @@ public: const char* begin() const { return this->ab_buffer; } + char* next_available() { return &this->ab_buffer[this->ab_size]; } + auto_buffer& push_back(char ch) { if (this->ab_size == this->ab_capacity) { diff --git a/src/base/auto_pid.hh b/src/base/auto_pid.hh index 702af1ef..ff44b992 100644 --- a/src/base/auto_pid.hh +++ b/src/base/auto_pid.hh @@ -62,10 +62,7 @@ public: { } - ~auto_pid() noexcept - { - this->reset(); - } + ~auto_pid() noexcept { this->reset(); } auto_pid& operator=(auto_pid&& other) noexcept { @@ -77,10 +74,7 @@ public: auto_pid& operator=(const auto_pid& other) = delete; - pid_t in() const - { - return this->ap_child; - } + pid_t in() const { return this->ap_child; } bool in_child() const { @@ -89,9 +83,7 @@ public: return this->ap_child == 0; } - pid_t release() && - { - return std::exchange(this->ap_child, -1); } + pid_t release() && { return std::exchange(this->ap_child, -1); } int status() const { @@ -107,6 +99,13 @@ public: return WIFEXITED(this->ap_status); } + int term_signal() const + { + static_assert(ProcState == process_state::finished, + "wait_for_child() must be called first"); + return WTERMSIG(this->ap_status); + } + int exit_status() const { static_assert(ProcState == process_state::finished, diff --git a/src/command_executor.cc b/src/command_executor.cc index 28ec5bab..ba2e377e 100644 --- a/src/command_executor.cc +++ b/src/command_executor.cc @@ -168,14 +168,7 @@ bind_sql_parameters(exec_context& ec, sqlite3_stmt* stmt) return Err(um); } - ov_iter = ec.ec_override.find(name); - if (ov_iter != ec.ec_override.end()) { - sqlite3_bind_text(stmt, - lpc, - ov_iter->second.c_str(), - ov_iter->second.length(), - SQLITE_TRANSIENT); - } else if (name[0] == '$') { + if (name[0] == '$') { const auto& lvars = ec.ec_local_vars.top(); const auto& gvars = ec.ec_global_vars; std::map::const_iterator local_var, diff --git a/src/command_executor.hh b/src/command_executor.hh index d215021d..42db4c95 100644 --- a/src/command_executor.hh +++ b/src/command_executor.hh @@ -120,6 +120,37 @@ struct exec_context { void clear_output(); + struct user {}; + struct file_open { + std::string fo_name; + }; + + using provenance_t = mapbox::util::variant; + + struct provenance_guard { + explicit provenance_guard(exec_context* context, provenance_t prov) + : pg_context(context) + { + this->pg_context->ec_provenance.push_back(prov); + } + + provenance_guard(const provenance_guard&) = delete; + provenance_guard(provenance_guard&& other) + : pg_context(other.pg_context) + { + other.pg_context = nullptr; + } + + ~provenance_guard() + { + if (this->pg_context != nullptr) { + this->pg_context->ec_provenance.pop_back(); + } + } + + exec_context* pg_context; + }; + struct source_guard { source_guard(exec_context* context) : sg_context(context) {} @@ -214,13 +245,25 @@ struct exec_context { this->ec_local_vars.pop(); } + template + nonstd::optional get_provenance() const + { + for (const auto& elem : this->ec_provenance) { + if (elem.is()) { + return elem.get(); + } + } + + return nonstd::nullopt; + } + vis_line_t ec_top_line{0_vl}; bool ec_dry_run{false}; perm_t ec_perms{perm_t::READ_WRITE}; - std::map ec_override; logline_value_vector* ec_line_values; std::stack> ec_local_vars; + std::vector ec_provenance; std::map ec_global_vars; std::vector ec_path_stack; std::vector ec_source; diff --git a/src/file_converter_manager.cc b/src/file_converter_manager.cc index ee5d7820..f06953e3 100644 --- a/src/file_converter_manager.cc +++ b/src/file_converter_manager.cc @@ -78,7 +78,6 @@ convert(const external_file_format& eff, const std::string& filename) nullptr, }; - setenv("TZ", "UTC", 1); execvp(eff.eff_converter.c_str(), (char**) args); if (errno == ENOENT) { fprintf(stderr, diff --git a/src/fs-extension-functions.cc b/src/fs-extension-functions.cc index a9d34fc6..39b1d89e 100644 --- a/src/fs-extension-functions.cc +++ b/src/fs-extension-functions.cc @@ -29,24 +29,29 @@ * @file fs-extension-functions.cc */ +#include #include #include -#include #include #include #include #include #include +#include "base/auto_fd.hh" +#include "base/auto_mem.hh" +#include "base/auto_pid.hh" +#include "base/lnav.console.hh" +#include "base/opt_util.hh" #include "config.h" +#include "lnav.hh" #include "sqlite-extension-func.hh" #include "sqlite3.h" #include "vtab_module.hh" +#include "yajlpp/yajlpp_def.hh" -using namespace mapbox; - -static util::variant +static mapbox::util::variant sql_basename(const char* path_in) { int text_end = -1; @@ -72,7 +77,7 @@ sql_basename(const char* path_in) } } -static util::variant +static mapbox::util::variant sql_dirname(const char* path_in) { ssize_t text_end; @@ -161,6 +166,175 @@ sql_realpath(const char* path) return resolved_path; } +struct shell_exec_options { + std::map> po_env; +}; + +static const json_path_container shell_exec_env_handlers = { + yajlpp::pattern_property_handler(R"((?[^=]+))") + .for_field(&shell_exec_options::po_env), +}; + +static const typed_json_path_container + shell_exec_option_handlers = { + yajlpp::property_handler("env").with_children(shell_exec_env_handlers), +}; + +static blob_auto_buffer +sql_shell_exec(const char* cmd, + nonstd::optional input, + nonstd::optional opts_json) +{ + static const intern_string_t SRC = intern_string::lookup("options"); + + if (lnav_data.ld_flags & LNF_SECURE_MODE) { + throw sqlite_func_error("not available in secure mode"); + } + + shell_exec_options options; + + if (opts_json) { + auto parse_res + = shell_exec_option_handlers.parser_for(SRC).of(opts_json.value()); + + if (parse_res.isErr()) { + throw lnav::console::user_message::error( + "invalid options parameter") + .with_reason(parse_res.unwrapErr()[0]); + } + + options = parse_res.unwrap(); + } + + auto in_pipe_res = auto_pipe::for_child_fd(STDIN_FILENO); + if (in_pipe_res.isErr()) { + throw lnav::console::user_message::error("cannot open input pipe") + .with_reason(in_pipe_res.unwrapErr()); + } + auto in_pipe = in_pipe_res.unwrap(); + auto out_pipe_res = auto_pipe::for_child_fd(STDOUT_FILENO); + if (out_pipe_res.isErr()) { + throw lnav::console::user_message::error("cannot open output pipe") + .with_reason(out_pipe_res.unwrapErr()); + } + auto out_pipe = out_pipe_res.unwrap(); + auto err_pipe_res = auto_pipe::for_child_fd(STDERR_FILENO); + if (err_pipe_res.isErr()) { + throw lnav::console::user_message::error("cannot open error pipe") + .with_reason(err_pipe_res.unwrapErr()); + } + auto err_pipe = err_pipe_res.unwrap(); + auto child_pid_res = lnav::pid::from_fork(); + if (child_pid_res.isErr()) { + throw lnav::console::user_message::error("cannot fork()") + .with_reason(child_pid_res.unwrapErr()); + } + + auto child_pid = child_pid_res.unwrap(); + + in_pipe.after_fork(child_pid.in()); + out_pipe.after_fork(child_pid.in()); + err_pipe.after_fork(child_pid.in()); + + if (child_pid.in_child()) { + const char* args[] = { + getenv_opt("SHELL").value_or("bash"), + "-c", + cmd, + nullptr, + }; + + for (const auto& epair : options.po_env) { + if (epair.second.has_value()) { + setenv(epair.first.c_str(), epair.second->c_str(), 1); + } else { + unsetenv(epair.first.c_str()); + } + } + + execvp(args[0], (char**) args); + _exit(EXIT_FAILURE); + } + + auto out_reader = std::async(std::launch::async, [&out_pipe]() { + auto buffer = auto_buffer::alloc(4096); + + while (true) { + if (buffer.available() < 4096) { + buffer.expand_by(4096); + } + + auto rc = read(out_pipe.read_end(), + buffer.next_available(), + buffer.available()); + if (rc < 0) { + break; + } + if (rc == 0) { + break; + } + buffer.resize_by(rc); + } + + return buffer; + }); + + auto err_reader = std::async(std::launch::async, [&err_pipe]() { + auto buffer = auto_buffer::alloc(4096); + + while (true) { + if (buffer.available() < 4096) { + buffer.expand_by(4096); + } + + auto rc = read(err_pipe.read_end(), + buffer.next_available(), + buffer.available()); + if (rc < 0) { + break; + } + if (rc == 0) { + break; + } + buffer.resize_by(rc); + } + + return buffer; + }); + + if (input) { + auto sf = input.value(); + + while (!sf.empty()) { + auto rc = write(in_pipe.write_end(), sf.data(), sf.length()); + if (rc < 0) { + break; + } + sf = sf.substr(rc); + } + in_pipe.close(); + } + + auto retval = blob_auto_buffer{out_reader.get()}; + + auto finished_child = std::move(child_pid).wait_for_child(); + + if (!finished_child.was_normal_exit()) { + throw sqlite_func_error("child failed with signal {}", + finished_child.term_signal()); + } + + if (finished_child.exit_status() != EXIT_SUCCESS) { + throw lnav::console::user_message::error( + attr_line_t("child failed with exit code ") + .append(lnav::roles::number( + fmt::to_string(finished_child.exit_status())))) + .with_reason(err_reader.get().to_string()); + } + + return retval; +} + int fs_extension_functions(struct FuncDef** basic_funcs, struct FuncDefAgg** agg_funcs) @@ -246,6 +420,28 @@ fs_extension_functions(struct FuncDef** basic_funcs, .with_parameter({"path", "The path to resolve."}) .with_tags({"filename"})), + sqlite_func_adapter::builder( + help_text("shell_exec", + "Executes a shell command and returns its output.") + .sql_function() + .with_parameter({"cmd", "The command to execute."}) + .with_parameter(help_text{ + "input", + "A blob of data to write to the command's standard input."} + .optional()) + .with_parameter( + help_text{"options", + "A JSON object containing options for the " + "execution with the following properties:"} + .optional() + .with_parameter(help_text{ + "env", + "An object containing the environment variables " + "to set or, if NULL, to unset."} + .optional())) + .with_tags({"shell"})) + .with_flags(SQLITE_DIRECTONLY | SQLITE_UTF8), + /* * TODO: add other functions like normpath, ... */ diff --git a/src/help_text_formatter.cc b/src/help_text_formatter.cc index e0b92a0b..0564ba2a 100644 --- a/src/help_text_formatter.cc +++ b/src/help_text_formatter.cc @@ -53,7 +53,8 @@ get_related(const help_text& ht) auto tagged = help_text::TAGGED.equal_range(tag); for (auto tag_iter = tagged.first; tag_iter != tagged.second; - ++tag_iter) { + ++tag_iter) + { if (tag_iter->second == &ht) { continue; } @@ -377,6 +378,17 @@ format_help_text_for_term(const help_text& ht, .append(attr_line_t::from_ansi_str(param.ht_summary), &(tws.with_indent(2 + max_param_name_width + 3))) .append("\n"); + if (!param.ht_parameters.empty()) { + for (const auto& sub_param : param.ht_parameters) { + alb.indent(body_indent + max_param_name_width + 3) + .append(lnav::roles::variable(sub_param.ht_name)) + .append(" - ") + .append( + attr_line_t::from_ansi_str(sub_param.ht_summary), + &(tws.with_indent(2 + max_param_name_width + 5))) + .append("\n"); + } + } } } if (htc == help_text_content::full && !ht.ht_results.empty()) { @@ -637,6 +649,20 @@ format_help_text_for_rst(const help_text& ht, param.ht_name, param.ht_nargs == help_nargs_t::HN_REQUIRED ? "\\*" : "", param.ht_summary); + + if (!param.ht_parameters.empty()) { + fprintf(rst_file, "\n"); + for (const auto& sub_param : param.ht_parameters) { + fmt::fprintf( + rst_file, + " * **%s%s** --- %s\n", + sub_param.ht_name, + sub_param.ht_nargs == help_nargs_t::HN_REQUIRED + ? "\\*" + : "", + sub_param.ht_summary); + } + } } } fmt::fprintf(rst_file, "\n"); diff --git a/src/internals/sql-ref.rst b/src/internals/sql-ref.rst index bf1e0a6c..41ff28f1 100644 --- a/src/internals/sql-ref.rst +++ b/src/internals/sql-ref.rst @@ -127,6 +127,8 @@ CASE *\[base-expr\]* WHEN *cmp-expr* ELSE *\[else-expr\]* END **Parameters** * **base-expr** --- The base expression that is used for comparison in the branches * **cmp-expr** --- The expression to test if this branch should be taken + + * **then-expr\*** --- The result for this branch. * **else-expr** --- The result of this CASE if no branches matched. **Examples** @@ -395,6 +397,8 @@ UPDATE *table* SET *column-name* WHERE *\[cond\]* **Parameters** * **table\*** --- The table to update * **column-name** --- The columns in the table to update. + + * **expr\*** --- The values to place into the column. * **cond** --- The condition used to determine whether a row should be updated. **Examples** @@ -2475,14 +2479,14 @@ parse_url(*url*) .. code-block:: custsqlite ;SELECT parse_url('https://example.com/search?q=hello%20world') - {"scheme":"https","user":null,"password":null,"host":"example.com","port":null,"path":"/search","query":"q=hello%20world","parameters":{"q":"hello world"},"fragment":null} + {"scheme":"https","username":null,"password":null,"host":"example.com","port":null,"path":"/search","query":"q=hello%20world","parameters":{"q":"hello world"},"fragment":null} To parse the URL 'https://alice@[fe80::14ff:4ee5:1215:2fb2]': .. code-block:: custsqlite ;SELECT parse_url('https://alice@[fe80::14ff:4ee5:1215:2fb2]') - {"scheme":"https","user":"alice","password":null,"host":"[fe80::14ff:4ee5:1215:2fb2]","port":null,"path":"/","query":null,"parameters":null,"fragment":null} + {"scheme":"https","username":"alice","password":null,"host":"[fe80::14ff:4ee5:1215:2fb2]","port":null,"path":"/","query":null,"parameters":null,"fragment":null} **See Also** :ref:`anonymize`, :ref:`char`, :ref:`charindex`, :ref:`decode`, :ref:`encode`, :ref:`endswith`, :ref:`extract`, :ref:`group_concat`, :ref:`group_spooky_hash_agg`, :ref:`gunzip`, :ref:`gzip`, :ref:`humanize_duration`, :ref:`humanize_file_size`, :ref:`instr`, :ref:`leftstr`, :ref:`length`, :ref:`logfmt2json`, :ref:`lower`, :ref:`ltrim`, :ref:`padc`, :ref:`padl`, :ref:`padr`, :ref:`printf`, :ref:`proper`, :ref:`regexp_capture_into_json`, :ref:`regexp_capture`, :ref:`regexp_match`, :ref:`regexp_replace`, :ref:`replace`, :ref:`replicate`, :ref:`reverse`, :ref:`rightstr`, :ref:`rtrim`, :ref:`sparkline`, :ref:`spooky_hash`, :ref:`startswith`, :ref:`strfilter`, :ref:`substr`, :ref:`trim`, :ref:`unicode`, :ref:`unparse_url`, :ref:`unparse_url`, :ref:`upper`, :ref:`xpath` @@ -3105,6 +3109,26 @@ rtrim(*str*, *\[chars\]*) ---- +.. _shell_exec: + +shell_exec(*cmd*, *\[input\]*, *\[options\]*) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + Executes a shell command and returns its output. + + **Parameters** + * **cmd\*** --- The command to execute. + * **input** --- A blob of data to write to the command's standard input. + * **options** --- A JSON object containing options for the execution with the following properties: + + * **env** --- An object containing the environment variables to set or, if NULL, to unset. + + **See Also** + + +---- + + .. _sign: sign(*num*) diff --git a/src/lnav.cc b/src/lnav.cc index 77d2588a..6721d437 100644 --- a/src/lnav.cc +++ b/src/lnav.cc @@ -2882,6 +2882,9 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%' .with_filename(file_path); isc::to().send( [ul](auto& clooper) { clooper.add_request(ul); }); + } else if (file_path.find("://") != std::string::npos) { + lnav_data.ld_commands.emplace_back( + fmt::format(FMT_STRING(":open {}"), file_path)); } #endif else if (is_glob(file_path)) diff --git a/src/lnav_commands.cc b/src/lnav_commands.cc index de53903b..270257bc 100644 --- a/src/lnav_commands.cc +++ b/src/lnav_commands.cc @@ -80,6 +80,7 @@ #include "sysclip.hh" #include "tailer/tailer.looper.hh" #include "text_anonymizer.hh" +#include "url_handler.cfg.hh" #include "url_loader.hh" #include "yajl/api/yajl_parse.h" #include "yajlpp/json_op.hh" @@ -2478,6 +2479,12 @@ com_open(exec_context& ec, std::string cmdline, std::vector& args) std::vector> files_to_front; std::vector closed_files; + logfile_open_options loo; + + auto prov = ec.get_provenance(); + if (prov) { + loo.with_filename(prov->fo_name); + } for (auto fn : split_args) { file_location_t file_loc; @@ -2538,12 +2545,59 @@ com_open(exec_context& ec, std::string cmdline, std::vector& args) retval = ""; } #endif + } else if (fn.find("://") != std::string::npos) { + const auto& cfg + = injector::get(); + + auto* cu = curl_url(); + auto set_rc = curl_url_set( + cu, CURLUPART_URL, fn.c_str(), CURLU_NON_SUPPORT_SCHEME); + if (set_rc != CURLUE_OK) { + return Err(lnav::console::user_message::error( + attr_line_t("invalid URL: ") + .append(lnav::roles::file(fn))) + .with_reason(curl_url_strerror(set_rc))); + } + + char* scheme_part = nullptr; + auto get_rc + = curl_url_get(cu, CURLUPART_SCHEME, &scheme_part, 0); + if (get_rc != CURLUE_OK) { + return Err(lnav::console::user_message::error( + attr_line_t("cannot get scheme from URL: ") + .append(lnav::roles::file(fn))) + .with_reason(curl_url_strerror(set_rc))); + } + + auto proto_iter = cfg.c_schemes.find(scheme_part); + if (proto_iter == cfg.c_schemes.end()) { + return Err( + lnav::console::user_message::error( + attr_line_t("no defined handler for URL scheme: ") + .append(lnav::roles::file(scheme_part))) + .with_reason(curl_url_strerror(set_rc))); + } + + auto path_and_args + = fmt::format(FMT_STRING("{} {}"), + proto_iter->second.p_handler.pp_value, + fn); + + exec_context::provenance_guard pg(&ec, + exec_context::file_open{fn}); + + auto exec_res = execute_file(ec, path_and_args); + if (exec_res.isErr()) { + return exec_res; + } + + retval = "info: watching -- " + fn; } else if (is_glob(fn.c_str())) { - fc.fc_file_names.emplace(fn, logfile_open_options()); + fc.fc_file_names.emplace(fn, loo); retval = "info: watching -- " + fn; } else if (stat(fn.c_str(), &st) == -1) { if (fn.find(':') != std::string::npos) { - fc.fc_file_names.emplace(fn, logfile_open_options()); + fc.fc_file_names.emplace(fn, loo); retval = "info: watching -- " + fn; } else { auto um = lnav::console::user_message::error( @@ -2573,6 +2627,9 @@ com_open(exec_context& ec, std::string cmdline, std::vector& args) } else { auto desc = fmt::format(FMT_STRING("FIFO [{}]"), lnav_data.ld_fifo_counter++); + if (prov) { + desc = prov->fo_name; + } auto create_piper_res = lnav::piper::create_looper( desc, std::move(fifo_fd), auto_fd{}); if (create_piper_res.isErr()) { @@ -2602,8 +2659,7 @@ com_open(exec_context& ec, std::string cmdline, std::vector& args) if (dir_wild[dir_wild.size() - 1] == '/') { dir_wild.resize(dir_wild.size() - 1); } - fc.fc_file_names.emplace(dir_wild + "/*", - logfile_open_options()); + fc.fc_file_names.emplace(dir_wild + "/*", loo); retval = "info: watching -- " + dir_wild; } else if (!S_ISREG(st.st_mode)) { auto um = lnav::console::user_message::error( @@ -2627,7 +2683,7 @@ com_open(exec_context& ec, std::string cmdline, std::vector& args) return Err(um); } else { fn = abspath.in(); - fc.fc_file_names.emplace(fn, logfile_open_options()); + fc.fc_file_names.emplace(fn, loo); retval = "info: opened -- " + fn; files_to_front.emplace_back(fn, file_loc); @@ -4173,12 +4229,34 @@ com_sh(exec_context& ec, std::string cmdline, std::vector& args) nullptr, }; + for (const auto& pair : ec.ec_local_vars.top()) { + pair.second.match( + [&pair](const std::string& val) { + setenv(pair.first.c_str(), val.c_str(), 1); + }, + [&pair](const string_fragment& sf) { + setenv(pair.first.c_str(), sf.to_string().c_str(), 1); + }, + [](null_value_t) {}, + [&pair](int64_t val) { + setenv( + pair.first.c_str(), fmt::to_string(val).c_str(), 1); + }, + [&pair](double val) { + setenv( + pair.first.c_str(), fmt::to_string(val).c_str(), 1); + }); + } + execvp(exec_args[0], (char**) exec_args); _exit(EXIT_FAILURE); } + auto display_name = ec.get_provenance() + .value_or(exec_context::file_open{carg}) + .fo_name; auto create_piper_res - = lnav::piper::create_looper(carg, + = lnav::piper::create_looper(display_name, std::move(out_pipe.read_end()), std::move(err_pipe.read_end())); @@ -4190,11 +4268,12 @@ com_sh(exec_context& ec, std::string cmdline, std::vector& args) return Err(um); } - lnav_data.ld_active_files.fc_file_names[carg].with_piper( + lnav_data.ld_active_files.fc_file_names[display_name].with_piper( create_piper_res.unwrap()); lnav_data.ld_child_pollers.emplace_back( child_poller{std::move(child), [](auto& fc, auto& child) {}}); - lnav_data.ld_files_to_front.emplace_back(carg, file_location_t{}); + lnav_data.ld_files_to_front.emplace_back(display_name, + file_location_t{}); } return Ok(std::string()); diff --git a/src/lnav_config.cc b/src/lnav_config.cc index d8144c60..6d9fd67b 100644 --- a/src/lnav_config.cc +++ b/src/lnav_config.cc @@ -96,6 +96,9 @@ static auto tc = injector::bind::to_instance( static auto scc = injector::bind::to_instance( +[]() { return &lnav_config.lc_sysclip; }); +static auto uh = injector::bind::to_instance( + +[]() { return &lnav_config.lc_url_handlers; }); + static auto lsc = injector::bind::to_instance( +[]() { return &lnav_config.lc_log_source; }); @@ -1257,6 +1260,33 @@ static const struct json_path_container log_source_handlers = { .with_children(log_source_watch_handlers), }; +static const struct json_path_container url_scheme_handlers = { + yajlpp::property_handler("handler") + .with_description( + "The name of the lnav script that can handle URLs " + "with of this scheme. This should not include the '.lnav' suffix.") + .with_pattern(R"(^[\w\-]+(?!\.lnav)$)") + .for_field(&lnav::url_handler::scheme::p_handler), +}; + +static const struct json_path_container url_handlers = { + yajlpp::pattern_property_handler(R"((?\w+))") + .with_description("Definition of a custom URL scheme") + .with_obj_provider( + [](const yajlpp_provider_context& ypc, _lnav_config* root) { + auto& retval = root->lc_url_handlers + .c_schemes[ypc.get_substr("url_scheme")]; + return &retval; + }) + .with_path_provider<_lnav_config>( + [](struct _lnav_config* cfg, std::vector& paths_out) { + for (const auto& iter : cfg->lc_url_handlers.c_schemes) { + paths_out.emplace_back(iter.first); + } + }) + .with_children(url_scheme_handlers), +}; + static const struct json_path_container tuning_handlers = { yajlpp::property_handler("archive-manager") .with_description("Settings related to opening archive files") @@ -1276,6 +1306,9 @@ static const struct json_path_container tuning_handlers = { yajlpp::property_handler("clipboard") .with_description("Settings related to the clipboard") .with_children(sysclip_handlers), + yajlpp::property_handler("url-scheme") + .with_description("Settings related to custom URL handling") + .with_children(url_handlers), }; const char* DEFAULT_CONFIG_SCHEMA diff --git a/src/lnav_config.hh b/src/lnav_config.hh index e38f8483..278f5265 100644 --- a/src/lnav_config.hh +++ b/src/lnav_config.hh @@ -54,6 +54,7 @@ #include "sysclip.cfg.hh" #include "tailer/tailer.looper.cfg.hh" #include "top_status_source.cfg.hh" +#include "url_handler.cfg.hh" /** * Check if an experimental feature should be enabled by @@ -115,6 +116,7 @@ struct _lnav_config { lnav::logfile::config lc_logfile; tailer::config lc_tailer; sysclip::config lc_sysclip; + lnav::url_handler::config lc_url_handlers; logfile_sub_source_ns::config lc_log_source; }; diff --git a/src/logfile.cc b/src/logfile.cc index c3c9b1ab..c6b53420 100644 --- a/src/logfile.cc +++ b/src/logfile.cc @@ -118,6 +118,9 @@ logfile::open(std::string filename, const logfile_open_options& loo, auto_fd fd) (long long) lf->lf_stat.st_size, (long long) lf->lf_stat.st_mtime, lf->lf_filename.c_str()); + if (lf->lf_actual_path) { + log_info(" actual_path=%s", lf->lf_actual_path->c_str()); + } if (!lf->lf_options.loo_filename.empty()) { lf->set_filename(lf->lf_options.loo_filename); diff --git a/src/piper.looper.cc b/src/piper.looper.cc index b8372d9f..a9e1e649 100644 --- a/src/piper.looper.cc +++ b/src/piper.looper.cc @@ -98,6 +98,8 @@ enum class read_mode_t { void looper::loop() { + static const auto FORCE_MTIME_UPDATE_DURATION = 8h; + const auto& cfg = injector::get(); struct pollfd pfd[2]; struct { @@ -122,11 +124,14 @@ looper::loop() this->l_name.c_str(), this->l_stdout.get(), this->l_stderr.get()); + this->l_stdout.non_blocking(); captured_fds[0].lb.set_fd(this->l_stdout); if (this->l_stderr.has_value()) { + this->l_stderr.non_blocking(); captured_fds[1].lb.set_fd(this->l_stderr); } captured_fds[1].cf_level = LEVEL_ERROR; + auto last_write = std::chrono::system_clock::now(); do { static const auto TIMEOUT = std::chrono::duration_cast(1s).count(); @@ -156,9 +161,16 @@ looper::loop() // update the timestamp to keep the file alive from any // cleanup processes if (outfd.has_value()) { - log_perror(futimes(outfd.get(), nullptr)); + auto now = std::chrono::system_clock::now(); + + if ((now - last_write) >= FORCE_MTIME_UPDATE_DURATION) { + last_write = now; + log_perror(futimes(outfd.get(), nullptr)); + } } continue; + } else { + last_write = std::chrono::system_clock::now(); } for (auto& cap : captured_fds) { while (this->l_looping) { diff --git a/src/root-config.json b/src/root-config.json index b8a2ea6c..8bf8194c 100644 --- a/src/root-config.json +++ b/src/root-config.json @@ -78,6 +78,11 @@ } } } + }, + "url-scheme": { + "docker": { + "handler": "docker-url-handler" + } } } } diff --git a/src/scripts/docker-url-handler.lnav b/src/scripts/docker-url-handler.lnav new file mode 100755 index 00000000..d250b5c4 --- /dev/null +++ b/src/scripts/docker-url-handler.lnav @@ -0,0 +1,17 @@ +# +# @synopsis: docker-url-handler +# @description: Internal script to handle opening docker URLs +# + +;SELECT CASE path + WHEN '/' THEN + 'docker logs -f ' || hostname + ELSE + 'docker exec ' || hostname || ' tail -n +0 -F "' || path || '"' + END AS cmd + FROM (SELECT + jget(url, '/host') AS hostname, + jget(url, '/path') AS path + FROM (SELECT parse_url($1) AS url)) + +:sh eval $cmd diff --git a/src/scripts/pcap_log-converter.sh b/src/scripts/pcap_log-converter.sh index 5b9642b2..94ef5379 100755 --- a/src/scripts/pcap_log-converter.sh +++ b/src/scripts/pcap_log-converter.sh @@ -6,5 +6,8 @@ if ! command -v tshark; then exit 1 fi +# We want tshark output to come in UTC +export TZ=UTC + # Use tshark to convert the pcap file into a JSON-lines log file exec tshark -T ek -P -V -t ad -r $2 diff --git a/src/scripts/scripts.am b/src/scripts/scripts.am index b180f967..3505e3f8 100644 --- a/src/scripts/scripts.am +++ b/src/scripts/scripts.am @@ -1,6 +1,7 @@ BUILTIN_LNAVSCRIPTS = \ $(srcdir)/scripts/dhclient-summary.lnav \ + $(srcdir)/scripts/docker-url-handler.lnav \ $(srcdir)/scripts/lnav-pop-view.lnav \ $(srcdir)/scripts/partition-by-boot.lnav \ $(srcdir)/scripts/rename-stdin.lnav \ diff --git a/src/service_tags.hh b/src/service_tags.hh index 6d134278..702b13ba 100644 --- a/src/service_tags.hh +++ b/src/service_tags.hh @@ -34,14 +34,11 @@ namespace services { -struct main_t { -}; -struct ui_t { -}; -struct curl_streamer_t { -}; -struct remote_tailer_t { -}; +struct main_t {}; +struct ui_t {}; +struct curl_streamer_t {}; +struct remote_tailer_t {}; +struct url_handler_t {}; } // namespace services diff --git a/src/string-extension-functions.cc b/src/string-extension-functions.cc index 29a02301..23606684 100644 --- a/src/string-extension-functions.cc +++ b/src/string-extension-functions.cc @@ -635,17 +635,18 @@ const char* curl_url_strerror(CURLUcode error); #endif static json_string -sql_parse_url(string_fragment url_frag) +sql_parse_url(std::string url) { static auto* CURL_HANDLE = get_curl_easy(); auto_mem cu(curl_url_cleanup); cu = curl_url(); - auto rc = curl_url_set(cu, CURLUPART_URL, url_frag.data(), 0); + auto rc = curl_url_set( + cu, CURLUPART_URL, url.c_str(), CURLU_NON_SUPPORT_SCHEME); if (rc != CURLUE_OK) { throw lnav::console::user_message::error( - attr_line_t("invalid URL: ").append_quoted(url_frag.to_string())) + attr_line_t("invalid URL: ").append(lnav::roles::file(url))) .with_reason(curl_url_strerror(rc)); } @@ -663,7 +664,7 @@ sql_parse_url(string_fragment url_frag) } else { root.gen(); } - root.gen("user"); + root.gen("username"); rc = curl_url_get(cu, CURLUPART_USER, url_part.out(), CURLU_URLDECODE); if (rc == CURLUE_OK) { root.gen(string_fragment::from_c_str(url_part.in())); @@ -708,6 +709,11 @@ sql_parse_url(string_fragment url_frag) robin_hood::unordered_set seen_keys; yajlpp_map query_map(gen); + for (size_t lpc = 0; url_part.in()[lpc]; lpc++) { + if (url_part.in()[lpc] == '+') { + url_part.in()[lpc] = ' '; + } + } auto query_frag = string_fragment::from_c_str(url_part.in()); auto remaining = query_frag; @@ -727,6 +733,7 @@ sql_parse_url(string_fragment url_frag) kv_pair_encoded.data(), kv_pair_encoded.length(), &out_len); + auto kv_pair_frag = string_fragment::from_bytes(kv_pair.in(), out_len); auto eq_index_opt = kv_pair_frag.find('='); @@ -788,7 +795,7 @@ struct url_parts { }; static const json_path_container url_params_handlers = { - yajlpp::pattern_property_handler("(?.+)") + yajlpp::pattern_property_handler("(?.*)") .for_field(&url_parts::up_parameters), }; @@ -812,7 +819,7 @@ sql_unparse_url(string_fragment in) auto parse_res = url_parts_handlers.parser_for(SRC).of(in); if (parse_res.isErr()) { - throw parse_res.unwrapErr(); + throw parse_res.unwrapErr()[0]; } auto up = parse_res.unwrap(); diff --git a/src/textfile_sub_source.cc b/src/textfile_sub_source.cc index 31cba74e..96a1d7af 100644 --- a/src/textfile_sub_source.cc +++ b/src/textfile_sub_source.cc @@ -51,7 +51,9 @@ textfile_sub_source::text_line_count() auto rend_iter = this->tss_rendered_files.find(lf->get_filename()); if (rend_iter == this->tss_rendered_files.end()) { auto* lfo = (line_filter_observer*) lf->get_logline_observer(); - retval = lfo->lfo_filter_state.tfs_index.size(); + if (lfo != nullptr) { + retval = lfo->lfo_filter_state.tfs_index.size(); + } } else { retval = rend_iter->second.rf_text_source->text_line_count(); } @@ -72,7 +74,9 @@ textfile_sub_source::text_value_for_line(textview_curses& tc, if (rend_iter == this->tss_rendered_files.end()) { auto* lfo = dynamic_cast( lf->get_logline_observer()); - if (line < 0 || line >= lfo->lfo_filter_state.tfs_index.size()) { + if (lfo == nullptr || line < 0 + || line >= lfo->lfo_filter_state.tfs_index.size()) + { value_out.clear(); } else { auto ll = lf->begin() + lfo->lfo_filter_state.tfs_index[line]; @@ -119,7 +123,9 @@ textfile_sub_source::text_attrs_for_line(textview_curses& tc, } else { auto* lfo = dynamic_cast(lf->get_logline_observer()); - if (row >= 0 && row < lfo->lfo_filter_state.tfs_index.size()) { + if (lfo != nullptr && row >= 0 + && row < lfo->lfo_filter_state.tfs_index.size()) + { auto ll = lf->begin() + lfo->lfo_filter_state.tfs_index[row]; value_out.emplace_back(lr, SA_LEVEL.value(ll->get_msg_level())); @@ -171,7 +177,9 @@ textfile_sub_source::text_size_for_line(textview_curses& tc, if (rend_iter == this->tss_rendered_files.end()) { auto* lfo = dynamic_cast( lf->get_logline_observer()); - if (line < 0 || line >= lfo->lfo_filter_state.tfs_index.size()) { + if (lfo == nullptr || line < 0 + || line >= lfo->lfo_filter_state.tfs_index.size()) + { } else { retval = lf->message_byte_length( diff --git a/src/url_handler.cc b/src/url_handler.cc new file mode 100644 index 00000000..a44dc9dc --- /dev/null +++ b/src/url_handler.cc @@ -0,0 +1,259 @@ +/** + * Copyright (c) 2023, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "url_handler.hh" + +#include + +#include "base/fs_util.hh" +#include "base/injector.hh" +#include "base/paths.hh" +#include "lnav.hh" +#include "service_tags.hh" +#include "url_handler.cfg.hh" + +namespace lnav { +namespace url_handler { + +void +looper::handler_looper::loop_body() +{ + pollfd pfd[1]; + + pfd[0].events = POLLIN; + pfd[0].fd = this->hl_line_buffer.get_fd(); + pfd[0].revents = 0; + + log_debug("doing url handler poll"); + auto prc = poll(pfd, 1, 100); + log_debug("poll rc %d", prc); + if (prc > 0) { + auto load_res + = this->hl_line_buffer.load_next_line(this->hl_last_range); + + if (load_res.isErr()) { + log_error("failed to load next line: %s", + load_res.unwrapErr().c_str()); + this->s_looping = false; + } else { + auto li = load_res.unwrap(); + + log_debug("li %d %d:%d", + li.li_partial, + li.li_file_range.fr_offset, + li.li_file_range.fr_size); + if (!li.li_partial && !li.li_file_range.empty()) { + auto read_res + = this->hl_line_buffer.read_range(li.li_file_range); + + if (read_res.isErr()) { + log_error("cannot read line: %s", + read_res.unwrapErr().c_str()); + } else { + auto cmd = trim(to_string(read_res.unwrap())); + log_debug("url handler command: %s", cmd.c_str()); + + isc::to().send( + [cmd](auto& mlooper) { + auto exec_res + = execute_any(lnav_data.ld_exec_context, cmd); + if (exec_res.isErr()) { + auto um = exec_res.unwrapErr(); + log_error( + "wtf %s", + um.to_attr_line().get_string().c_str()); + } + }); + } + this->hl_last_range = li.li_file_range; + } + } + + if (this->hl_line_buffer.is_pipe_closed()) { + log_info("URL handler finished"); + this->s_looping = false; + } + } +} + +Result +looper::open(std::string url) +{ + const auto& cfg = injector::get(); + + log_info("open request for URL: %s", url.c_str()); + + auto* cu = curl_url(); + auto set_rc = curl_url_set( + cu, CURLUPART_URL, url.c_str(), CURLU_NON_SUPPORT_SCHEME); + if (set_rc != CURLUE_OK) { + return Err( + lnav::console::user_message::error( + attr_line_t("invalid URL: ").append(lnav::roles::file(url))) + .with_reason(curl_url_strerror(set_rc))); + } + + char* scheme_part; + auto get_rc = curl_url_get(cu, CURLUPART_SCHEME, &scheme_part, 0); + if (get_rc != CURLUE_OK) { + return Err(lnav::console::user_message::error( + attr_line_t("cannot get scheme from URL: ") + .append(lnav::roles::file(url))) + .with_reason(curl_url_strerror(set_rc))); + } + + auto proto_iter = cfg.c_schemes.find(scheme_part); + if (proto_iter == cfg.c_schemes.end()) { + return Err(lnav::console::user_message::error( + attr_line_t("no defined handler for URL scheme: ") + .append(lnav::roles::file(scheme_part))) + .with_reason(curl_url_strerror(set_rc))); + } + + log_info("found URL handler: %s", + proto_iter->second.p_handler.pp_value.c_str()); + auto err_pipe_res = auto_pipe::for_child_fd(STDERR_FILENO); + if (err_pipe_res.isErr()) { + return Err( + lnav::console::user_message::error( + attr_line_t("cannot open URL: ").append(lnav::roles::file(url))) + .with_reason(err_pipe_res.unwrapErr())); + } + auto err_pipe = err_pipe_res.unwrap(); + auto out_pipe_res = auto_pipe::for_child_fd(STDOUT_FILENO); + if (out_pipe_res.isErr()) { + return Err( + lnav::console::user_message::error( + attr_line_t("cannot open URL: ").append(lnav::roles::file(url))) + .with_reason(out_pipe_res.unwrapErr())); + } + auto out_pipe = out_pipe_res.unwrap(); + auto child_pid_res = lnav::pid::from_fork(); + if (child_pid_res.isErr()) { + return Err( + lnav::console::user_message::error( + attr_line_t("cannot open URL: ").append(lnav::roles::file(url))) + .with_reason(child_pid_res.unwrapErr())); + } + + auto child_pid = child_pid_res.unwrap(); + + out_pipe.after_fork(child_pid.in()); + err_pipe.after_fork(child_pid.in()); + + auto name = proto_iter->second.p_handler.pp_value; + if (child_pid.in_child()) { + auto dev_null = ::open("/dev/null", O_RDONLY | O_CLOEXEC); + + dup2(dev_null, STDIN_FILENO); + + char* host_part = nullptr; + curl_url_get(cu, CURLUPART_HOST, &host_part, 0); + std::string host_part_str; + if (host_part != nullptr) { + host_part_str = host_part; + } + + auto source_path = ghc::filesystem::path{ + proto_iter->second.p_handler.pp_location.sl_source.get()}; + auto new_path = lnav::filesystem::build_path({ + source_path.parent_path(), + lnav::paths::dotlnav() / "formats/default", + }); + setenv("PATH", new_path.c_str(), 1); + setenv("URL_HOSTNAME", host_part_str.c_str(), 1); + + const char* args[] = { + name.c_str(), + nullptr, + }; + + execvp(name.c_str(), (char**) args); + _exit(EXIT_FAILURE); + } + + auto error_queue = std::make_shared>(); + std::thread err_reader([err = std::move(err_pipe.read_end()), + name, + error_queue, + child_pid = child_pid.in()]() mutable { + line_buffer lb; + file_range pipe_range; + bool done = false; + + log_debug("error reader"); + lb.set_fd(err); + while (!done) { + auto load_res = lb.load_next_line(pipe_range); + + if (load_res.isErr()) { + done = true; + } else { + auto li = load_res.unwrap(); + + pipe_range = li.li_file_range; + if (li.li_file_range.empty()) { + done = true; + } else { + lb.read_range(li.li_file_range) + .then([name, error_queue, child_pid](auto sbr) { + auto line_str = string_fragment( + sbr.get_data(), 0, sbr.length()) + .trim("\n"); + if (error_queue->size() < 5) { + error_queue->emplace_back(line_str.to_string()); + } + + log_debug("%s[%d]: %.*s", + name.c_str(), + child_pid, + line_str.length(), + line_str.data()); + }); + } + } + } + }); + err_reader.detach(); + + auto child = std::make_shared( + url, std::move(child_pid), std::move(out_pipe.read_end())); + this->s_children.add_child_service(child); + this->l_children[url] = child; + + return Ok(); +} + +void +looper::close(std::string url) +{ +} + +} // namespace url_handler +} // namespace lnav diff --git a/src/url_handler.cfg.hh b/src/url_handler.cfg.hh new file mode 100644 index 00000000..51b14a95 --- /dev/null +++ b/src/url_handler.cfg.hh @@ -0,0 +1,52 @@ +/** + * Copyright (c) 2023, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef lnav_url_handler_cfg_hh +#define lnav_url_handler_cfg_hh + +#include +#include + +#include "yajlpp/yajlpp.hh" + +namespace lnav { +namespace url_handler { + +struct scheme { + positioned_property p_handler; +}; + +struct config { + std::map c_schemes; +}; + +} // namespace url_handler +} // namespace lnav + +#endif diff --git a/src/url_handler.hh b/src/url_handler.hh new file mode 100644 index 00000000..61715004 --- /dev/null +++ b/src/url_handler.hh @@ -0,0 +1,95 @@ +/** + * Copyright (c) 2023, Timothy Stack + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of Timothy Stack nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// XXX This code is unused right now, but it's nice and I don't want to delete +// it just yet. + +#ifndef lnav_url_handler_hh +#define lnav_url_handler_hh + +#include +#include + +#include "base/auto_fd.hh" +#include "base/auto_pid.hh" +#include "base/isc.hh" +#include "base/lnav.console.hh" +#include "base/result.h" +#include "line_buffer.hh" +#include "mapbox/variant.hpp" + +namespace lnav { +namespace url_handler { + +class looper : public isc::service { +public: + Result open(std::string url); + + void close(std::string url); + +private: + class handler_looper : public isc::service { + public: + handler_looper(std::string url, + auto_pid pid, + auto_fd infd) + : isc::service(url), hl_state(std::move(pid)) + { + this->hl_line_buffer.set_fd(infd); + } + + protected: + void loop_body() override; + + std::chrono::milliseconds compute_timeout( + mstime_t current_time) const override + { + return std::chrono::milliseconds{0}; + } + + public: + struct handler_completed {}; + + using state_v = mapbox::util::variant, + handler_completed>; + + file_range hl_last_range; + line_buffer hl_line_buffer; + state_v hl_state; + }; + + struct child {}; + + std::map> l_children; +}; + +} // namespace url_handler +} // namespace lnav + +#endif diff --git a/src/yajlpp/yajlpp_def.hh b/src/yajlpp/yajlpp_def.hh index 6a67e590..d4331277 100644 --- a/src/yajlpp/yajlpp_def.hh +++ b/src/yajlpp/yajlpp_def.hh @@ -842,6 +842,14 @@ struct json_path_handler : public json_path_handler_base { return 1; }; + this->add_cb(null_field_cb); + this->jph_null_cb = [args...](yajlpp_parse_context* ypc) { + auto* obj = ypc->ypc_obj_stack.top(); + + json_path_handler::get_field(obj, args...) = nonstd::nullopt; + + return 1; + }; this->jph_gen_callback = [args...](yajlpp_gen_context& ygc, const json_path_handler_base& jph, yajl_gen handle) { @@ -1360,7 +1368,9 @@ public: const string_fragment& json) { if (this->yp_parse_context.parse_doc(json)) { - return Ok(std::move(this->yp_obj)); + if (this->yp_errors.empty()) { + return Ok(std::move(this->yp_obj)); + } } return Err(std::move(this->yp_errors)); diff --git a/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out b/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out index e9fed923..34f7a36a 100644 --- a/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out +++ b/test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out @@ -3761,6 +3761,20 @@ For support questions, email: +shell_exec(cmd, [input], [options]) +══════════════════════════════════════════════════════════════════════ + Executes a shell command and returns its output. +Parameters + cmd The command to execute. + input A blob of data to write to the command's + standard input. + options A JSON object containing options for the + execution with the following properties: + env - An object containing the environment variables to + set or, if NULL, to unset. +See Also + + sign(num) ══════════════════════════════════════════════════════════════════════ Returns the sign of the given number as -1, 0, or 1 @@ -4330,6 +4344,7 @@ For support questions, email: comparison in the branches cmp-expr The expression to test if this branch should be taken + then-expr - The result for this branch. else-expr The result of this CASE if no branches matched. @@ -4424,6 +4439,7 @@ For support questions, email: Parameters table The table to update column-name The columns in the table to update. + expr - The values to place into the column. cond The condition used to determine whether a row should be updated. diff --git a/test/expected/test_sql_str_func.sh_0947bfe7ec626eaa0409a45b10fcbb634fb12eb7.out b/test/expected/test_sql_str_func.sh_0947bfe7ec626eaa0409a45b10fcbb634fb12eb7.out index e1446538..49ed7e27 100644 --- a/test/expected/test_sql_str_func.sh_0947bfe7ec626eaa0409a45b10fcbb634fb12eb7.out +++ b/test/expected/test_sql_str_func.sh_0947bfe7ec626eaa0409a45b10fcbb634fb12eb7.out @@ -1,2 +1,2 @@ Row 0: - Column parse_url('https://example.com/'): {"scheme":"https","user":null,"password":null,"host":"example.com","port":null,"path":"/","query":null,"parameters":null,"fragment":null} + Column parse_url('https://example.com/'): {"scheme":"https","username":null,"password":null,"host":"example.com","port":null,"path":"/","query":null,"parameters":null,"fragment":null} diff --git a/test/expected/test_sql_str_func.sh_51766b600fd158a9e0677f6b0fa31b83537b2e5b.out b/test/expected/test_sql_str_func.sh_51766b600fd158a9e0677f6b0fa31b83537b2e5b.out index 061b9ede..5576c1f8 100644 --- a/test/expected/test_sql_str_func.sh_51766b600fd158a9e0677f6b0fa31b83537b2e5b.out +++ b/test/expected/test_sql_str_func.sh_51766b600fd158a9e0677f6b0fa31b83537b2e5b.out @@ -1,2 +1,2 @@ Row 0: - Column parse_url('https://example.com/search?flag&flag2&=def'): {"scheme":"https","user":null,"password":null,"host":"example.com","port":null,"path":"/search","query":"flag&flag2&=def","parameters":{"flag":null,"flag2":null,"":"def"},"fragment":null} + Column parse_url('https://example.com/search?flag&flag2&=def'): {"scheme":"https","username":null,"password":null,"host":"example.com","port":null,"path":"/search","query":"flag&flag2&=def","parameters":{"flag":null,"flag2":null,"":"def"},"fragment":null} diff --git a/test/expected/test_sql_str_func.sh_6ff984d8ed3e5099376d19f0dd20d5fd1ed42494.out b/test/expected/test_sql_str_func.sh_6ff984d8ed3e5099376d19f0dd20d5fd1ed42494.out index d1dcc93a..0a56fcc1 100644 --- a/test/expected/test_sql_str_func.sh_6ff984d8ed3e5099376d19f0dd20d5fd1ed42494.out +++ b/test/expected/test_sql_str_func.sh_6ff984d8ed3e5099376d19f0dd20d5fd1ed42494.out @@ -1,2 +1,2 @@ Row 0: - Column parse_url('https://example.com/sea%26rch?flag&flag2&=def#frag1%20space'): {"scheme":"https","user":null,"password":null,"host":"example.com","port":null,"path":"/sea&rch","query":"flag&flag2&=def","parameters":{"flag":null,"flag2":null,"":"def"},"fragment":"frag1 space"} + Column parse_url('https://example.com/sea%26rch?flag&flag2&=def#frag1%20space'): {"scheme":"https","username":null,"password":null,"host":"example.com","port":null,"path":"/sea&rch","query":"flag&flag2&=def","parameters":{"flag":null,"flag2":null,"":"def"},"fragment":"frag1 space"} diff --git a/test/expected/test_sql_str_func.sh_805ca5e97fbf1ed56f2e920befd963255ba190b6.out b/test/expected/test_sql_str_func.sh_805ca5e97fbf1ed56f2e920befd963255ba190b6.out index 84c7397c..01169ceb 100644 --- a/test/expected/test_sql_str_func.sh_805ca5e97fbf1ed56f2e920befd963255ba190b6.out +++ b/test/expected/test_sql_str_func.sh_805ca5e97fbf1ed56f2e920befd963255ba190b6.out @@ -1,2 +1,2 @@ Row 0: - Column parse_url('https://example.com/search?flag&flag2'): {"scheme":"https","user":null,"password":null,"host":"example.com","port":null,"path":"/search","query":"flag&flag2","parameters":{"flag":null,"flag2":null},"fragment":null} + Column parse_url('https://example.com/search?flag&flag2'): {"scheme":"https","username":null,"password":null,"host":"example.com","port":null,"path":"/search","query":"flag&flag2","parameters":{"flag":null,"flag2":null},"fragment":null} diff --git a/test/expected/test_sql_str_func.sh_a515ba81cc3655c602da28cd0fa1a186d5e9a6e1.err b/test/expected/test_sql_str_func.sh_a515ba81cc3655c602da28cd0fa1a186d5e9a6e1.err index 475a9b23..9a586b45 100644 --- a/test/expected/test_sql_str_func.sh_a515ba81cc3655c602da28cd0fa1a186d5e9a6e1.err +++ b/test/expected/test_sql_str_func.sh_a515ba81cc3655c602da28cd0fa1a186d5e9a6e1.err @@ -1 +1 @@ -error: sqlite3_exec failed -- lnav-error:{"level":"error","message":{"str":"invalid URL: “https://example.com:100000”","attrs":[]},"reason":{"str":"Port number was not a decimal number between 0 and 65535","attrs":[]},"snippets":[],"help":{"str":"","attrs":[]}} +error: sqlite3_exec failed -- lnav-error:{"level":"error","message":{"str":"invalid URL: https://example.com:100000","attrs":[{"start":13,"end":39,"type":"role","value":51}]},"reason":{"str":"Port number was not a decimal number between 0 and 65535","attrs":[]},"snippets":[],"help":{"str":"","attrs":[]}} diff --git a/test/expected/test_sql_str_func.sh_b088735cf46f23ca3d5fb3da41f07a6a3b1cba35.out b/test/expected/test_sql_str_func.sh_b088735cf46f23ca3d5fb3da41f07a6a3b1cba35.out index e93f55d5..7b9ef252 100644 --- a/test/expected/test_sql_str_func.sh_b088735cf46f23ca3d5fb3da41f07a6a3b1cba35.out +++ b/test/expected/test_sql_str_func.sh_b088735cf46f23ca3d5fb3da41f07a6a3b1cba35.out @@ -1,2 +1,2 @@ Row 0: - Column parse_url('https://example.com'): {"scheme":"https","user":null,"password":null,"host":"example.com","port":null,"path":"/","query":null,"parameters":null,"fragment":null} + Column parse_url('https://example.com'): {"scheme":"https","username":null,"password":null,"host":"example.com","port":null,"path":"/","query":null,"parameters":null,"fragment":null} diff --git a/test/expected/test_sql_str_func.sh_bac7f6531a2adf70cd1871fb13eab26dff133b7c.out b/test/expected/test_sql_str_func.sh_bac7f6531a2adf70cd1871fb13eab26dff133b7c.out index daf8e0af..ee5f9c78 100644 --- a/test/expected/test_sql_str_func.sh_bac7f6531a2adf70cd1871fb13eab26dff133b7c.out +++ b/test/expected/test_sql_str_func.sh_bac7f6531a2adf70cd1871fb13eab26dff133b7c.out @@ -1,2 +1,2 @@ Row 0: - Column parse_url('https://example.com/search?flag'): {"scheme":"https","user":null,"password":null,"host":"example.com","port":null,"path":"/search","query":"flag","parameters":{"flag":null},"fragment":null} + Column parse_url('https://example.com/search?flag'): {"scheme":"https","username":null,"password":null,"host":"example.com","port":null,"path":"/search","query":"flag","parameters":{"flag":null},"fragment":null}