[loader] make sure values defined in formats have a corresponding capture in at least one pattern

NEWS

@@ -82,6 +82,8 @@ lnav v0.11.0:
        scrolling in a view.
 
      Breaking Changes:
+     * Formats definitions are now checked to ensure that values have a
+       corresponding capture in at least one pattern.
      * Added a 'language' column to the lnav_view_filters table that
        specifies the language of the 'pattern' column, either 'regex'
        or 'sql'.

src/formats/alb_log.json

@@ -19,7 +19,7 @@
                 "kind": "string",
                 "identifier": true
             },
-            "alb": {
+            "elb": {
                 "kind": "string",
                 "identifier": true
             },

src/formats/engine_log.json

@@ -17,10 +17,6 @@
             "logger": {
                 "kind": "string",
                 "identifier": true
-            },
-            "thread": {
-                "kind": "string",
-                "identifier": true
             }
         },
         "level-field": "level",

src/formats/strace_log.json

@@ -20,9 +20,6 @@
                 "identifier": true,
                 "rewriter": ":pipe-line-to explain-syscall.sh ${syscall}"
             },
-            "args": {
-                "kind": "string"
-            },
             "rc": {
                 "kind": "integer",
                 "foreign-key": true

src/log_format.cc

@@ -1769,7 +1769,7 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
             this->elf_has_module_format = true;
         }
 
-        for (pcre_named_capture::iterator name_iter = pat.p_pcre->named_begin();
+        for (auto name_iter = pat.p_pcre->named_begin();
              name_iter != pat.p_pcre->named_end();
              ++name_iter)
         {
@@ -1903,6 +1903,31 @@ external_log_format::build(std::vector<lnav::console::user_message>& errors)
             vd->vd_meta.lvm_kind = value_kind_t::VALUE_TEXT;
         }
 
+        if (this->elf_type == elf_type_t::ELF_TYPE_TEXT) {
+            bool found_in_pattern = false;
+            for (const auto& pat : this->elf_patterns) {
+                auto cap_index = pat.second->p_pcre->name_index(
+                    vd->vd_meta.lvm_name.get());
+                if (cap_index >= 0) {
+                    found_in_pattern = true;
+                    break;
+                }
+            }
+            if (!found_in_pattern) {
+                errors.emplace_back(
+                    lnav::console::user_message::error(
+                        attr_line_t("invalid value ")
+                            .append_quoted(lnav::roles::symbol(
+                                fmt::format(FMT_STRING("/{}/value/{}"),
+                                            this->elf_name,
+                                            vd->vd_meta.lvm_name.get()))))
+                        .with_reason(
+                            attr_line_t("no patterns have a capture named ")
+                                .append_quoted(vd->vd_meta.lvm_name.get()))
+                        .with_snippets(this->get_snippets()));
+            }
+        }
+
         for (act_iter = vd->vd_action_list.begin();
              act_iter != vd->vd_action_list.end();
              ++act_iter)

src/log_format_loader.cc

@@ -916,19 +916,19 @@ read_id(yajlpp_parse_context* ypc, const unsigned char* str, size_t len)
 }
 
 struct json_path_container root_format_handler
-    = json_path_container{json_path_handler("$schema", read_id)
-                              .with_synopsis(
-                                  "The URI of the schema for this file")
-                              .with_description(
-                                  "Specifies the type of this file"),
+        = json_path_container{
+                json_path_handler("$schema", read_id)
+                        .with_synopsis("The URI of the schema for this file")
+                        .with_description("Specifies the type of this file"),
 
-                          yajlpp::pattern_property_handler(
-                              "(?<format_name>\\w+)")
-                              .with_description(
-                                  "The definition of a log file format.")
-                              .with_obj_provider(ensure_format)
-                              .with_children(format_handlers)}
-          .with_schema_id(DEFAULT_FORMAT_SCHEMA);
+                yajlpp::pattern_property_handler(
+                        "(?<format_name>\\w+)")
+                        .with_description(
+                                "The definition of a log file format.")
+                        .with_obj_provider(ensure_format)
+                        .with_children(format_handlers),
+        }
+                .with_schema_id(DEFAULT_FORMAT_SCHEMA);
 
 static void
 write_sample_file()

test/bad-config/formats/invalid-properties/format.json

@@ -13,6 +13,11 @@
                 "line": "1428634687123: 1234 abc"
             }
         ],
+        "value": {
+            "non-existent": {
+                "kind": "string"
+            }
+        },
         "highlights": {
             "hl1": {
                 "color": "not a color",

test/expected/test_format_loader.sh_3f1d6f35e8a9ae4fd3e91ffaa82a037b5a847ab7.err

@@ -1,6 +1,6 @@
 ✘ error: “abc(def” is not a valid regular expression for property “/invalid_props_log/search-table/bad_table_regex/pattern”
  reason: missing )
- --> {test_dir}/bad-config/formats/invalid-properties/format.json:24
+ --> {test_dir}/bad-config/formats/invalid-properties/format.json:29
  |                 "pattern": "abc(def"    
  --> /invalid_props_log/search-table/bad_table_regex/pattern
  | abc(def                                 
@@ -98,6 +98,9 @@
          std        = “^(?<timestamp>\d+): (?<pid>\w+) (?<body>.*)$”
          with-level = “^(?<timestamp>\d+)\| (?<level>\w+) (?<body>\w+)$”
          
+✘ error: invalid value “/invalid_props_log/value/non-existent”
+ reason: no patterns have a capture named “non-existent”
+ --> {test_dir}/bad-config/formats/invalid-properties/format.json:4
 ✘ error: invalid value for property “/invalid_props_log/timestamp-field”
  reason: “ts” was not found in the pattern at /invalid_props_log/regex/std
  --> {test_dir}/bad-config/formats/invalid-properties/format.json:4
@@ -105,10 +108,10 @@
            body, pid, timestamp
 ✘ error: “not a color” is not a valid color value for property “/invalid_props_log/highlights/hl1/color”
  reason: Unknown color: 'not a color'.  See https://jonasjacek.github.io/colors/ for a list of supported color names
- --> {test_dir}/bad-config/formats/invalid-properties/format.json:18
+ --> {test_dir}/bad-config/formats/invalid-properties/format.json:23
 ✘ error: “also not a color” is not a valid color value for property “/invalid_props_log/highlights/hl1/background-color”
  reason: Unknown color: 'also not a color'.  See https://jonasjacek.github.io/colors/ for a list of supported color names
- --> {test_dir}/bad-config/formats/invalid-properties/format.json:19
+ --> {test_dir}/bad-config/formats/invalid-properties/format.json:24
 ✘ error: “no_regexes_log” is not a valid log format
  reason: no regexes specified
  --> {test_dir}/bad-config/formats/no-regexes/format.json:4

test/formats/xmlmsg/format.json

@@ -26,16 +26,6 @@
                 "kind": "integer",
                 "description": "Line number – in the module – where log entry was emitted"
             },
-            "head": {
-                "kind": "string",
-                "description": "<head>",
-                "hidden": true
-            },
-            "source": {
-                "kind": "string",
-                "description": "request <source>",
-                "hidden": true
-            },
             "msg_data": {
                 "kind": "xml",
                 "rewriter": ";SELECT node_path FROM xpath('//*', :msg_data)"