[log_format] use DOTALL for format patterns

This commit is contained in:
Timothy Stack 2021-06-03 15:59:01 -07:00
parent 27bc9cf117
commit 516d13d772
12 changed files with 24 additions and 23 deletions

View File

@ -5,10 +5,10 @@
"description": "A generic format for logs, like cron, that have a date at the start of a block.", "description": "A generic format for logs, like cron, that have a date at the start of a block.",
"regex": { "regex": {
"std": { "std": {
"pattern": "^(?<timestamp>\\S{3,8} \\w{3}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\w+ \\d{4})(?<body>(?:.|\\n)*)$" "pattern": "^(?<timestamp>\\S{3,8} \\w{3}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\w+ \\d{4})(?<body>.*)$"
}, },
"sq-brackets": { "sq-brackets": {
"pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?)Z?\\](?<body>(?:.|\\n)*)$" "pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?)Z?\\](?<body>.*)$"
} }
}, },
"sample": [ "sample": [

View File

@ -5,7 +5,7 @@
"description": "The log format for the engine.log files from RHEV/oVirt", "description": "The log format for the engine.log files from RHEV/oVirt",
"regex": { "regex": {
"std": { "std": {
"pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}+)\\s+(?<level>\\w+)\\s+\\[(?<logger>[^\\]]+)\\]\\s+\\((?<tid>[^\\)]+)\\)\\s+(?<body>(?:-|\\n)*)" "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}+)\\s+(?<level>\\w+)\\s+\\[(?<logger>[^\\]]+)\\]\\s+\\((?<tid>[^\\)]+)\\)\\s+(?<body>.*)"
} }
}, },
"opid-field": "tid", "opid-field": "tid",

View File

@ -5,7 +5,7 @@
"description": "Log for the fsck_hfs tool on Mac OS X.", "description": "Log for the fsck_hfs tool on Mac OS X.",
"regex": { "regex": {
"std": { "std": {
"pattern": "^(?<device>[^:]+): fsck_hfs (?:run|started) at (?<timestamp>\\S{3,8} \\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\d{4})(?<body>(?:.|\\n)*)" "pattern": "^(?<device>[^:]+): fsck_hfs (?:run|started) at (?<timestamp>\\S{3,8} \\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\d{4})(?<body>.*)"
} }
}, },
"value": { "value": {

View File

@ -6,7 +6,7 @@
"url": "https://code.google.com/p/google-glog/", "url": "https://code.google.com/p/google-glog/",
"regex": { "regex": {
"std": { "std": {
"pattern": "^(?<level>[IWECF])(?<timestamp>\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6}) +(?<thread>\\d+) (?<src_file>[^:]+):(?<src_line>\\d+)\\] (?<body>(?:.|\\n)*)" "pattern": "^(?<level>[IWECF])(?<timestamp>\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6}) +(?<thread>\\d+) (?<src_file>[^:]+):(?<src_line>\\d+)\\] (?<body>.*)"
} }
}, },
"level-field": "level", "level-field": "level",

View File

@ -6,13 +6,13 @@
"url": "", "url": "",
"regex": { "regex": {
"jvm": { "jvm": {
"pattern": "^(?<alert_level>\\w+)\\s+\\|\\s+jvm (?<jvm_no>\\d+)\\s+\\|\\s(?<timestamp>\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2})\\s+\\| (?<timestamp_f>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) \\[(?<function>\\w+-\\d+)\\]\\s+(?<debug_level>\\w+)\\s+(?<class>[\\w.]+)\\s+-\\s+(?<body>\\S.*)" "pattern": "^(?<alert_level>\\w+)\\s+\\|\\s+jvm (?<jvm_no>\\d+)\\s+\\|\\s(?<timestamp>\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2})\\s+\\| (?<timestamp_f>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) \\[(?<function>\\w+-\\d+)\\]\\s+(?<debug_level>\\w+)\\s+(?<class>[\\w.]+)\\s+-\\s+(?<body>.*)"
}, },
"dump": { "dump": {
"pattern": "^(?<alert_level>\\w+)\\s+\\|\\s+jvm (?<jvm_no>\\d+)\\s+\\|\\s(?<timestamp>\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2})\\s+\\| JVMDUMP\\w+\\s(?<body>\\S.*)$" "pattern": "^(?<alert_level>\\w+)\\s+\\|\\s+jvm (?<jvm_no>\\d+)\\s+\\|\\s(?<timestamp>\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2})\\s+\\| JVMDUMP\\w+\\s(?<body>.*)$"
}, },
"tasko": { "tasko": {
"pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) \\[(?<function>\\w+-\\d+)\\]\\s+(?<alert_level>\\w+)\\s+(?<class>[\\w.]+)\\s+-\\s+(?<body>\\S.*)$" "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) \\[(?<function>\\w+-\\d+)\\]\\s+(?<alert_level>\\w+)\\s+(?<class>[\\w.]+)\\s+-\\s+(?<body>.*)$"
} }
}, },
"level-field": "alert_level", "level-field": "alert_level",

View File

@ -6,7 +6,7 @@
"url": "http://theforeman.org/", "url": "http://theforeman.org/",
"regex": { "regex": {
"log": { "log": {
"pattern": "^\\[\\s?(?<alert_level>\\w+)\\s(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})\\s(?<module>\\w+)\\]\\s+(?<message>\\S.*)$" "pattern": "^\\[\\s?(?<alert_level>\\w+)\\s(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})\\s(?<module>\\w+)\\]\\s+(?<message>.*)$"
} }
}, },
"level-field": "alert_level", "level-field": "alert_level",

View File

@ -7,7 +7,7 @@
"regex": { "regex": {
"std": { "std": {
"module-format": true, "module-format": true,
"pattern": "^(?<login>\\S+)\\s*: (?:(?<error_msg>[^;]+);)?\\s*TTY=(?<tty>[^;]+)\\s+;\\s*PWD=(?<pwd>[^;]+)\\s+;\\s*USER=(?<user>[^;]+)\\s+;\\s*COMMAND=(?<command>(\\n|.)*)$" "pattern": "^(?<login>\\S+)\\s*: (?:(?<error_msg>[^;]+);)?\\s*TTY=(?<tty>[^;]+)\\s+;\\s*PWD=(?<pwd>[^;]+)\\s+;\\s*USER=(?<user>[^;]+)\\s+;\\s*COMMAND=(?<command>.*)$"
} }
}, },
"level-field": "error_msg", "level-field": "error_msg",

View File

@ -6,10 +6,10 @@
"url": "http://en.wikipedia.org/wiki/Syslog", "url": "http://en.wikipedia.org/wiki/Syslog",
"regex": { "regex": {
"std": { "std": {
"pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?Z))(?: (?<log_hostname>[a-zA-Z0-9:][^ ]+[a-zA-Z0-9]))?(?:(?: (?<log_procname>(?:[^\\[:]+|[^:]+))(?:\\[(?<log_pid>\\d+)\\])?:\\s*(?<body>(?:.|\\n)*))$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))" "pattern": "^(?<timestamp>(?:\\S{3,8}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?Z))(?: (?<log_hostname>[a-zA-Z0-9:][^ ]+[a-zA-Z0-9]))?(?:(?: (?<log_procname>(?:[^\\[:]+|[^:]+))(?:\\[(?<log_pid>\\d+)\\])?:\\s*(?<body>.*))$|:?(?:(?: ---)? last message repeated \\d+ times?(?: ---)?))"
}, },
"rfc5424": { "rfc5424": {
"pattern": "^<(?<log_pri>\\d+)>(?<syslog_version>\\d+) (?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{6})?(?:[^ ]+)?) (?<log_hostname>[^ ]+|-) (?<log_procname>[^ ]+|-) (?<log_pid>[^ ]+|-) (?<log_msgid>[^ ]+|-) (?<log_struct>\\[(?:[^\\]\"]|\"(?:\\.|[^\"])+\")*\\]|-|)\\s+(?<body>(?:.|\\n)*)" "pattern": "^<(?<log_pri>\\d+)>(?<syslog_version>\\d+) (?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{6})?(?:[^ ]+)?) (?<log_hostname>[^ ]+|-) (?<log_procname>[^ ]+|-) (?<log_pid>[^ ]+|-) (?<log_msgid>[^ ]+|-) (?<log_struct>\\[(?:[^\\]\"]|\"(?:\\.|[^\"])+\")*\\]|-|)\\s+(?<body>.*)"
} }
}, },
"level-field": "body", "level-field": "body",

View File

@ -6,10 +6,10 @@
"url": "http://www.ovirt.org/develop/developer-guide/vdsm/log-files/", "url": "http://www.ovirt.org/develop/developer-guide/vdsm/log-files/",
"regex": { "regex": {
"v4.1": { "v4.1": {
"pattern": "(?s)^(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}(?:\\+\\d{4})?) (?<level>\\w+)\\s+\\((?<thread>.+?)\\) \\[(?<logger>.+?)\\]\\s+?(?<body>(?:.|[\\r\\n])*?)\\s+?\\((?<src_file>\\w+?):(?<src_line>\\d+)\\)(\\n(?<traceback>Traceback.*?)(?=\\n(?P=timestamp)|$))?" "pattern": "(?s)^(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}(?:\\+\\d{4})?) (?<level>\\w+)\\s+\\((?<thread>.+?)\\) \\[(?<logger>.+?)\\]\\s+?(?<body>.*?)\\s+?\\((?<src_file>\\w+?):(?<src_line>\\d+)\\)(\\n(?<traceback>Traceback.*?)(?=\\n(?P=timestamp)|$))?"
}, },
"v4": { "v4": {
"pattern": "^(?<tid>[^:]+)::(?<level>[^:]+)::(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}(?:,\\d{3})?)::(?<module>[^:]+)::(?<src_line>[^:]+)::(?<logger>[^:]+)::\\((?<func>[^\\)]+)\\)(?<body>(?:.|\\n)*)" "pattern": "^(?<tid>[^:]+)::(?<level>[^:]+)::(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}(?:,\\d{3})?)::(?<module>[^:]+)::(?<src_line>[^:]+)::(?<logger>[^:]+)::\\((?<func>[^\\)]+)\\)(?<body>.*)"
} }
}, },
"level-field": "level", "level-field": "level",

View File

@ -6,34 +6,34 @@
"url": "http://kb.vmware.com/kb/2004201", "url": "http://kb.vmware.com/kb/2004201",
"regex": { "regex": {
"6.0+": { "6.0+": {
"pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?<level>\\w+) (?<prc>[\\w\\-]+)\\[(?<tid>\\w+)\\] \\[(?<src>[^ \\]]+)\\s*(?: sub=(?<sub>.+?))?(?: opI(?:D|d)=(?<opid>[^ \\]]+))?(?: user=(?<user>[^ \\]]+))?\\]\\s*(?<body>.*)(?:\\n(?:.|\\n)*)?$" "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?<level>\\w+) (?<prc>[\\w\\-]+)\\[(?<tid>\\w+)\\] \\[(?<src>[^ \\]]+)\\s*(?: sub=(?<sub>.+?))?(?: opI(?:D|d)=(?<opid>[^ \\]]+))?(?: user=(?<user>[^ \\]]+))?\\]\\s*(?<body>.*)(?:\\n.*)?$"
}, },
"section": { "section": {
"pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?:- last log rotation time, \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2}))?\\s*(ESX KMX Agent started.|Section for VMware VirtualCenter,.*|- time the service was last started(?: \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z)?, Section for (?:[^,]+), pid=(?<tid>\\w+).*)" "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) (?:- last log rotation time, \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2}))?\\s*(ESX KMX Agent started.|Section for VMware VirtualCenter,.*|- time the service was last started(?: \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z)?, Section for (?:[^,]+), pid=(?<tid>\\w+).*)"
}, },
"5.0+": { "5.0+": {
"pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) \\[(?<tid>\\w+) (?<level>\\w+) '(?<comp>[^']+)'(?: opID=(?<opid>[^ \\]]+))?(?: user=(?<user>[^ \\]]+))?\\](?<body>.*)(?:\\n(?:.|\\n)*)?$" "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) \\[(?<tid>\\w+) (?<level>\\w+) '(?<comp>[^']+)'(?: opID=(?<opid>[^ \\]]+))?(?: user=(?<user>[^ \\]]+))?\\](?<body>.*)(?:\\n.*)?$"
}, },
"pre-5.0": { "pre-5.0": {
"pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (?<tid>\\w+) (?<level>\\w+) '(?<comp>[^']+)'(?: opID=(?<opid>[^ \\]]+))?(?: user=(?<user>[^ \\]]+))?\\](?<body>.*)(?:\\n(?:.|\\n)*)?$" "pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}) (?<tid>\\w+) (?<level>\\w+) '(?<comp>[^']+)'(?: opID=(?<opid>[^ \\]]+))?(?: user=(?<user>[^ \\]]+))?\\](?<body>.*)(?:\\n.*)?$"
}, },
"ls-log": { "ls-log": {
"pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) (?<tid>[\\w\\-]+)\\s+(?<level>\\w+)\\s+(?<comp>[^\\]]+)\\]\\s+(?<body>(?:.|\\n)*)" "pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) (?<tid>[\\w\\-]+)\\s+(?<level>\\w+)\\s+(?<comp>[^\\]]+)\\]\\s+(?<body>.*)"
}, },
"hdr-ftr": { "hdr-ftr": {
"pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}(T| )\\d{2}:\\d{2}:\\d{2}(?:.|,)\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) \\[(?<prc>[^\\[]+)\\[(?<tid>\\w+)\\]:\\s+(?<body>.*)\\]$" "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}(T| )\\d{2}:\\d{2}:\\d{2}(?:.|,)\\d{3}(?:Z|[-+]\\d{2}:\\d{2})) \\[(?<prc>[^\\[]+)\\[(?<tid>\\w+)\\]:\\s+(?<body>.*)\\]$"
}, },
"pylog": { "pylog": {
"pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?(?:Z|[-+]\\d{2}:\\d{2})) (?<prc>[^:]+):\\s+(?<tid>\\w+):\\s+(?<comp>[^:]+):(?<line>\\d+)?\\s+(?<level>\\w+):?\\s+(?<body>.*)(?:\\n(?:.|\\n)*)?$" "pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3})?(?:Z|[-+]\\d{2}:\\d{2})) (?<prc>[^:]+):\\s+(?<tid>\\w+):\\s+(?<comp>[^:]+):(?<line>\\d+)?\\s+(?<level>\\w+):?\\s+(?<body>.*)(?:\\n.*)?$"
}, },
"vum-log4cpp": { "vum-log4cpp": {
"pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}:\\d{3}) '(?<category>[^']*)' (?<tid>\\d+) (?<level>[a-zA-Z]+)\\]\\s+(?>\\[(?<file>\\S+), (?<line>\\d+)\\])? (?<body>.*$)" "pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}:\\d{3}) '(?<category>[^']*)' (?<tid>\\d+) (?<level>[a-zA-Z]+)\\]\\s+(?>\\[(?<file>\\S+), (?<line>\\d+)\\])? (?<body>.*$)"
}, },
"pylog2": { "pylog2": {
"pattern": "^(?<prc>[^:]+):(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3})\\[(?<tid>\\w+)\\](?<file>[^:]+):(?<line>\\d+) \\[(?<level>[a-zA-Z]+)\\]\\s+(?<body>(?:.|\\n)*)$" "pattern": "^(?<prc>[^:]+):(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3})\\[(?<tid>\\w+)\\](?<file>[^:]+):(?<line>\\d+) \\[(?<level>[a-zA-Z]+)\\]\\s+(?<body>.*)$"
}, },
"pylog3": { "pylog3": {
"pattern": "^(?<prc>[^:]+): (?<tid>\\d+): (?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) (?<file>[^:]+):(?<line>\\d+) (?<level>[a-zA-Z]+)\\s+(?<body>(?:.|\\n)*)$" "pattern": "^(?<prc>[^:]+): (?<tid>\\d+): (?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) (?<file>[^:]+):(?<line>\\d+) (?<level>[a-zA-Z]+)\\s+(?<body>.*)$"
} }
}, },
"level-field": "level", "level-field": "level",

View File

@ -365,8 +365,9 @@ void grep_proc<LineType>::check_poll_set(const std::vector<struct pollfd> &pollf
this->gp_pipe_range = li.li_file_range; this->gp_pipe_range = li.li_file_range;
this->gp_line_buffer.read_range(li.li_file_range).then([this](auto sbr) { this->gp_line_buffer.read_range(li.li_file_range).then([this](auto sbr) {
char buf[1024]; auto_mem<char> buf;
buf = (char *) malloc(sbr.length() + 1);
sbr.rtrim(is_line_ending); sbr.rtrim(is_line_ending);
memcpy(buf, sbr.get_data(), sbr.length()); memcpy(buf, sbr.get_data(), sbr.length());
buf[sbr.length()] = '\0'; buf[sbr.length()] = '\0';

View File

@ -1548,7 +1548,7 @@ void external_log_format::build(std::vector<std::string> &errors) {
} }
try { try {
pat.p_pcre = std::make_unique<pcrepp>(pat.p_string); pat.p_pcre = std::make_unique<pcrepp>(pat.p_string, PCRE_DOTALL);
} }
catch (const pcrepp::error &e) { catch (const pcrepp::error &e) {
errors.push_back("error:" + errors.push_back("error:" +