checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
---
2
2
SHA256:
3
- metadata.gz: bb787befdf7b8b3d1451c48cff1462e0f3ca40db0bf3c97b3a76ae112e509cdf
4
- data.tar.gz: f783cd9143f07e7c5c95cc5da9aeb3a80c20549befa29ca8bcc7814b31fde0ee
3
+ metadata.gz: 9535e605e2e5052e4c85e9ae0e2d5aa4562513b9dcba4c024cbfcc6d73e7f043
4
+ data.tar.gz: 6bc692e214299628fac9dcad0961f5666d08c79c635f10d055d4edd9d1af5b3a
5
5
SHA512:
6
- metadata.gz: 3052cb7600aeeea43ac4c2d7ccf610cfc552e8d88c5e309487d90a843a054ff57d94ae139effa2d46e343a95a095b3c18b42f9d447d4e2b8e644103fd0ad2b6f
7
- data.tar.gz: 9b48ea10a3f51661f9343d691271a0ac55a5c971834033401b54ecd6a001507e414bc5240d3d5ccaf4699e8b6d410d2142c697a269abb1fd7d855fea57adfa84
6
+ metadata.gz: 31a5e40658c2c6a56ace05eea2d1d231808e60f7f4a2b2ddbf079e2f0535de04cefaa5935ce17b19cb160ce7443d1215851520f8e7495b2d29f6c66a44027c7f
7
+ data.tar.gz: 47d28fa8bb3ab09db2da0dd5005cfe59109d327a59d5c3fcffc2d3ea3cda521c6b0ba149018c2dd0e005788de9b0b85330ec88c502bb91eab38f9392d8ee9ca0
data/.travis.yml CHANGED
@@ -2,8 +2,7 @@ sudo: false
2
2
language: ruby
3
3
4
4
rvm:
5
- - 2.2.10
6
- - 2.3.7
7
- - 2.4.4
8
- - 2.5.1
5
+ - 2.3
6
+ - 2.4
7
+ - 2.5.3
8
+ - 2.6.0
9
-
data/README.md CHANGED
@@ -95,8 +95,7 @@ You can use this parser without `multiline_start_regexp` when you know your data
95
95
96
96
## Configurations
97
97
98
- * See also: [TimeParameters Plugin Overview](https://docs.fluentd.org/v1.0/articles/timeparameters-plugin-overview)
98
+ * See also: [Config: Parse Section - Fluentd](https://docs.fluentd.org/configuration/parse-section)
99
- * See also: [Parser Plugin Overview](https://docs.fluentd.org/v1.0/articles/parser-plugin-overview)
100
99
101
100
* **time_format** (string) (optional): The format of the time field.
102
101
* **grok_pattern** (string) (optional): The pattern of grok. You cannot specify multiple grok pattern with this.
@@ -105,6 +104,17 @@ You can use this parser without `multiline_start_regexp` when you know your data
105
104
* **grok_name_key** (string) (optional): The key name to store grok section's name
106
105
* **multi_line_start_regexp** (string) (optional): The regexp to match beginning of multiline. This is only for "multiline_grok".
107
106
107
+ ### \<grok\> section (optional) (multiple)
108
+
109
+ * **name** (string) (optional): The name of this grok section
110
+ * **pattern** (string) (required): The pattern of grok
111
+ * **keep_time_key** (bool) (optional): If true, keep time field in the record.
112
+ * **time_key** (string) (optional): Specify time field for event time. If the event doesn't have this field, current time is used.
113
+ * Default value: `time`.
114
+ * **time_format** (string) (optional): Process value using specified format. This is available only when time_type is string
115
+ * **timezone** (string) (optional): Use specified timezone. one can parse/format the time value in the specified timezone.
116
+
117
+
108
118
## Examples
109
119
110
120
### Using grok\_failure\_key
@@ -184,6 +194,28 @@ This will add keys like following:
184
194
Add `grokfailure` key to the record if the record does not match any grok pattern.
185
195
See also test code for more details.
186
196
197
+ ## How to parse time value using specific timezone
198
+
199
+ ```aconf
200
+ <source>
201
+ @type tail
202
+ path /path/to/log
203
+ tag grokked_log
204
+ <parse>
205
+ @type grok
206
+ <grok>
207
+ name mylog-without-timezone
208
+ pattern %{DATESTAMP:time} %{GREEDYDATE:message}
209
+ timezone Asia/Tokyo
210
+ </grok>
211
+ </parse>
212
+ </source>
213
+ ```
214
+
215
+ This will parse the `time` value as "Asia/Tokyo" timezone.
216
+
217
+ See [Config: Parse Section - Fluentd](https://docs.fluentd.org/configuration/parse-section) for more details about timezone.
218
+
187
219
## How to write Grok patterns
188
220
189
221
Grok patterns look like `%{PATTERN_NAME:name}` where ":name" is optional. If "name" is provided, then it
@@ -271,7 +303,7 @@ Here is a sample config using the Grok parser with `in_tail` and the `types` par
271
303
272
304
If you want to use this plugin with Fluentd v0.12.x or earlier, you can use this plugin version v1.x.
273
305
274
- See also: [Plugin Management | Fluentd](http://docs.fluentd.org/articles/plugin-management#plugin-version-management)
306
+ See also: [Plugin Management | Fluentd](https://docs.fluentd.org/deployment/plugin-management)
275
307
276
308
## License
277
309
data/appveyor.yml CHANGED
@@ -1,10 +1,15 @@
1
1
version: '{build}'
2
+
3
+ # init:
4
+ # - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
5
+
2
6
install:
3
7
- SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
4
- - "%devkit%\\devkitvars.bat"
8
+ - IF %ridk%==0 "%devkit%\\devkitvars.bat"
5
9
- ruby --version
6
10
- gem --version
7
- - bundle install
11
+ - IF %ridk%==0 bundle install
12
+ - IF %ridk%==1 ridk.cmd exec bundle install
8
13
build: off
9
14
test_script:
10
15
- bundle exec rake test
@@ -12,20 +17,34 @@ test_script:
12
17
# https://www.appveyor.com/docs/installed-software/#ruby
13
18
environment:
14
19
matrix:
20
+ - ruby_version: "25-x64"
21
+ ridk: 1
22
+ - ruby_version: "25"
23
+ ridk: 1
24
+ - ruby_version: "24-x64"
25
+ ridk: 1
26
+ - ruby_version: "24"
27
+ ridk: 1
15
28
- ruby_version: "23-x64"
16
29
devkit: C:\Ruby23-x64\DevKit
30
+ ridk: 0
17
31
- ruby_version: "23"
18
32
devkit: C:\Ruby23\DevKit
33
+ ridk: 0
19
34
- ruby_version: "22-x64"
20
35
devkit: C:\Ruby23-x64\DevKit
36
+ ridk: 0
21
37
- ruby_version: "21-x64"
22
38
devkit: C:\Ruby23-x64\DevKit
39
+ ridk: 0
23
40
- ruby_version: "22"
24
41
devkit: C:\Ruby23\DevKit
25
42
WIN_RAPID: true
43
+ ridk: 0
26
44
- ruby_version: "21"
27
45
devkit: C:\Ruby23\DevKit
28
46
WIN_RAPID: true
47
+ ridk: 0
29
48
matrix:
30
49
allow_failures:
31
50
- ruby_version: "21"
data/fluent-plugin-grok-parser.gemspec CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
5
5
Gem::Specification.new do |spec|
6
6
spec.name = "fluent-plugin-grok-parser"
7
- spec.version = "2.3.0"
7
+ spec.version = "2.6.1"
8
8
spec.authors = ["kiyoto", "Kenji Okimoto"]
9
9
spec.email = ["kiyoto@treasure-data.com", "okimoto@clear-code.com"]
10
10
spec.summary = %q{Fluentd plugin to support Logstash-inspired Grok format for parsing logs}
@@ -19,5 +19,5 @@ Gem::Specification.new do |spec|
19
19
spec.add_development_dependency "bundler"
20
20
spec.add_development_dependency "rake"
21
21
spec.add_development_dependency "test-unit", ">=3.1.5"
22
- spec.add_runtime_dependency "fluentd", ">=0.14.6"
22
+ spec.add_runtime_dependency "fluentd", ">=0.14.6", "< 2"
23
23
end
data/lib/fluent/plugin/grok.rb CHANGED
@@ -13,7 +13,7 @@ module Fluent
13
13
(?<pattern>[A-z0-9]+)
14
14
(?::(?<subname>[@\[\]A-z0-9_:.-]+?)
15
15
(?::(?<type>(?:string|bool|integer|float|
16
- time(?::.+)?|
16
+ time(?::.+?)?|
17
17
array(?::.)?)))?)?
18
18
)
19
19
\}/x
@@ -27,6 +27,8 @@ module Fluent
27
27
@multiline_mode = false
28
28
@conf = conf
29
29
@plugin = plugin
30
+ @time_format = nil
31
+ @timezone = nil
30
32
if @plugin.respond_to?(:firstline?)
31
33
@multiline_mode = true
32
34
end
@@ -36,6 +38,12 @@ module Fluent
36
38
if @plugin.respond_to?(:keep_time_key)
37
39
@keep_time_key = @plugin.keep_time_key
38
40
end
41
+ if @plugin.respond_to?(:time_format)
42
+ @time_format = @plugin.time_format
43
+ end
44
+ if @plugin.respond_to?(:timezone)
45
+ @timezone = @plugin.timezone
46
+ end
39
47
end
40
48
41
49
def add_patterns_from_file(path)
@@ -48,10 +56,10 @@ module Fluent
48
56
49
57
def setup
50
58
if @plugin.grok_pattern
51
- @parsers[:grok_pattern] = expand_pattern_expression(@plugin.grok_pattern, @conf)
59
+ @parsers[:grok_pattern] = expand_pattern_expression_grok_pattern(@plugin.grok_pattern, @conf)
52
60
else
53
61
@plugin.grok_confs.each.with_index do |grok_conf, index|
54
- @parsers[grok_conf.name || index] = expand_pattern_expression(grok_conf.pattern, grok_conf)
62
+ @parsers[grok_conf.name || index] = expand_pattern_expression_grok_section(grok_conf)
55
63
end
56
64
end
57
65
@parsers.reject! do |key, parser|
@@ -64,7 +72,7 @@ module Fluent
64
72
65
73
private
66
74
67
- def expand_pattern_expression(grok_pattern, conf)
75
+ def expand_pattern_expression_grok_pattern(grok_pattern, conf)
68
76
regexp, types = expand_pattern(grok_pattern)
69
77
$log.info "Expanded the pattern #{grok_pattern} into #{regexp}"
70
78
_conf = conf.to_h
@@ -83,6 +91,40 @@ module Fluent
83
91
nil
84
92
end
85
93
94
+ def expand_pattern_expression_grok_section(conf)
95
+ regexp, types = expand_pattern(conf.pattern)
96
+ $log.info "Expanded the pattern #{conf.pattern} into #{regexp}"
97
+ _conf = conf.to_h
98
+ unless types.empty?
99
+ _conf["types"] = types.map{|subname,type| "#{subname}:#{type}" }.join(",")
100
+ end
101
+ if conf["multiline"] || @multiline_mode
102
+ _conf["multiline"] = conf["multiline"] || @multiline_mode
103
+ end
104
+ if conf["keep_time_key"] || @keep_time_key
105
+ _conf["keep_time_key"] = conf["keep_time_key"] || @keep_time_key
106
+ end
107
+ if conf["time_key"]
108
+ _conf["time_key"] = conf["time_key"]
109
+ end
110
+ if conf["time_format"] || @time_format
111
+ _conf["time_format"] = conf["time_format"] || @time_format
112
+ end
113
+ if conf["timezone"] || @timezone
114
+ _conf["timezone"] = conf["timezone"] || @timezone
115
+ end
116
+ _conf["expression"] = regexp
117
+ config = Fluent::Config::Element.new("parse", "", _conf, [])
118
+ parser = Fluent::Plugin::RegexpParser.new
119
+ parser.configure(config)
120
+ parser
121
+ rescue GrokPatternNotFoundError => e
122
+ raise e
123
+ rescue => e
124
+ $log.error(error: e)
125
+ nil
126
+ end
127
+
86
128
def expand_pattern(pattern)
87
129
# It's okay to modify in place. no need to expand it more than once.
88
130
type_map = {}
@@ -97,7 +139,7 @@ module Fluent
97
139
else
98
140
replacement_pattern = "(?:#{curr_pattern})"
99
141
end
100
- pattern.sub!(m[0]) do |s|
142
+ pattern = pattern.sub(m[0]) do |s|
101
143
replacement_pattern
102
144
end
103
145
end
data/lib/fluent/plugin/parser_grok.rb CHANGED
@@ -22,6 +22,14 @@ module Fluent
22
22
config_param :name, :string, default: nil
23
23
desc "The pattern of grok"
24
24
config_param :pattern, :string
25
+ desc "If true, keep time field in the record."
26
+ config_param :keep_time_key, :bool, default: false
27
+ desc "Specify time field for event time. If the event doesn't have this field, current time is used."
28
+ config_param :time_key, :string, default: "time"
29
+ desc "Process value using specified format. This is available only when time_type is string"
30
+ config_param :time_format, :string, default: nil
31
+ desc "Use specified timezone. one can parse/format the time value in the specified timezone."
32
+ config_param :timezone, :string, default: nil
25
33
end
26
34
27
35
def initialize
data/patterns/grok-patterns CHANGED
@@ -37,7 +37,7 @@ PATH (?:%{UNIXPATH}|%{WINPATH})
37
37
UNIXPATH (/([\w_%!$@:.,+~-]+|\\.)*)+
38
38
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
39
39
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
40
- URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
40
+ URIPROTO [A-Za-z]([A-Za-z0-9+\-.]+)+
41
41
URIHOST %{IPORHOST}(?::%{POSINT:port})?
42
42
# uripath comes loosely from RFC1738, but mostly from what Firefox
43
43
# doesn't turn into %XX
data/patterns/haproxy CHANGED
@@ -31,7 +31,7 @@ HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:captured_response_headers}
31
31
# HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:response_header_content_type}\|%{DATA:response_header_content_encoding}\|%{DATA:response_header_cache_control}\|%{DATA:response_header_last_modified}
32
32
33
33
# parse a haproxy 'httplog' line
34
- HAPROXYHTTPBASE %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?"
34
+ HAPROXYHTTPBASE %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(<BADREQ>|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?"?
35
35
36
36
HAPROXYHTTP (?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{HAPROXYHTTPBASE}
37
37
data/patterns/httpd CHANGED
@@ -7,9 +7,9 @@ HTTPD_COMBINEDLOG %{HTTPD_COMMONLOG} %{QS:referrer} %{QS:agent}
7
7
8
8
# Error logs
9
9
HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:message}
10
- HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_message}:)?( \[client %{IPORHOST:clientip}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
10
+ HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}(:tid %{NUMBER:tid})?\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_message}:)?( \[client %{IPORHOST:clientip}:%{POSINT:clientport}\])?( %{DATA:errorcode}:)? %{GREEDYDATA:message}
11
11
HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
12
12
13
13
# Deprecated
14
14
COMMONAPACHELOG %{HTTPD_COMMONLOG}
15
- COMBINEDAPACHELOG %{HTTPD_COMBINEDLOG}
15
+ COMBINEDAPACHELOG %{HTTPD_COMBINEDLOG}
data/patterns/java CHANGED
@@ -7,8 +7,6 @@ JAVAMETHOD (?:(<(?:cl)?init>)|[a-zA-Z$_][a-zA-Z$_0-9]*)
7
7
JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\)
8
8
# Java Logs
9
9
JAVATHREAD (?:[A-Z]{2}-Processor[\d]+)
10
- JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+
11
- JAVAFILE (?:[A-Za-z0-9_.-]+)
12
10
JAVALOGMESSAGE (.*)
13
11
# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM
14
12
CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM)
data/patterns/linux-syslog CHANGED
@@ -11,6 +11,6 @@ SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message}
11
11
# IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424)
12
12
SYSLOG5424PRI <%{NONNEGINT:syslog5424_pri}>
13
13
SYSLOG5424SD \[%{DATA}\]+
14
- SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)
14
+ SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{IPORHOST:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|)
15
15
16
16
SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg}
data/test/test_grok_parser.rb CHANGED
@@ -158,37 +158,49 @@ class GrokParserTest < ::Test::Unit::TestCase
158
158
end
159
159
end
160
160
161
- test "no grok patterns" do
162
- assert_raise Fluent::ConfigError do
163
- create_driver('')
161
+ sub_test_case "configure" do
162
+ test "no grok patterns" do
163
+ assert_raise Fluent::ConfigError do
164
+ create_driver('')
165
+ end
164
166
end
165
- end
166
167
167
- test "invalid config value type" do
168
- assert_raise Fluent::ConfigError do
169
- create_driver(%[
168
+ test "invalid config value type" do
169
+ assert_raise Fluent::ConfigError do
170
+ create_driver(%[
171
+ <grok>
172
+ pattern %{PATH:path:foo}
173
+ </grok>
174
+ ])
175
+ end
176
+ end
177
+
178
+ test "invalid config value type and normal grok pattern" do
179
+ d = create_driver(%[
170
180
<grok>
171
181
pattern %{PATH:path:foo}
172
182
</grok>
183
+ <grok>
184
+ pattern %{IP:ip_address}
185
+ </grok>
173
186
])
187
+ assert_equal(1, d.instance.instance_variable_get(:@grok).parsers.size)
188
+ logs = $log.instance_variable_get(:@logger).instance_variable_get(:@logdev).logs
189
+ error_logs = logs.grep(/error_class/)
190
+ assert_equal(1, error_logs.size)
191
+ error_message = error_logs.first[/error="(.+)"/, 1]
192
+ assert_equal("unknown value conversion for key:'path', type:'foo'", error_message)
174
193
end
175
- end
176
194
177
- test "invalid config value type and normal grok pattern" do
178
- d = create_driver(%[
179
- <grok>
180
- pattern %{PATH:path:foo}
181
- </grok>
182
- <grok>
183
- pattern %{IP:ip_address}
184
- </grok>
185
- ])
195
+ test "keep original configuration" do
196
+ config = %[
197
+ <grok>
198
+ pattern %{INT:user_id:integer} paid %{NUMBER:paid_amount:float}
199
+ </grok>
200
+ ]
201
+ d = create_driver(config)
202
+ assert_equal("%{INT:user_id:integer} paid %{NUMBER:paid_amount:float}", d.instance.config.elements("grok").first["pattern"])
203
+ end
186
- assert_equal(1, d.instance.instance_variable_get(:@grok).parsers.size)
187
- logs = $log.instance_variable_get(:@logger).instance_variable_get(:@logdev).logs
188
- error_logs = logs.grep(/error_class/)
189
- assert_equal(1, error_logs.size)
190
- error_message = error_logs.first[/error="(.+)"/, 1]
191
- assert_equal("unknown value conversion for key:'path', type:'foo'", error_message)
192
204
end
193
205
194
206
sub_test_case "grok_name_key" do
@@ -321,6 +333,89 @@ class GrokParserTest < ::Test::Unit::TestCase
321
333
end
322
334
end
323
335
336
+ sub_test_case "grok section" do
337
+ test "complex pattern" do
338
+ d = create_driver(%[
339
+ <grok>
340
+ pattern %{COMBINEDAPACHELOG}
341
+ time_key timestamp
342
+ time_format %d/%b/%Y:%H:%M:%S %z
343
+ </grok>
344
+ ])
345
+ expected_record = {
346
+ "clientip" => "127.0.0.1",
347
+ "ident" => "192.168.0.1",
348
+ "auth" => "-",
349
+ "verb" => "GET",
350
+ "request" => "/",
351
+ "httpversion" => "1.1",
352
+ "response" => "200",
353
+ "bytes" => "777",
354
+ "referrer" => "\"-\"",
355
+ "agent" => "\"Opera/12.0\""
356
+ }
357
+ d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') do |time, record|
358
+ assert_equal(expected_record, record)
359
+ assert_equal(event_time("28/Feb/2013:12:00:00 +0900", format: "%d/%b/%Y:%H:%M:%S %z"), time)
360
+ end
361
+ end
362
+
363
+ test "leading time type with following other type" do
364
+ d = create_driver(%[
365
+ <grok>
366
+ pattern \\[%{HTTPDATE:log_timestamp:time:%d/%b/%Y:%H:%M:%S %z}\\] %{GREEDYDATA:message}
367
+ </grok>
368
+ ])
369
+ expected_record = {
370
+ "log_timestamp" => event_time("03/Feb/2019:06:47:21 +0530", format: "%d/%b/%Y:%H:%M:%S %z"),
371
+ "message" => "Python-urllib/2.7"
372
+ }
373
+ d.instance.parse('[03/Feb/2019:06:47:21 +0530] Python-urllib/2.7') do |time, record|
374
+ assert_equal(expected_record, record)
375
+ end
376
+ end
377
+
378
+ test "timezone" do
379
+ d = create_driver(%[
380
+ <grok>
381
+ pattern %{TIMESTAMP_ISO8601:time} %{GREEDYDATA:message}
382
+ time_key time
383
+ time_format %Y-%m-%d %H:%M:%S
384
+ timezone Europe/Berlin
385
+ </grok>
386
+ ])
387
+ d.instance.parse("2019-02-01 12:34:56 This is test") do |time, record|
388
+ assert_equal(event_time("2019-02-01 12:34:56 +0100"), time)
389
+ assert_equal({ "message" => "This is test" }, record)
390
+ end
391
+ end
392
+
393
+ test "multiple timezone" do
394
+ d = create_driver(%[
395
+ <grok>
396
+ pattern %{TIMESTAMP_ISO8601:time} 1 %{GREEDYDATA:message}
397
+ time_key time
398
+ time_format %Y-%m-%d %H:%M:%S
399
+ timezone Europe/Berlin
400
+ </grok>
401
+ <grok>
402
+ pattern %{TIMESTAMP_ISO8601:time} 2 %{GREEDYDATA:message}
403
+ time_key time
404
+ time_format %Y-%m-%d %H:%M:%S
405
+ timezone Asia/Aden
406
+ </grok>
407
+ ])
408
+ d.instance.parse("2019-02-01 12:34:56 1 This is test") do |time, record|
409
+ assert_equal(event_time("2019-02-01 12:34:56 +0100"), time)
410
+ assert_equal({ "message" => "This is test" }, record)
411
+ end
412
+ d.instance.parse("2019-02-01 12:34:56 2 This is test") do |time, record|
413
+ assert_equal(event_time("2019-02-01 12:34:56 +0300"), time)
414
+ assert_equal({ "message" => "This is test" }, record)
415
+ end
416
+ end
417
+ end
418
+
324
419
private
325
420
326
421
def create_driver(conf)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
--- !ruby/object:Gem::Specification
2
2
name: fluent-plugin-grok-parser
3
3
version: !ruby/object:Gem::Version
4
- version: 2.3.0
4
+ version: 2.6.1
5
5
platform: ruby
6
6
authors:
7
7
- kiyoto
@@ -9,7 +9,7 @@ authors:
9
9
autorequire:
10
10
bindir: bin
11
11
cert_chain: []
12
- date: 2018-09-10 00:00:00.000000000 Z
12
+ date: 2019-09-11 00:00:00.000000000 Z
13
13
dependencies:
14
14
- !ruby/object:Gem::Dependency
15
15
name: bundler
@@ -60,6 +60,9 @@ dependencies:
60
60
- - ">="
61
61
- !ruby/object:Gem::Version
62
62
version: 0.14.6
63
+ - - "<"
64
+ - !ruby/object:Gem::Version
65
+ version: '2'
63
66
type: :runtime
64
67
prerelease: false
65
68
version_requirements: !ruby/object:Gem::Requirement
@@ -67,6 +70,9 @@ dependencies:
67
70
- - ">="
68
71
- !ruby/object:Gem::Version
69
72
version: 0.14.6
73
+ - - "<"
74
+ - !ruby/object:Gem::Version
75
+ version: '2'
70
76
description:
71
77
email:
72
78
- kiyoto@treasure-data.com
@@ -132,8 +138,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
132
138
- !ruby/object:Gem::Version
133
139
version: '0'
134
140
requirements: []
135
- rubyforge_project:
141
+ rubygems_version: 3.0.3
136
- rubygems_version: 2.7.6
137
142
signing_key:
138
143
specification_version: 4
139
144
summary: Fluentd plugin to support Logstash-inspired Grok format for parsing logs