From 4deb6238a376e76296d8ef354524eaa93d685632 Mon Sep 17 00:00:00 2001
From: Daniel Nelson <daniel@wavesofdawn.com>
Date: Thu, 19 Oct 2017 16:36:32 -0700
Subject: [PATCH] Add support for decimal timestamps to ts-epoch modifier
(#3358)
---
plugins/inputs/logparser/README.md | 15 ++++-
plugins/inputs/logparser/grok/grok.go | 26 ++++++--
plugins/inputs/logparser/grok/grok_test.go | 71 ++++++++++++++++++++++
3 files changed, 107 insertions(+), 5 deletions(-)
diff --git a/plugins/inputs/logparser/README.md b/plugins/inputs/logparser/README.md
index f823cfe6..2febb819 100644
--- a/plugins/inputs/logparser/README.md
+++ b/plugins/inputs/logparser/README.md
@@ -100,7 +100,7 @@ current time.
- ts-rfc3339 ("2006-01-02T15:04:05Z07:00")
- ts-rfc3339nano ("2006-01-02T15:04:05.999999999Z07:00")
- ts-httpd ("02/Jan/2006:15:04:05 -0700")
- - ts-epoch (seconds since unix epoch)
+ - ts-epoch (seconds since unix epoch, may contain decimal)
- ts-epochnano (nanoseconds since unix epoch)
- ts-"CUSTOM"
@@ -130,6 +130,19 @@ This example input and config parses a file using a custom timestamp conversion:
patterns = ['%{TIMESTAMP_ISO8601:timestamp:ts-"2006-01-02 15:04:05"} value=%{NUMBER:value:int}']
```
+This example input and config parses a file using a timestamp in unix time:
+
+```
+1466004605 value=42
+1466004605.123456789 value=42
+```
+
+```toml
+[[inputs.logparser]]
+ [inputs.logparser.grok]
+ patterns = ['%{NUMBER:timestamp:ts-epoch} value=%{NUMBER:value:int}']
+```
+
This example parses a file using a built-in conversion and a custom pattern:
```
diff --git a/plugins/inputs/logparser/grok/grok.go b/plugins/inputs/logparser/grok/grok.go
index 0be18b54..491a1374 100644
--- a/plugins/inputs/logparser/grok/grok.go
+++ b/plugins/inputs/logparser/grok/grok.go
@@ -253,12 +253,30 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) {
case STRING:
fields[k] = strings.Trim(v, `"`)
case EPOCH:
- iv, err := strconv.ParseInt(v, 10, 64)
+ parts := strings.SplitN(v, ".", 2)
+ if len(parts) == 0 {
+ log.Printf("E! Error parsing %s to timestamp: %s", v, err)
+ break
+ }
+
+ sec, err := strconv.ParseInt(parts[0], 10, 64)
if err != nil {
- log.Printf("E! Error parsing %s to int: %s", v, err)
- } else {
- timestamp = time.Unix(iv, 0)
+ log.Printf("E! Error parsing %s to timestamp: %s", v, err)
+ break
+ }
+ ts := time.Unix(sec, 0)
+
+ if len(parts) == 2 {
+ padded := fmt.Sprintf("%-9s", parts[1])
+ nsString := strings.Replace(padded[:9], " ", "0", -1)
+ nanosec, err := strconv.ParseInt(nsString, 10, 64)
+ if err != nil {
+ log.Printf("E! Error parsing %s to timestamp: %s", v, err)
+ break
+ }
+ ts = ts.Add(time.Duration(nanosec) * time.Nanosecond)
}
+ timestamp = ts
case EPOCH_NANO:
iv, err := strconv.ParseInt(v, 10, 64)
if err != nil {
diff --git a/plugins/inputs/logparser/grok/grok_test.go b/plugins/inputs/logparser/grok/grok_test.go
index 6d07b6ec..480502d6 100644
--- a/plugins/inputs/logparser/grok/grok_test.go
+++ b/plugins/inputs/logparser/grok/grok_test.go
@@ -385,6 +385,77 @@ func TestParseEpoch(t *testing.T) {
assert.Equal(t, time.Unix(1466004605, 0), metricA.Time())
}
+func TestParseEpochDecimal(t *testing.T) {
+ var tests = []struct {
+ name string
+ line string
+ noMatch bool
+ err error
+ tags map[string]string
+ fields map[string]interface{}
+ time time.Time
+ }{
+ {
+ name: "ns precision",
+ line: "1466004605.359052000 value=42",
+ tags: map[string]string{},
+ fields: map[string]interface{}{
+ "value": int64(42),
+ },
+ time: time.Unix(0, 1466004605359052000),
+ },
+ {
+ name: "ms precision",
+ line: "1466004605.359 value=42",
+ tags: map[string]string{},
+ fields: map[string]interface{}{
+ "value": int64(42),
+ },
+ time: time.Unix(0, 1466004605359000000),
+ },
+ {
+ name: "second precision",
+ line: "1466004605 value=42",
+ tags: map[string]string{},
+ fields: map[string]interface{}{
+ "value": int64(42),
+ },
+ time: time.Unix(0, 1466004605000000000),
+ },
+ {
+ name: "sub ns precision",
+ line: "1466004605.123456789123 value=42",
+ tags: map[string]string{},
+ fields: map[string]interface{}{
+ "value": int64(42),
+ },
+ time: time.Unix(0, 1466004605123456789),
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ parser := &Parser{
+ Patterns: []string{"%{NUMBER:ts:ts-epoch} value=%{NUMBER:value:int}"},
+ }
+ assert.NoError(t, parser.Compile())
+ m, err := parser.ParseLine(tt.line)
+
+ if tt.noMatch {
+ require.Nil(t, m)
+ require.Nil(t, err)
+ return
+ }
+
+ require.Equal(t, tt.err, err)
+
+ require.NotNil(t, m)
+ require.Equal(t, tt.tags, m.Tags())
+ require.Equal(t, tt.fields, m.Fields())
+ require.Equal(t, tt.time, m.Time())
+ })
+ }
+}
+
func TestParseEpochErrors(t *testing.T) {
p := &Parser{
Patterns: []string{"%{MYAPP}"},
--
GitLab