From 20b4e8c779ef81caee1dd25fd974dc81b4271b0d Mon Sep 17 00:00:00 2001
From: "Chris H (CruftMaster)" <chris.hoolihan@hailocab.com>
Date: Thu, 3 Mar 2016 17:26:14 +0000
Subject: [PATCH] GREEDY field templates for the graphite parser, and support
for multiple specific field names
closes #789
---
CHANGELOG.md | 1 +
docs/DATA_FORMATS_INPUT.md | 21 ++++++--
plugins/parsers/graphite/parser.go | 27 +++++++---
plugins/parsers/graphite/parser_test.go | 65 ++++++++++++++++++++++---
4 files changed, 99 insertions(+), 15 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1b7e6fd1..0332a4ed 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@
- [#849](https://github.com/influxdata/telegraf/issues/849): Adding ability to parse single values as an input data type.
- [#844](https://github.com/influxdata/telegraf/pull/844): postgres_extensible plugin added. Thanks @menardorama!
- [#866](https://github.com/influxdata/telegraf/pull/866): couchbase input plugin. Thanks @ljosa!
+- [#789](https://github.com/influxdata/telegraf/pull/789): Support multiple field specification and `field*` in graphite templates. Thanks @chrusty!
### Bugfixes
- [#890](https://github.com/influxdata/telegraf/issues/890): Create TLS config even if only ssl_ca is provided.
diff --git a/docs/DATA_FORMATS_INPUT.md b/docs/DATA_FORMATS_INPUT.md
index 12c4d4cd..fd8ef853 100644
--- a/docs/DATA_FORMATS_INPUT.md
+++ b/docs/DATA_FORMATS_INPUT.md
@@ -220,16 +220,31 @@ So the following template:
```toml
templates = [
- "measurement.measurement.field.region"
+ "measurement.measurement.field.field.region"
]
```
would result in the following Graphite -> Telegraf transformation.
```
-cpu.usage.idle.us-west 100
-=> cpu_usage,region=us-west idle=100
+cpu.usage.idle.percent.us-west 100
+=> cpu_usage,region=us-west idle_percent=100
+```
+
+The field key can also be derived from the second "half" of the input metric-name by specifying ```field*```:
+```toml
+templates = [
+ "measurement.measurement.region.field*"
+]
+```
+
+would result in the following Graphite -> Telegraf transformation.
+
+```
+cpu.usage.us-west.idle.percentage 100
+=> cpu_usage,region=us-west idle_percentage=100
```
+(This cannot be used in conjunction with "measurement*"!)
#### Filter Templates:
diff --git a/plugins/parsers/graphite/parser.go b/plugins/parsers/graphite/parser.go
index 5e881506..8c31cd76 100644
--- a/plugins/parsers/graphite/parser.go
+++ b/plugins/parsers/graphite/parser.go
@@ -231,6 +231,7 @@ func (p *GraphiteParser) ApplyTemplate(line string) (string, map[string]string,
type template struct {
tags []string
defaultTags map[string]string
+ greedyField bool
greedyMeasurement bool
separator string
}
@@ -248,6 +249,8 @@ func NewTemplate(pattern string, defaultTags map[string]string, separator string
}
if tag == "measurement*" {
template.greedyMeasurement = true
+ } else if tag == "field*" {
+ template.greedyField = true
}
}
@@ -265,7 +268,7 @@ func (t *template) Apply(line string) (string, map[string]string, string, error)
var (
measurement []string
tags = make(map[string]string)
- field string
+ field []string
)
// Set any default tags
@@ -273,6 +276,18 @@ func (t *template) Apply(line string) (string, map[string]string, string, error)
tags[k] = v
}
+ // See if an invalid combination has been specified in the template:
+ for _, tag := range t.tags {
+ if tag == "measurement*" {
+ t.greedyMeasurement = true
+ } else if tag == "field*" {
+ t.greedyField = true
+ }
+ }
+ if t.greedyField && t.greedyMeasurement {
+ return "", nil, "", fmt.Errorf("either 'field*' or 'measurement*' can be used in each template (but not both together): %q", strings.Join(t.tags, t.separator))
+ }
+
for i, tag := range t.tags {
if i >= len(fields) {
continue
@@ -281,10 +296,10 @@ func (t *template) Apply(line string) (string, map[string]string, string, error)
if tag == "measurement" {
measurement = append(measurement, fields[i])
} else if tag == "field" {
- if len(field) != 0 {
- return "", nil, "", fmt.Errorf("'field' can only be used once in each template: %q", line)
- }
- field = fields[i]
+ field = append(field, fields[i])
+ } else if tag == "field*" {
+ field = append(field, fields[i:]...)
+ break
} else if tag == "measurement*" {
measurement = append(measurement, fields[i:]...)
break
@@ -293,7 +308,7 @@ func (t *template) Apply(line string) (string, map[string]string, string, error)
}
}
- return strings.Join(measurement, t.separator), tags, field, nil
+ return strings.Join(measurement, t.separator), tags, strings.Join(field, t.separator), nil
}
// matcher determines which template should be applied to a given metric
diff --git a/plugins/parsers/graphite/parser_test.go b/plugins/parsers/graphite/parser_test.go
index ccf478c7..5200cfbd 100644
--- a/plugins/parsers/graphite/parser_test.go
+++ b/plugins/parsers/graphite/parser_test.go
@@ -94,6 +94,20 @@ func TestTemplateApply(t *testing.T) {
measurement: "cpu.load",
tags: map[string]string{"zone": "us-west"},
},
+ {
+ test: "conjoined fields",
+ input: "prod.us-west.server01.cpu.util.idle.percent",
+ template: "env.zone.host.measurement.measurement.field*",
+ measurement: "cpu.util",
+ tags: map[string]string{"env": "prod", "zone": "us-west", "host": "server01"},
+ },
+ {
+ test: "multiple fields",
+ input: "prod.us-west.server01.cpu.util.idle.percent.free",
+ template: "env.zone.host.measurement.measurement.field.field.reading",
+ measurement: "cpu.util",
+ tags: map[string]string{"env": "prod", "zone": "us-west", "host": "server01", "reading": "free"},
+ },
}
for _, test := range tests {
@@ -187,6 +201,12 @@ func TestParse(t *testing.T) {
template: "measurement",
err: `field "cpu" time: strconv.ParseFloat: parsing "14199724z57825": invalid syntax`,
},
+ {
+ test: "measurement* and field* (invalid)",
+ input: `prod.us-west.server01.cpu.util.idle.percent 99.99 1419972457825`,
+ template: "env.zone.host.measurement*.field*",
+ err: `either 'field*' or 'measurement*' can be used in each template (but not both together): "env.zone.host.measurement*.field*"`,
+ },
}
for _, test := range tests {
@@ -574,15 +594,48 @@ func TestApplyTemplateField(t *testing.T) {
}
}
-func TestApplyTemplateFieldError(t *testing.T) {
+func TestApplyTemplateMultipleFieldsTogether(t *testing.T) {
p, err := NewGraphiteParser("_",
- []string{"current.* measurement.field.field"}, nil)
+ []string{"current.* measurement.measurement.field.field"}, nil)
assert.NoError(t, err)
- _, _, _, err = p.ApplyTemplate("current.users.logged_in")
- if err == nil {
- t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s", err,
- "'field' can only be used once in each template: current.users.logged_in")
+ measurement, _, field, err := p.ApplyTemplate("current.users.logged_in.ssh")
+
+ assert.Equal(t, "current_users", measurement)
+
+ if field != "logged_in_ssh" {
+ t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
+ field, "logged_in_ssh")
+ }
+}
+
+func TestApplyTemplateMultipleFieldsApart(t *testing.T) {
+ p, err := NewGraphiteParser("_",
+ []string{"current.* measurement.measurement.field.method.field"}, nil)
+ assert.NoError(t, err)
+
+ measurement, _, field, err := p.ApplyTemplate("current.users.logged_in.ssh.total")
+
+ assert.Equal(t, "current_users", measurement)
+
+ if field != "logged_in_total" {
+ t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
+ field, "logged_in_total")
+ }
+}
+
+func TestApplyTemplateGreedyField(t *testing.T) {
+ p, err := NewGraphiteParser("_",
+ []string{"current.* measurement.measurement.field*"}, nil)
+ assert.NoError(t, err)
+
+ measurement, _, field, err := p.ApplyTemplate("current.users.logged_in")
+
+ assert.Equal(t, "current_users", measurement)
+
+ if field != "logged_in" {
+ t.Errorf("Parser.ApplyTemplate unexpected result. got %s, exp %s",
+ field, "logged_in")
}
}
--
GitLab