blob: 270eb493d5ee784d5a7bf3c895f995f7ea7bc339 [file] [log] [blame]
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package configutil
import (
"context"
"errors"
"fmt"
"time"
"github.com/hashicorp/go-secure-stdlib/parseutil"
monitoring "cloud.google.com/go/monitoring/apiv3"
"github.com/armon/go-metrics"
"github.com/armon/go-metrics/circonus"
"github.com/armon/go-metrics/datadog"
"github.com/armon/go-metrics/prometheus"
stackdriver "github.com/google/go-metrics-stackdriver"
stackdrivervault "github.com/google/go-metrics-stackdriver/vault"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/hcl"
"github.com/hashicorp/hcl/hcl/ast"
"github.com/hashicorp/vault/helper/metricsutil"
"github.com/mitchellh/cli"
"google.golang.org/api/option"
)
const (
PrometheusDefaultRetentionTime = 24 * time.Hour
UsageGaugeDefaultPeriod = 10 * time.Minute
MaximumGaugeCardinalityDefault = 500
LeaseMetricsEpsilonDefault = time.Hour
NumLeaseMetricsTimeBucketsDefault = 168
)
// Telemetry is the telemetry configuration for the server
type Telemetry struct {
FoundKeys []string `hcl:",decodedFields"`
UnusedKeys UnusedKeyMap `hcl:",unusedKeyPositions"`
StatsiteAddr string `hcl:"statsite_address"`
StatsdAddr string `hcl:"statsd_address"`
DisableHostname bool `hcl:"disable_hostname"`
EnableHostnameLabel bool `hcl:"enable_hostname_label"`
MetricsPrefix string `hcl:"metrics_prefix"`
UsageGaugePeriod time.Duration
UsageGaugePeriodRaw interface{} `hcl:"usage_gauge_period,alias:UsageGaugePeriod"`
MaximumGaugeCardinality int `hcl:"maximum_gauge_cardinality"`
// Circonus: see https://github.com/circonus-labs/circonus-gometrics
// for more details on the various configuration options.
// Valid configuration combinations:
// - CirconusAPIToken
// metric management enabled (search for existing check or create a new one)
// - CirconusSubmissionUrl
// metric management disabled (use check with specified submission_url,
// broker must be using a public SSL certificate)
// - CirconusAPIToken + CirconusCheckSubmissionURL
// metric management enabled (use check with specified submission_url)
// - CirconusAPIToken + CirconusCheckID
// metric management enabled (use check with specified id)
// CirconusAPIToken is a valid API Token used to create/manage check. If provided,
// metric management is enabled.
// Default: none
CirconusAPIToken string `hcl:"circonus_api_token"`
// CirconusAPIApp is an app name associated with API token.
// Default: "consul"
CirconusAPIApp string `hcl:"circonus_api_app"`
// CirconusAPIURL is the base URL to use for contacting the Circonus API.
// Default: "https://api.circonus.com/v2"
CirconusAPIURL string `hcl:"circonus_api_url"`
// CirconusSubmissionInterval is the interval at which metrics are submitted to Circonus.
// Default: 10s
CirconusSubmissionInterval string `hcl:"circonus_submission_interval"`
// CirconusCheckSubmissionURL is the check.config.submission_url field from a
// previously created HTTPTRAP check.
// Default: none
CirconusCheckSubmissionURL string `hcl:"circonus_submission_url"`
// CirconusCheckID is the check id (not check bundle id) from a previously created
// HTTPTRAP check. The numeric portion of the check._cid field.
// Default: none
CirconusCheckID string `hcl:"circonus_check_id"`
// CirconusCheckForceMetricActivation will force enabling metrics, as they are encountered,
// if the metric already exists and is NOT active. If check management is enabled, the default
// behavior is to add new metrics as they are encountered. If the metric already exists in the
// check, it will *NOT* be activated. This setting overrides that behavior.
// Default: "false"
CirconusCheckForceMetricActivation string `hcl:"circonus_check_force_metric_activation"`
// CirconusCheckInstanceID serves to uniquely identify the metrics coming from this "instance".
// It can be used to maintain metric continuity with transient or ephemeral instances as
// they move around within an infrastructure.
// Default: hostname:app
CirconusCheckInstanceID string `hcl:"circonus_check_instance_id"`
// CirconusCheckSearchTag is a special tag which, when coupled with the instance id, helps to
// narrow down the search results when neither a Submission URL or Check ID is provided.
// Default: service:app (e.g. service:consul)
CirconusCheckSearchTag string `hcl:"circonus_check_search_tag"`
// CirconusCheckTags is a comma separated list of tags to apply to the check. Note that
// the value of CirconusCheckSearchTag will always be added to the check.
// Default: none
CirconusCheckTags string `hcl:"circonus_check_tags"`
// CirconusCheckDisplayName is the name for the check which will be displayed in the Circonus UI.
// Default: value of CirconusCheckInstanceID
CirconusCheckDisplayName string `hcl:"circonus_check_display_name"`
// CirconusBrokerID is an explicit broker to use when creating a new check. The numeric portion
// of broker._cid. If metric management is enabled and neither a Submission URL nor Check ID
// is provided, an attempt will be made to search for an existing check using Instance ID and
// Search Tag. If one is not found, a new HTTPTRAP check will be created.
// Default: use Select Tag if provided, otherwise, a random Enterprise Broker associated
// with the specified API token or the default Circonus Broker.
// Default: none
CirconusBrokerID string `hcl:"circonus_broker_id"`
// CirconusBrokerSelectTag is a special tag which will be used to select a broker when
// a Broker ID is not provided. The best use of this is to as a hint for which broker
// should be used based on *where* this particular instance is running.
// (e.g. a specific geo location or datacenter, dc:sfo)
// Default: none
CirconusBrokerSelectTag string `hcl:"circonus_broker_select_tag"`
// Dogstats:
// DogStatsdAddr is the address of a dogstatsd instance. If provided,
// metrics will be sent to that instance
DogStatsDAddr string `hcl:"dogstatsd_addr"`
// DogStatsdTags are the global tags that should be sent with each packet to dogstatsd
// It is a list of strings, where each string looks like "my_tag_name:my_tag_value"
DogStatsDTags []string `hcl:"dogstatsd_tags"`
// Prometheus:
// PrometheusRetentionTime is the retention time for prometheus metrics if greater than 0.
// Default: 24h
PrometheusRetentionTime time.Duration `hcl:"-"`
PrometheusRetentionTimeRaw interface{} `hcl:"prometheus_retention_time"`
// Stackdriver:
// StackdriverProjectID is the project to publish stackdriver metrics to.
StackdriverProjectID string `hcl:"stackdriver_project_id"`
// StackdriverLocation is the GCP or AWS region of the monitored resource.
StackdriverLocation string `hcl:"stackdriver_location"`
// StackdriverNamespace is the namespace identifier, such as a cluster name.
StackdriverNamespace string `hcl:"stackdriver_namespace"`
// StackdriverDebugLogs will write additional stackdriver related debug logs to stderr.
StackdriverDebugLogs bool `hcl:"stackdriver_debug_logs"`
// How often metrics for lease expiry will be aggregated
LeaseMetricsEpsilon time.Duration
LeaseMetricsEpsilonRaw interface{} `hcl:"lease_metrics_epsilon"`
// Number of buckets by time that will be used in lease aggregation
NumLeaseMetricsTimeBuckets int `hcl:"num_lease_metrics_buckets"`
// Whether or not telemetry should add labels for namespaces
LeaseMetricsNameSpaceLabels bool `hcl:"add_lease_metrics_namespace_labels"`
// FilterDefault is the default for whether to allow a metric that's not
// covered by the prefix filter.
FilterDefault *bool `hcl:"filter_default"`
// PrefixFilter is a list of filter rules to apply for allowing
// or blocking metrics by prefix.
PrefixFilter []string `hcl:"prefix_filter"`
}
func (t *Telemetry) Validate(source string) []ConfigError {
return ValidateUnusedFields(t.UnusedKeys, source)
}
func (t *Telemetry) GoString() string {
return fmt.Sprintf("*%#v", *t)
}
func parseTelemetry(result *SharedConfig, list *ast.ObjectList) error {
if len(list.Items) > 1 {
return fmt.Errorf("only one 'telemetry' block is permitted")
}
// Get our one item
item := list.Items[0]
if result.Telemetry == nil {
result.Telemetry = &Telemetry{}
}
if err := hcl.DecodeObject(&result.Telemetry, item.Val); err != nil {
return multierror.Prefix(err, "telemetry:")
}
if result.Telemetry.PrometheusRetentionTimeRaw != nil {
var err error
if result.Telemetry.PrometheusRetentionTime, err = parseutil.ParseDurationSecond(result.Telemetry.PrometheusRetentionTimeRaw); err != nil {
return err
}
result.Telemetry.PrometheusRetentionTimeRaw = nil
} else {
result.Telemetry.PrometheusRetentionTime = PrometheusDefaultRetentionTime
}
if result.Telemetry.UsageGaugePeriodRaw != nil {
if result.Telemetry.UsageGaugePeriodRaw == "none" {
result.Telemetry.UsageGaugePeriod = 0
} else {
var err error
if result.Telemetry.UsageGaugePeriod, err = parseutil.ParseDurationSecond(result.Telemetry.UsageGaugePeriodRaw); err != nil {
return err
}
result.Telemetry.UsageGaugePeriodRaw = nil
}
} else {
result.Telemetry.UsageGaugePeriod = UsageGaugeDefaultPeriod
}
if result.Telemetry.MaximumGaugeCardinality == 0 {
result.Telemetry.MaximumGaugeCardinality = MaximumGaugeCardinalityDefault
}
if result.Telemetry.LeaseMetricsEpsilonRaw != nil {
if result.Telemetry.LeaseMetricsEpsilonRaw == "none" {
result.Telemetry.LeaseMetricsEpsilonRaw = 0
} else {
var err error
if result.Telemetry.LeaseMetricsEpsilon, err = parseutil.ParseDurationSecond(result.Telemetry.LeaseMetricsEpsilonRaw); err != nil {
return err
}
result.Telemetry.LeaseMetricsEpsilonRaw = nil
}
} else {
result.Telemetry.LeaseMetricsEpsilon = LeaseMetricsEpsilonDefault
}
if result.Telemetry.NumLeaseMetricsTimeBuckets == 0 {
result.Telemetry.NumLeaseMetricsTimeBuckets = NumLeaseMetricsTimeBucketsDefault
}
return nil
}
type SetupTelemetryOpts struct {
Config *Telemetry
Ui cli.Ui
ServiceName string
DisplayName string
UserAgent string
ClusterName string
}
// SetupTelemetry is used to setup the telemetry sub-systems and returns the
// in-memory sink to be used in http configuration
func SetupTelemetry(opts *SetupTelemetryOpts) (*metrics.InmemSink, *metricsutil.ClusterMetricSink, bool, error) {
if opts == nil {
return nil, nil, false, errors.New("nil opts passed into SetupTelemetry")
}
if opts.Config == nil {
opts.Config = &Telemetry{}
}
/* Setup telemetry
Aggregate on 10 second intervals for 1 minute. Expose the
metrics over stderr when there is a SIGUSR1 received.
*/
inm := metrics.NewInmemSink(10*time.Second, time.Minute)
metrics.DefaultInmemSignal(inm)
if opts.Config.MetricsPrefix != "" {
opts.ServiceName = opts.Config.MetricsPrefix
}
metricsConf := metrics.DefaultConfig(opts.ServiceName)
metricsConf.EnableHostname = !opts.Config.DisableHostname
metricsConf.EnableHostnameLabel = opts.Config.EnableHostnameLabel
if opts.Config.FilterDefault != nil {
metricsConf.FilterDefault = *opts.Config.FilterDefault
}
// Configure the statsite sink
var fanout metrics.FanoutSink
var prometheusEnabled bool
// Configure the Prometheus sink
if opts.Config.PrometheusRetentionTime != 0 {
prometheusEnabled = true
prometheusOpts := prometheus.PrometheusOpts{
Expiration: opts.Config.PrometheusRetentionTime,
}
sink, err := prometheus.NewPrometheusSinkFrom(prometheusOpts)
if err != nil {
return nil, nil, false, err
}
fanout = append(fanout, sink)
}
if opts.Config.StatsiteAddr != "" {
sink, err := metrics.NewStatsiteSink(opts.Config.StatsiteAddr)
if err != nil {
return nil, nil, false, err
}
fanout = append(fanout, sink)
}
// Configure the statsd sink
if opts.Config.StatsdAddr != "" {
sink, err := metrics.NewStatsdSink(opts.Config.StatsdAddr)
if err != nil {
return nil, nil, false, err
}
fanout = append(fanout, sink)
}
// Configure the Circonus sink
if opts.Config.CirconusAPIToken != "" || opts.Config.CirconusCheckSubmissionURL != "" {
cfg := &circonus.Config{}
cfg.Interval = opts.Config.CirconusSubmissionInterval
cfg.CheckManager.API.TokenKey = opts.Config.CirconusAPIToken
cfg.CheckManager.API.TokenApp = opts.Config.CirconusAPIApp
cfg.CheckManager.API.URL = opts.Config.CirconusAPIURL
cfg.CheckManager.Check.SubmissionURL = opts.Config.CirconusCheckSubmissionURL
cfg.CheckManager.Check.ID = opts.Config.CirconusCheckID
cfg.CheckManager.Check.ForceMetricActivation = opts.Config.CirconusCheckForceMetricActivation
cfg.CheckManager.Check.InstanceID = opts.Config.CirconusCheckInstanceID
cfg.CheckManager.Check.SearchTag = opts.Config.CirconusCheckSearchTag
cfg.CheckManager.Check.DisplayName = opts.Config.CirconusCheckDisplayName
cfg.CheckManager.Check.Tags = opts.Config.CirconusCheckTags
cfg.CheckManager.Broker.ID = opts.Config.CirconusBrokerID
cfg.CheckManager.Broker.SelectTag = opts.Config.CirconusBrokerSelectTag
if cfg.CheckManager.API.TokenApp == "" {
cfg.CheckManager.API.TokenApp = opts.ServiceName
}
if cfg.CheckManager.Check.DisplayName == "" {
cfg.CheckManager.Check.DisplayName = opts.DisplayName
}
if cfg.CheckManager.Check.SearchTag == "" {
cfg.CheckManager.Check.SearchTag = fmt.Sprintf("service:%s", opts.ServiceName)
}
sink, err := circonus.NewCirconusSink(cfg)
if err != nil {
return nil, nil, false, err
}
sink.Start()
fanout = append(fanout, sink)
}
if opts.Config.DogStatsDAddr != "" {
var tags []string
if opts.Config.DogStatsDTags != nil {
tags = opts.Config.DogStatsDTags
}
sink, err := datadog.NewDogStatsdSink(opts.Config.DogStatsDAddr, metricsConf.HostName)
if err != nil {
return nil, nil, false, fmt.Errorf("failed to start DogStatsD sink: %w", err)
}
sink.SetTags(tags)
fanout = append(fanout, sink)
}
// Configure the stackdriver sink
if opts.Config.StackdriverProjectID != "" {
client, err := monitoring.NewMetricClient(context.Background(), option.WithUserAgent(opts.UserAgent))
if err != nil {
return nil, nil, false, fmt.Errorf("Failed to create stackdriver client: %v", err)
}
sink := stackdriver.NewSink(client, &stackdriver.Config{
LabelExtractor: stackdrivervault.Extractor,
Bucketer: stackdrivervault.Bucketer,
ProjectID: opts.Config.StackdriverProjectID,
Location: opts.Config.StackdriverLocation,
Namespace: opts.Config.StackdriverNamespace,
DebugLogs: opts.Config.StackdriverDebugLogs,
})
fanout = append(fanout, sink)
}
// Initialize the global sink
if len(fanout) > 1 {
// Hostname enabled will create poor quality metrics name for prometheus
if !opts.Config.DisableHostname {
opts.Ui.Warn("telemetry.disable_hostname has been set to false. Recommended setting is true for Prometheus to avoid poorly named metrics.")
}
} else {
metricsConf.EnableHostname = false
}
fanout = append(fanout, inm)
globalMetrics, err := metrics.NewGlobal(metricsConf, fanout)
if err != nil {
return nil, nil, false, err
}
// Intialize a wrapper around the global sink; this will be passed to Core
// and to any backend.
wrapper := metricsutil.NewClusterMetricSink(opts.ClusterName, globalMetrics)
wrapper.MaxGaugeCardinality = opts.Config.MaximumGaugeCardinality
wrapper.GaugeInterval = opts.Config.UsageGaugePeriod
wrapper.TelemetryConsts.LeaseMetricsEpsilon = opts.Config.LeaseMetricsEpsilon
wrapper.TelemetryConsts.LeaseMetricsNameSpaceLabels = opts.Config.LeaseMetricsNameSpaceLabels
wrapper.TelemetryConsts.NumLeaseMetricsTimeBuckets = opts.Config.NumLeaseMetricsTimeBuckets
// Parse the metric filters
telemetryAllowedPrefixes, telemetryBlockedPrefixes, err := parsePrefixFilter(opts.Config.PrefixFilter)
if err != nil {
return nil, nil, false, err
}
metrics.UpdateFilter(telemetryAllowedPrefixes, telemetryBlockedPrefixes)
return inm, wrapper, prometheusEnabled, nil
}
func parsePrefixFilter(prefixFilters []string) ([]string, []string, error) {
var telemetryAllowedPrefixes, telemetryBlockedPrefixes []string
for _, rule := range prefixFilters {
if rule == "" {
return nil, nil, fmt.Errorf("Cannot have empty filter rule in prefix_filter")
}
switch rule[0] {
case '+':
telemetryAllowedPrefixes = append(telemetryAllowedPrefixes, rule[1:])
case '-':
telemetryBlockedPrefixes = append(telemetryBlockedPrefixes, rule[1:])
default:
return nil, nil, fmt.Errorf("Filter rule must begin with either '+' or '-': %q", rule)
}
}
return telemetryAllowedPrefixes, telemetryBlockedPrefixes, nil
}