Add k8s object labels to tcp metrics

Co-authored-by: Dean Coakley <dean.s.coakley@gmail.com>

collector/nginx_plus.go

@@ -15,30 +15,37 @@ type LabelUpdater interface {
 	DeleteUpstreamServerPeerLabels(peers []string)
 	UpdateUpstreamServerLabels(upstreamServerLabelValues map[string][]string)
 	DeleteUpstreamServerLabels(upstreamNames []string)
+	UpdateStreamUpstreamServerPeerLabels(streamUpstreamServerPeerLabels map[string][]string)
+	DeleteStreamUpstreamServerPeerLabels(peers []string)
+	UpdateStreamUpstreamServerLabels(streamUpstreamServerPeerLabels map[string][]string)
+	DeleteStreamUpstreamServerLabels(peers []string)
 	UpdateServerZoneLabels(serverZoneLabelValues map[string][]string)
 	DeleteServerZoneLabels(zoneNames []string)
 }
 
 // NginxPlusCollector collects NGINX Plus metrics. It implements prometheus.Collector interface.
 type NginxPlusCollector struct {
-	nginxClient                 *plusclient.NginxClient
+	nginxClient                    *plusclient.NginxClient
-	totalMetrics                map[string]*prometheus.Desc
+	totalMetrics                   map[string]*prometheus.Desc
-	serverZoneMetrics           map[string]*prometheus.Desc
+	serverZoneMetrics              map[string]*prometheus.Desc
-	upstreamMetrics             map[string]*prometheus.Desc
+	upstreamMetrics                map[string]*prometheus.Desc
-	upstreamServerMetrics       map[string]*prometheus.Desc
+	upstreamServerMetrics          map[string]*prometheus.Desc
-	streamServerZoneMetrics     map[string]*prometheus.Desc
+	streamServerZoneMetrics        map[string]*prometheus.Desc
-	streamUpstreamMetrics       map[string]*prometheus.Desc
+	streamZoneSyncMetrics          map[string]*prometheus.Desc
-	streamUpstreamServerMetrics map[string]*prometheus.Desc
+	streamUpstreamMetrics          map[string]*prometheus.Desc
-	streamZoneSyncMetrics       map[string]*prometheus.Desc
+	streamUpstreamServerMetrics    map[string]*prometheus.Desc
-	locationZoneMetrics         map[string]*prometheus.Desc
+	locationZoneMetrics            map[string]*prometheus.Desc
-	resolverMetrics             map[string]*prometheus.Desc
+	resolverMetrics                map[string]*prometheus.Desc
-	upMetric                    prometheus.Gauge
+	upMetric                       prometheus.Gauge
-	mutex                       sync.Mutex
+	mutex                          sync.Mutex
-	variableLabelNames          VariableLabelNames
+	variableLabelNames             VariableLabelNames
-	upstreamServerLabels        map[string][]string
+	upstreamServerLabels           map[string][]string
-	serverZoneLabels            map[string][]string
+	streamUpstreamServerLabels     map[string][]string
-	upstreamServerPeerLabels    map[string][]string
+	serverZoneLabels               map[string][]string
-	variableLabelsMutex         sync.RWMutex
+	streamServerZoneLabels         map[string][]string
+	upstreamServerPeerLabels       map[string][]string
+	streamUpstreamServerPeerLabels map[string][]string
+	variableLabelsMutex            sync.RWMutex
 }
 
 // UpdateUpstreamServerPeerLabels updates the Upstream Server Peer Labels
@@ -59,6 +66,24 @@ func (c *NginxPlusCollector) DeleteUpstreamServerPeerLabels(peers []string) {
 	c.variableLabelsMutex.Unlock()
 }
 
+// UpdateStreamUpstreamServerPeerLabels updates the Upstream Server Peer Labels
+func (c *NginxPlusCollector) UpdateStreamUpstreamServerPeerLabels(streamUpstreamServerPeerLabels map[string][]string) {
+	c.variableLabelsMutex.Lock()
+	for k, v := range streamUpstreamServerPeerLabels {
+		c.streamUpstreamServerPeerLabels[k] = v
+	}
+	c.variableLabelsMutex.Unlock()
+}
+
+// DeleteStreamUpstreamServerPeerLabels deletes the Upstream Server Peer Labels
+func (c *NginxPlusCollector) DeleteStreamUpstreamServerPeerLabels(peers []string) {
+	c.variableLabelsMutex.Lock()
+	for _, k := range peers {
+		delete(c.streamUpstreamServerPeerLabels, k)
+	}
+	c.variableLabelsMutex.Unlock()
+}
+
 // UpdateUpstreamServerLabels updates the Upstream Server Labels
 func (c *NginxPlusCollector) UpdateUpstreamServerLabels(upstreamServerLabelValues map[string][]string) {
 	c.variableLabelsMutex.Lock()
@@ -77,6 +102,24 @@ func (c *NginxPlusCollector) DeleteUpstreamServerLabels(upstreamNames []string)
 	c.variableLabelsMutex.Unlock()
 }
 
+// UpdateStreamUpstreamServerLabels updates the Upstream Server Labels
+func (c *NginxPlusCollector) UpdateStreamUpstreamServerLabels(streamUpstreamServerLabelValues map[string][]string) {
+	c.variableLabelsMutex.Lock()
+	for k, v := range streamUpstreamServerLabelValues {
+		c.streamUpstreamServerLabels[k] = v
+	}
+	c.variableLabelsMutex.Unlock()
+}
+
+// DeleteStreamUpstreamServerLabels deletes the Upstream Server Labels
+func (c *NginxPlusCollector) DeleteStreamUpstreamServerLabels(streamUpstreamNames []string) {
+	c.variableLabelsMutex.Lock()
+	for _, k := range streamUpstreamNames {
+		delete(c.streamUpstreamServerLabels, k)
+	}
+	c.variableLabelsMutex.Unlock()
+}
+
 // UpdateServerZoneLabels updates the Server Zone Labels
 func (c *NginxPlusCollector) UpdateServerZoneLabels(serverZoneLabelValues map[string][]string) {
 	c.variableLabelsMutex.Lock()
@@ -95,50 +138,93 @@ func (c *NginxPlusCollector) DeleteServerZoneLabels(zoneNames []string) {
 	c.variableLabelsMutex.Unlock()
 }
 
+// UpdateStreamServerZoneLabels updates the Stream Server Zone Labels
+func (c *NginxPlusCollector) UpdateStreamServerZoneLabels(streamServerZoneLabelValues map[string][]string) {
+	c.variableLabelsMutex.Lock()
+	for k, v := range streamServerZoneLabelValues {
+		c.streamServerZoneLabels[k] = v
+	}
+	c.variableLabelsMutex.Unlock()
+}
+
+// DeleteStreamServerZoneLabels deletes the Stream Server Zone Labels
+func (c *NginxPlusCollector) DeleteStreamServerZoneLabels(zoneNames []string) {
+	c.variableLabelsMutex.Lock()
+	for _, k := range zoneNames {
+		delete(c.streamServerZoneLabels, k)
+	}
+	c.variableLabelsMutex.Unlock()
+}
+
 func (c *NginxPlusCollector) getUpstreamServerLabelValues(upstreamName string) []string {
 	c.variableLabelsMutex.RLock()
 	defer c.variableLabelsMutex.RUnlock()
 	return c.upstreamServerLabels[upstreamName]
 }
 
+func (c *NginxPlusCollector) getStreamUpstreamServerLabelValues(upstreamName string) []string {
+	c.variableLabelsMutex.RLock()
+	defer c.variableLabelsMutex.RUnlock()
+	return c.streamUpstreamServerLabels[upstreamName]
+}
+
 func (c *NginxPlusCollector) getServerZoneLabelValues(zoneName string) []string {
 	c.variableLabelsMutex.RLock()
 	defer c.variableLabelsMutex.RUnlock()
 	return c.serverZoneLabels[zoneName]
 }
 
+func (c *NginxPlusCollector) getStreamServerZoneLabelValues(zoneName string) []string {
+	c.variableLabelsMutex.RLock()
+	defer c.variableLabelsMutex.RUnlock()
+	return c.streamServerZoneLabels[zoneName]
+}
+
 func (c *NginxPlusCollector) getUpstreamServerPeerLabelValues(peer string) []string {
 	c.variableLabelsMutex.RLock()
 	defer c.variableLabelsMutex.RUnlock()
 	return c.upstreamServerPeerLabels[peer]
 }
 
+func (c *NginxPlusCollector) getStreamUpstreamServerPeerLabelValues(peer string) []string {
+	c.variableLabelsMutex.RLock()
+	defer c.variableLabelsMutex.RUnlock()
+	return c.streamUpstreamServerPeerLabels[peer]
+}
+
 // VariableLabelNames holds all the variable label names for the different metrics
 type VariableLabelNames struct {
-	UpstreamServerVariableLabelNames     []string
+	UpstreamServerVariableLabelNames           []string
-	ServerZoneVariableLabelNames         []string
+	ServerZoneVariableLabelNames               []string
-	UpstreamServerPeerVariableLabelNames []string
+	StreamServerZoneVariableLabelNames         []string
+	UpstreamServerPeerVariableLabelNames       []string
+	StreamUpstreamServerPeerVariableLabelNames []string
+	StreamUpstreamServerVariableLabelNames     []string
 }
 
 // NewVariableLabels creates a new struct for VariableNames for the collector
 func NewVariableLabelNames(upstreamServerVariableLabelNames []string, serverZoneVariableLabelNames []string,
-	upstreamServerPeerVariableLabelNames []string) VariableLabelNames {
+	upstreamServerPeerVariableLabelNames []string, streamUpstreamServerVariableLabelNames []string) VariableLabelNames {
 	return VariableLabelNames{
-		UpstreamServerVariableLabelNames:     upstreamServerVariableLabelNames,
+		UpstreamServerVariableLabelNames:       upstreamServerVariableLabelNames,
-		ServerZoneVariableLabelNames:         serverZoneVariableLabelNames,
+		ServerZoneVariableLabelNames:           serverZoneVariableLabelNames,
-		UpstreamServerPeerVariableLabelNames: upstreamServerPeerVariableLabelNames,
+		UpstreamServerPeerVariableLabelNames:   upstreamServerPeerVariableLabelNames,
+		StreamUpstreamServerVariableLabelNames: streamUpstreamServerVariableLabelNames,
 	}
 }
 
 // NewNginxPlusCollector creates an NginxPlusCollector.
 func NewNginxPlusCollector(nginxClient *plusclient.NginxClient, namespace string, variableLabelNames VariableLabelNames, constLabels map[string]string) *NginxPlusCollector {
 	upstreamServerVariableLabelNames := append(variableLabelNames.UpstreamServerVariableLabelNames, variableLabelNames.UpstreamServerPeerVariableLabelNames...)
+	streamUpstreamServerVariableLabelNames := append(variableLabelNames.StreamUpstreamServerVariableLabelNames, variableLabelNames.StreamUpstreamServerPeerVariableLabelNames...)
 	return &NginxPlusCollector{
-		variableLabelNames:       variableLabelNames,
+		variableLabelNames:             variableLabelNames,
-		upstreamServerLabels:     make(map[string][]string),
+		upstreamServerLabels:           make(map[string][]string),
-		serverZoneLabels:         make(map[string][]string),
+		serverZoneLabels:               make(map[string][]string),
-		upstreamServerPeerLabels: make(map[string][]string),
+		upstreamServerPeerLabels:       make(map[string][]string),
-		nginxClient:              nginxClient,
+		streamUpstreamServerPeerLabels: make(map[string][]string),
+		streamUpstreamServerLabels:     make(map[string][]string),
+		nginxClient:                    nginxClient,
 		totalMetrics: map[string]*prometheus.Desc{
 			"connections_accepted":  newGlobalMetric(namespace, "connections_accepted", "Accepted client connections", constLabels),
 			"connections_dropped":   newGlobalMetric(namespace, "connections_dropped", "Dropped client connections", constLabels),
@@ -151,26 +237,26 @@ func NewNginxPlusCollector(nginxClient *plusclient.NginxClient, namespace string
 			"ssl_session_reuses":    newGlobalMetric(namespace, "ssl_session_reuses", "Session reuses during SSL handshake", constLabels),
 		},
 		serverZoneMetrics: map[string]*prometheus.Desc{
-			"processing":    newServerZoneMetric(namespace, "processing", "Client requests that are currently being processed", variableLabelNames.ServerZoneVariableLabelNames, constLabels),
+			"processing":    newServerZoneMetric(namespace, "processing", "Client requests that are currently being processed", variableLabelNames.StreamServerZoneVariableLabelNames, constLabels),
-			"requests":      newServerZoneMetric(namespace, "requests", "Total client requests", variableLabelNames.ServerZoneVariableLabelNames, constLabels),
+			"requests":      newServerZoneMetric(namespace, "requests", "Total client requests", variableLabelNames.StreamServerZoneVariableLabelNames, constLabels),
-			"responses_1xx": newServerZoneMetric(namespace, "responses", "Total responses sent to clients", variableLabelNames.ServerZoneVariableLabelNames, MergeLabels(constLabels, prometheus.Labels{"code": "1xx"})),
+			"responses_1xx": newServerZoneMetric(namespace, "responses", "Total responses sent to clients", variableLabelNames.StreamServerZoneVariableLabelNames, MergeLabels(constLabels, prometheus.Labels{"code": "1xx"})),
-			"responses_2xx": newServerZoneMetric(namespace, "responses", "Total responses sent to clients", variableLabelNames.ServerZoneVariableLabelNames, MergeLabels(constLabels, prometheus.Labels{"code": "2xx"})),
+			"responses_2xx": newServerZoneMetric(namespace, "responses", "Total responses sent to clients", variableLabelNames.StreamServerZoneVariableLabelNames, MergeLabels(constLabels, prometheus.Labels{"code": "2xx"})),
-			"responses_3xx": newServerZoneMetric(namespace, "responses", "Total responses sent to clients", variableLabelNames.ServerZoneVariableLabelNames, MergeLabels(constLabels, prometheus.Labels{"code": "3xx"})),
+			"responses_3xx": newServerZoneMetric(namespace, "responses", "Total responses sent to clients", variableLabelNames.StreamServerZoneVariableLabelNames, MergeLabels(constLabels, prometheus.Labels{"code": "3xx"})),
-			"responses_4xx": newServerZoneMetric(namespace, "responses", "Total responses sent to clients", variableLabelNames.ServerZoneVariableLabelNames, MergeLabels(constLabels, prometheus.Labels{"code": "4xx"})),
+			"responses_4xx": newServerZoneMetric(namespace, "responses", "Total responses sent to clients", variableLabelNames.StreamServerZoneVariableLabelNames, MergeLabels(constLabels, prometheus.Labels{"code": "4xx"})),
-			"responses_5xx": newServerZoneMetric(namespace, "responses", "Total responses sent to clients", variableLabelNames.ServerZoneVariableLabelNames, MergeLabels(constLabels, prometheus.Labels{"code": "5xx"})),
+			"responses_5xx": newServerZoneMetric(namespace, "responses", "Total responses sent to clients", variableLabelNames.StreamServerZoneVariableLabelNames, MergeLabels(constLabels, prometheus.Labels{"code": "5xx"})),
-			"discarded":     newServerZoneMetric(namespace, "discarded", "Requests completed without sending a response", variableLabelNames.ServerZoneVariableLabelNames, constLabels),
+			"discarded":     newServerZoneMetric(namespace, "discarded", "Requests completed without sending a response", variableLabelNames.StreamServerZoneVariableLabelNames, constLabels),
-			"received":      newServerZoneMetric(namespace, "received", "Bytes received from clients", variableLabelNames.ServerZoneVariableLabelNames, constLabels),
+			"received":      newServerZoneMetric(namespace, "received", "Bytes received from clients", variableLabelNames.StreamServerZoneVariableLabelNames, constLabels),
-			"sent":          newServerZoneMetric(namespace, "sent", "Bytes sent to clients", variableLabelNames.ServerZoneVariableLabelNames, constLabels),
+			"sent":          newServerZoneMetric(namespace, "sent", "Bytes sent to clients", variableLabelNames.StreamServerZoneVariableLabelNames, constLabels),
 		},
 		streamServerZoneMetrics: map[string]*prometheus.Desc{
-			"processing":   newStreamServerZoneMetric(namespace, "processing", "Client connections that are currently being processed", constLabels),
+			"processing":   newStreamServerZoneMetric(namespace, "processing", "Client connections that are currently being processed", variableLabelNames.ServerZoneVariableLabelNames, constLabels),
-			"connections":  newStreamServerZoneMetric(namespace, "connections", "Total connections", constLabels),
+			"connections":  newStreamServerZoneMetric(namespace, "connections", "Total connections", variableLabelNames.ServerZoneVariableLabelNames, constLabels),
-			"sessions_2xx": newStreamServerZoneMetric(namespace, "sessions", "Total sessions completed", MergeLabels(constLabels, prometheus.Labels{"code": "2xx"})),
+			"sessions_2xx": newStreamServerZoneMetric(namespace, "sessions", "Total sessions completed", variableLabelNames.ServerZoneVariableLabelNames, MergeLabels(constLabels, prometheus.Labels{"code": "2xx"})),
-			"sessions_4xx": newStreamServerZoneMetric(namespace, "sessions", "Total sessions completed", MergeLabels(constLabels, prometheus.Labels{"code": "4xx"})),
+			"sessions_4xx": newStreamServerZoneMetric(namespace, "sessions", "Total sessions completed", variableLabelNames.ServerZoneVariableLabelNames, MergeLabels(constLabels, prometheus.Labels{"code": "4xx"})),
-			"sessions_5xx": newStreamServerZoneMetric(namespace, "sessions", "Total sessions completed", MergeLabels(constLabels, prometheus.Labels{"code": "5xx"})),
+			"sessions_5xx": newStreamServerZoneMetric(namespace, "sessions", "Total sessions completed", variableLabelNames.ServerZoneVariableLabelNames, MergeLabels(constLabels, prometheus.Labels{"code": "5xx"})),
-			"discarded":    newStreamServerZoneMetric(namespace, "discarded", "Connections completed without creating a session", constLabels),
+			"discarded":    newStreamServerZoneMetric(namespace, "discarded", "Connections completed without creating a session", variableLabelNames.ServerZoneVariableLabelNames, constLabels),
-			"received":     newStreamServerZoneMetric(namespace, "received", "Bytes received from clients", constLabels),
+			"received":     newStreamServerZoneMetric(namespace, "received", "Bytes received from clients", variableLabelNames.ServerZoneVariableLabelNames, constLabels),
-			"sent":         newStreamServerZoneMetric(namespace, "sent", "Bytes sent to clients", constLabels),
+			"sent":         newStreamServerZoneMetric(namespace, "sent", "Bytes sent to clients", variableLabelNames.ServerZoneVariableLabelNames, constLabels),
 		},
 		upstreamMetrics: map[string]*prometheus.Desc{
 			"keepalives": newUpstreamMetric(namespace, "keepalives", "Idle keepalive connections", constLabels),
@@ -199,19 +285,19 @@ func NewNginxPlusCollector(nginxClient *plusclient.NginxClient, namespace string
 			"health_checks_unhealthy": newUpstreamServerMetric(namespace, "health_checks_unhealthy", "How many times the server became unhealthy (state 'unhealthy')", upstreamServerVariableLabelNames, constLabels),
 		},
 		streamUpstreamServerMetrics: map[string]*prometheus.Desc{
-			"state":                   newStreamUpstreamServerMetric(namespace, "state", "Current state", constLabels),
+			"state":                   newStreamUpstreamServerMetric(namespace, "state", "Current state", streamUpstreamServerVariableLabelNames, constLabels),
-			"active":                  newStreamUpstreamServerMetric(namespace, "active", "Active connections", constLabels),
+			"active":                  newStreamUpstreamServerMetric(namespace, "active", "Active connections", streamUpstreamServerVariableLabelNames, constLabels),
-			"sent":                    newStreamUpstreamServerMetric(namespace, "sent", "Bytes sent to this server", constLabels),
+			"sent":                    newStreamUpstreamServerMetric(namespace, "sent", "Bytes sent to this server", streamUpstreamServerVariableLabelNames, constLabels),
-			"received":                newStreamUpstreamServerMetric(namespace, "received", "Bytes received from this server", constLabels),
+			"received":                newStreamUpstreamServerMetric(namespace, "received", "Bytes received from this server", streamUpstreamServerVariableLabelNames, constLabels),
-			"fails":                   newStreamUpstreamServerMetric(namespace, "fails", "Number of unsuccessful attempts to communicate with the server", constLabels),
+			"fails":                   newStreamUpstreamServerMetric(namespace, "fails", "Number of unsuccessful attempts to communicate with the server", streamUpstreamServerVariableLabelNames, constLabels),
-			"unavail":                 newStreamUpstreamServerMetric(namespace, "unavail", "How many times the server became unavailable for client connections (state 'unavail') due to the number of unsuccessful attempts reaching the max_fails threshold", constLabels),
+			"unavail":                 newStreamUpstreamServerMetric(namespace, "unavail", "How many times the server became unavailable for client connections (state 'unavail') due to the number of unsuccessful attempts reaching the max_fails threshold", streamUpstreamServerVariableLabelNames, constLabels),
-			"connections":             newStreamUpstreamServerMetric(namespace, "connections", "Total number of client connections forwarded to this server", constLabels),
+			"connections":             newStreamUpstreamServerMetric(namespace, "connections", "Total number of client connections forwarded to this server", streamUpstreamServerVariableLabelNames, constLabels),
-			"connect_time":            newStreamUpstreamServerMetric(namespace, "connect_time", "Average time to connect to the upstream server", constLabels),
+			"connect_time":            newStreamUpstreamServerMetric(namespace, "connect_time", "Average time to connect to the upstream server", streamUpstreamServerVariableLabelNames, constLabels),
-			"first_byte_time":         newStreamUpstreamServerMetric(namespace, "first_byte_time", "Average time to receive the first byte of data", constLabels),
+			"first_byte_time":         newStreamUpstreamServerMetric(namespace, "first_byte_time", "Average time to receive the first byte of data", streamUpstreamServerVariableLabelNames, constLabels),
-			"response_time":           newStreamUpstreamServerMetric(namespace, "response_time", "Average time to receive the last byte of data", constLabels),
+			"response_time":           newStreamUpstreamServerMetric(namespace, "response_time", "Average time to receive the last byte of data", streamUpstreamServerVariableLabelNames, constLabels),
-			"health_checks_checks":    newStreamUpstreamServerMetric(namespace, "health_checks_checks", "Total health check requests", constLabels),
+			"health_checks_checks":    newStreamUpstreamServerMetric(namespace, "health_checks_checks", "Total health check requests", streamUpstreamServerVariableLabelNames, constLabels),
-			"health_checks_fails":     newStreamUpstreamServerMetric(namespace, "health_checks_fails", "Failed health checks", constLabels),
+			"health_checks_fails":     newStreamUpstreamServerMetric(namespace, "health_checks_fails", "Failed health checks", streamUpstreamServerVariableLabelNames, constLabels),
-			"health_checks_unhealthy": newStreamUpstreamServerMetric(namespace, "health_checks_unhealthy", "How many times the server became unhealthy (state 'unhealthy')", constLabels),
+			"health_checks_unhealthy": newStreamUpstreamServerMetric(namespace, "health_checks_unhealthy", "How many times the server became unhealthy (state 'unhealthy')", streamUpstreamServerVariableLabelNames, constLabels),
 		},
 		streamZoneSyncMetrics: map[string]*prometheus.Desc{
 			"bytes_in":        newStreamZoneSyncMetric(namespace, "bytes_in", "Bytes received by this node", constLabels),
@@ -359,22 +445,34 @@ func (c *NginxPlusCollector) Collect(ch chan<- prometheus.Metric) {
 	}
 
 	for name, zone := range stats.StreamServerZones {
+		labelValues := []string{name}
+		varLabelValues := c.getStreamServerZoneLabelValues(name)
+
+		if c.variableLabelNames.StreamServerZoneVariableLabelNames != nil && len(varLabelValues) != len(c.variableLabelNames.StreamServerZoneVariableLabelNames) {
+			log.Printf("wrong number of labels for stream server zone %v. For labels %v, got values: %v. Empty labels will be used instead",
+				name, c.variableLabelNames.StreamServerZoneVariableLabelNames, varLabelValues)
+			for range c.variableLabelNames.StreamServerZoneVariableLabelNames {
+				labelValues = append(labelValues, "")
+			}
+		} else {
+			labelValues = append(labelValues, varLabelValues...)
+		}
 		ch <- prometheus.MustNewConstMetric(c.streamServerZoneMetrics["processing"],
-			prometheus.GaugeValue, float64(zone.Processing), name)
+			prometheus.GaugeValue, float64(zone.Processing), labelValues...)
 		ch <- prometheus.MustNewConstMetric(c.streamServerZoneMetrics["connections"],
-			prometheus.CounterValue, float64(zone.Connections), name)
+			prometheus.CounterValue, float64(zone.Connections), labelValues...)
 		ch <- prometheus.MustNewConstMetric(c.streamServerZoneMetrics["sessions_2xx"],
-			prometheus.CounterValue, float64(zone.Sessions.Sessions2xx), name)
+			prometheus.CounterValue, float64(zone.Sessions.Sessions2xx), labelValues...)
 		ch <- prometheus.MustNewConstMetric(c.streamServerZoneMetrics["sessions_4xx"],
-			prometheus.CounterValue, float64(zone.Sessions.Sessions4xx), name)
+			prometheus.CounterValue, float64(zone.Sessions.Sessions4xx), labelValues...)
 		ch <- prometheus.MustNewConstMetric(c.streamServerZoneMetrics["sessions_5xx"],
-			prometheus.CounterValue, float64(zone.Sessions.Sessions5xx), name)
+			prometheus.CounterValue, float64(zone.Sessions.Sessions5xx), labelValues...)
 		ch <- prometheus.MustNewConstMetric(c.streamServerZoneMetrics["discarded"],
-			prometheus.CounterValue, float64(zone.Discarded), name)
+			prometheus.CounterValue, float64(zone.Discarded), labelValues...)
 		ch <- prometheus.MustNewConstMetric(c.streamServerZoneMetrics["received"],
-			prometheus.CounterValue, float64(zone.Received), name)
+			prometheus.CounterValue, float64(zone.Received), labelValues...)
 		ch <- prometheus.MustNewConstMetric(c.streamServerZoneMetrics["sent"],
-			prometheus.CounterValue, float64(zone.Sent), name)
+			prometheus.CounterValue, float64(zone.Sent), labelValues...)
 	}
 
 	for name, upstream := range stats.Upstreams {
@@ -450,33 +548,58 @@ func (c *NginxPlusCollector) Collect(ch chan<- prometheus.Metric) {
 
 	for name, upstream := range stats.StreamUpstreams {
 		for _, peer := range upstream.Peers {
+			labelValues := []string{name, peer.Server}
+			varLabelValues := c.getStreamUpstreamServerLabelValues(name)
+
+			if c.variableLabelNames.StreamUpstreamServerVariableLabelNames != nil && len(varLabelValues) != len(c.variableLabelNames.StreamUpstreamServerVariableLabelNames) {
+				log.Printf("wrong number of labels for stream server %v. For labels %v, got values: %v. Empty labels will be used instead",
+					name, c.variableLabelNames.StreamUpstreamServerVariableLabelNames, varLabelValues)
+				for range c.variableLabelNames.StreamUpstreamServerVariableLabelNames {
+					labelValues = append(labelValues, "")
+				}
+			} else {
+				labelValues = append(labelValues, varLabelValues...)
+			}
+
+			upstreamServer := fmt.Sprintf("%v/%v", name, peer.Server)
+			varPeerLabelValues := c.getStreamUpstreamServerPeerLabelValues(upstreamServer)
+			if c.variableLabelNames.StreamUpstreamServerPeerVariableLabelNames != nil && len(varPeerLabelValues) != len(c.variableLabelNames.StreamUpstreamServerPeerVariableLabelNames) {
+				log.Printf("wrong number of labels for stream upstream peer %v. For labels %v, got values: %v. Empty labels will be used instead",
+					peer.Server, c.variableLabelNames.StreamUpstreamServerPeerVariableLabelNames, varPeerLabelValues)
+				for range c.variableLabelNames.StreamUpstreamServerPeerVariableLabelNames {
+					labelValues = append(labelValues, "")
+				}
+			} else {
+				labelValues = append(labelValues, varPeerLabelValues...)
+			}
+
 			ch <- prometheus.MustNewConstMetric(c.streamUpstreamServerMetrics["state"],
-				prometheus.GaugeValue, upstreamServerStates[peer.State], name, peer.Server)
+				prometheus.GaugeValue, upstreamServerStates[peer.State], labelValues...)
 			ch <- prometheus.MustNewConstMetric(c.streamUpstreamServerMetrics["active"],
-				prometheus.GaugeValue, float64(peer.Active), name, peer.Server)
+				prometheus.GaugeValue, float64(peer.Active), labelValues...)
 			ch <- prometheus.MustNewConstMetric(c.streamUpstreamServerMetrics["connections"],
-				prometheus.CounterValue, float64(peer.Connections), name, peer.Server)
+				prometheus.CounterValue, float64(peer.Connections), labelValues...)
 			ch <- prometheus.MustNewConstMetric(c.streamUpstreamServerMetrics["connect_time"],
-				prometheus.GaugeValue, float64(peer.ConnectTime), name, peer.Server)
+				prometheus.GaugeValue, float64(peer.ConnectTime), labelValues...)
 			ch <- prometheus.MustNewConstMetric(c.streamUpstreamServerMetrics["first_byte_time"],
-				prometheus.GaugeValue, float64(peer.FirstByteTime), name, peer.Server)
+				prometheus.GaugeValue, float64(peer.FirstByteTime), labelValues...)
 			ch <- prometheus.MustNewConstMetric(c.streamUpstreamServerMetrics["response_time"],
-				prometheus.GaugeValue, float64(peer.ResponseTime), name, peer.Server)
+				prometheus.GaugeValue, float64(peer.ResponseTime), labelValues...)
 			ch <- prometheus.MustNewConstMetric(c.streamUpstreamServerMetrics["sent"],
-				prometheus.CounterValue, float64(peer.Sent), name, peer.Server)
+				prometheus.CounterValue, float64(peer.Sent), labelValues...)
 			ch <- prometheus.MustNewConstMetric(c.streamUpstreamServerMetrics["received"],
-				prometheus.CounterValue, float64(peer.Received), name, peer.Server)
+				prometheus.CounterValue, float64(peer.Received), labelValues...)
 			ch <- prometheus.MustNewConstMetric(c.streamUpstreamServerMetrics["fails"],
-				prometheus.CounterValue, float64(peer.Fails), name, peer.Server)
+				prometheus.CounterValue, float64(peer.Fails), labelValues...)
 			ch <- prometheus.MustNewConstMetric(c.streamUpstreamServerMetrics["unavail"],
-				prometheus.CounterValue, float64(peer.Unavail), name, peer.Server)
+				prometheus.CounterValue, float64(peer.Unavail), labelValues...)
 			if peer.HealthChecks != (plusclient.HealthChecks{}) {
 				ch <- prometheus.MustNewConstMetric(c.streamUpstreamServerMetrics["health_checks_checks"],
-					prometheus.CounterValue, float64(peer.HealthChecks.Checks), name, peer.Server)
+					prometheus.CounterValue, float64(peer.HealthChecks.Checks), labelValues...)
 				ch <- prometheus.MustNewConstMetric(c.streamUpstreamServerMetrics["health_checks_fails"],
-					prometheus.CounterValue, float64(peer.HealthChecks.Fails), name, peer.Server)
+					prometheus.CounterValue, float64(peer.HealthChecks.Fails), labelValues...)
 				ch <- prometheus.MustNewConstMetric(c.streamUpstreamServerMetrics["health_checks_unhealthy"],
-					prometheus.CounterValue, float64(peer.HealthChecks.Unhealthy), name, peer.Server)
+					prometheus.CounterValue, float64(peer.HealthChecks.Unhealthy), labelValues...)
 			}
 		}
 		ch <- prometheus.MustNewConstMetric(c.streamUpstreamMetrics["zombies"],
@@ -565,8 +688,10 @@ func newServerZoneMetric(namespace string, metricName string, docString string,
 	return prometheus.NewDesc(prometheus.BuildFQName(namespace, "server_zone", metricName), docString, labels, constLabels)
 }
 
-func newStreamServerZoneMetric(namespace string, metricName string, docString string, constLabels prometheus.Labels) *prometheus.Desc {
+func newStreamServerZoneMetric(namespace string, metricName string, docString string, variableLabelNames []string, constLabels prometheus.Labels) *prometheus.Desc {
-	return prometheus.NewDesc(prometheus.BuildFQName(namespace, "stream_server_zone", metricName), docString, []string{"server_zone"}, constLabels)
+	labels := []string{"server_zone"}
+	labels = append(labels, variableLabelNames...)
+	return prometheus.NewDesc(prometheus.BuildFQName(namespace, "stream_server_zone", metricName), docString, labels, constLabels)
 }
 
 func newUpstreamMetric(namespace string, metricName string, docString string, constLabels prometheus.Labels) *prometheus.Desc {
@@ -583,8 +708,10 @@ func newUpstreamServerMetric(namespace string, metricName string, docString stri
 	return prometheus.NewDesc(prometheus.BuildFQName(namespace, "upstream_server", metricName), docString, labels, constLabels)
 }
 
-func newStreamUpstreamServerMetric(namespace string, metricName string, docString string, constLabels prometheus.Labels) *prometheus.Desc {
+func newStreamUpstreamServerMetric(namespace string, metricName string, docString string, variableLabelNames []string, constLabels prometheus.Labels) *prometheus.Desc {
-	return prometheus.NewDesc(prometheus.BuildFQName(namespace, "stream_upstream_server", metricName), docString, []string{"upstream", "server"}, constLabels)
+	labels := []string{"upstream", "server"}
+	labels = append(labels, variableLabelNames...)
+	return prometheus.NewDesc(prometheus.BuildFQName(namespace, "stream_upstream_server", metricName), docString, labels, constLabels)
 }
 
 func newStreamZoneSyncMetric(namespace string, metricName string, docString string, constLabels prometheus.Labels) *prometheus.Desc {

exporter.go

@@ -380,7 +380,7 @@ func main() {
 		if err != nil {
 			log.Fatalf("Could not create Nginx Plus Client: %v", err)
 		}
-		variableLabelNames := collector.NewVariableLabelNames(nil, nil, nil)
+		variableLabelNames := collector.NewVariableLabelNames(nil, nil, nil, nil)
 		registry.MustRegister(collector.NewNginxPlusCollector(plusClient.(*plusclient.NginxClient), "nginxplus", variableLabelNames, constLabels.labels))
 	} else {
 		ossClient, err := createClientWithRetries(func() (interface{}, error) {