Merge pull request #73 from bnema/fix/initial-metrics-values

fix: initialize ManagedContainers metric at startup

commit: 024bbc1479c73e07a544d7de45358c2e58198c81
author: bnema · 8mo ago · 2026-02-11 20:47 UTC
committer: GitHub · 8mo ago · 2026-02-11 20:47 UTC
merge of: 65db0d9 + 8112fbd · diff against the first parent
signature: GPG UnverifiedB5690EEEBB952194 The key is unknown

3 files · +174 -7

M docs/config/telemetry.md +1 -1


@@ -100,7 +100,7 @@ Attributes: `domain`, `image`
100	100		\| `gordon.container.crash_loops` \| Counter \| - \| Crash loop detections \|
101	101		\| `gordon.container.managed` \| UpDownCounter \| - \| Currently tracked containers \|
102	102
103		-	Attributes: `domain`, `source` (restarts only: `monitor` or `api`)
	103	+	Attributes: `source` (restarts only: `monitor` or `api`); `gordon.container.managed` is a global gauge with no attributes
104	104
105	105		### Registry
106	106


@@ -100,7 +100,7 @@ Attributes: `domain`, `image`
100		\| `gordon.container.crash_loops` \| Counter \| - \| Crash loop detections \|	100		\| `gordon.container.crash_loops` \| Counter \| - \| Crash loop detections \|
101		\| `gordon.container.managed` \| UpDownCounter \| - \| Currently tracked containers \|	101		\| `gordon.container.managed` \| UpDownCounter \| - \| Currently tracked containers \|
102			102
103	-	Attributes: `domain`, `source` (restarts only: `monitor` or `api`)	103	+	Attributes: `source` (restarts only: `monitor` or `api`); `gordon.container.managed` is a global gauge with no attributes
104			104
105		### Registry	105		### Registry
106			106

M internal/usecase/container/service.go +15 -6


@@ -76,6 +76,7 @@ type Service struct {
76	76		metrics *telemetry.Metrics
77	77		containers map[string]*domain.Container
78	78		attachments map[string][]string // ownerDomain → []containerIDs
	79	+	managedCount int64 // tracks UpDownCounter value for delta computation
79	80		mu sync.RWMutex
80	81		deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)
81	82		monitor *Monitor

@@ -456,13 +457,14 @@ func (s *Service) activateDeployedContainer(ctx context.Context, domainName stri
456	457		s.mu.Lock()
457	458		_, wasTracked := s.containers[domainName]
458	459		s.containers[domainName] = container
	460	+	if !wasTracked {
	461	+	s.managedCount++
	462	+	}
459	463		s.mu.Unlock()
460	464
461	465		// Track managed container count (only increment for new domains, not replacements)
462	466		if !wasTracked && s.metrics != nil {
463		-	s.metrics.ManagedContainers.Add(ctx, 1, metric.WithAttributes(
464		-	attribute.String("domain", domainName),
465		-	))
	467	+	s.metrics.ManagedContainers.Add(ctx, 1)
466	468		}
467	469
468	470		s.publishContainerDeployed(ctx, domainName, container.ID)

@@ -679,6 +681,7 @@ func (s *Service) Remove(ctx context.Context, containerID string, force bool) er
679	681		if c.ID == containerID {
680	682		delete(s.containers, d)
681	683		removedDomain = d
	684	+	s.managedCount--
682	685		log.Info().Str("domain", d).Msg("container removed")
683	686		break
684	687		}

@@ -690,9 +693,7 @@ func (s *Service) Remove(ctx context.Context, containerID string, force bool) er
690	693
691	694		// Decrement managed container count
692	695		if removedDomain != "" && s.metrics != nil {
693		-	s.metrics.ManagedContainers.Add(ctx, -1, metric.WithAttributes(
694		-	attribute.String("domain", removedDomain),
695		-	))
	696	+	s.metrics.ManagedContainers.Add(ctx, -1)
696	697		}
697	698
698	699		// Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:

@@ -912,8 +913,16 @@ func (s *Service) SyncContainers(ctx context.Context) error {
912	913
913	914		s.mu.Lock()
914	915		s.containers = managed
	916	+	newCount := int64(len(managed))
	917	+	delta := newCount - s.managedCount
	918	+	s.managedCount = newCount
915	919		s.mu.Unlock()
916	920
	921	+	// Report initial/delta managed container count to OTel UpDownCounter.
	922	+	if delta != 0 && s.metrics != nil {
	923	+	s.metrics.ManagedContainers.Add(ctx, delta)
	924	+	}
	925	+
917	926		log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
918	927		return nil
919	928		}

 	metrics          *telemetry.Metrics
 	containers       map[string]*domain.Container
 	attachments      map[string][]string // ownerDomain → []containerIDs
+	managedCount     int64               // tracks UpDownCounter value for delta computation
 	mu               sync.RWMutex
 	deployMu         sync.Map // per-domain deploy locks (domain → *domainDeployLock)
 	monitor          *Monitor
 	s.mu.Lock()
 	_, wasTracked := s.containers[domainName]
 	s.containers[domainName] = container
+	if !wasTracked {
+		s.managedCount++
+	}
 	s.mu.Unlock()
 	// Track managed container count (only increment for new domains, not replacements)
 	if !wasTracked && s.metrics != nil {
-		s.metrics.ManagedContainers.Add(ctx, 1, metric.WithAttributes(
+		s.metrics.ManagedContainers.Add(ctx, 1)
-			attribute.String("domain", domainName),
-		))
 	}
 	s.publishContainerDeployed(ctx, domainName, container.ID)
 		if c.ID == containerID {
 			delete(s.containers, d)
 			removedDomain = d
+			s.managedCount--
 			log.Info().Str("domain", d).Msg("container removed")
 			break
 		}
 	// Decrement managed container count
 	if removedDomain != "" && s.metrics != nil {
-		s.metrics.ManagedContainers.Add(ctx, -1, metric.WithAttributes(
+		s.metrics.ManagedContainers.Add(ctx, -1)
-			attribute.String("domain", removedDomain),
-		))
 	}
 	// Note: We intentionally do NOT clean up deployMu entries to avoid a race condition:
 	s.mu.Lock()
 	s.containers = managed
+	newCount := int64(len(managed))
+	delta := newCount - s.managedCount
+	s.managedCount = newCount
 	s.mu.Unlock()
+	// Report initial/delta managed container count to OTel UpDownCounter.
+	if delta != 0 && s.metrics != nil {
+		s.metrics.ManagedContainers.Add(ctx, delta)
+	}
 	log.Info().Int(zerowrap.FieldCount, len(managed)).Msg("container state synchronized")
 	return nil
 }

M internal/usecase/container/service_test.go +158 -0


@@ -11,7 +11,12 @@ import (
11	11		"github.com/bnema/zerowrap"
12	12		"github.com/stretchr/testify/assert"
13	13		"github.com/stretchr/testify/mock"
	14	+	"github.com/stretchr/testify/require"
	15	+	"go.opentelemetry.io/otel"
	16	+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
	17	+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
14	18
	19	+	"github.com/bnema/gordon/internal/adapters/out/telemetry"
15	20		"github.com/bnema/gordon/internal/boundaries/out/mocks"
16	21		"github.com/bnema/gordon/internal/domain"
17	22		)

@@ -20,6 +25,159 @@ func testContext() context.Context {
20	25		return zerowrap.WithCtx(context.Background(), zerowrap.Default())
21	26		}
22	27
	28	+	func setupMetricsTest(t testing.T) (telemetry.Metrics, *sdkmetric.ManualReader) {
	29	+	t.Helper()
	30	+
	31	+	prev := otel.GetMeterProvider()
	32	+	t.Cleanup(func() {
	33	+	otel.SetMeterProvider(prev)
	34	+	})
	35	+
	36	+	reader := sdkmetric.NewManualReader()
	37	+	mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))
	38	+	otel.SetMeterProvider(mp)
	39	+	t.Cleanup(func() {
	40	+	_ = mp.Shutdown(context.Background())
	41	+	})
	42	+
	43	+	m, err := telemetry.NewMetrics()
	44	+	require.NoError(t, err)
	45	+	return m, reader
	46	+	}
	47	+
	48	+	func managedMetricState(t testing.T, reader sdkmetric.ManualReader) (int64, int, []metricdata.DataPoint[int64]) {
	49	+	t.Helper()
	50	+
	51	+	var rm metricdata.ResourceMetrics
	52	+	require.NoError(t, reader.Collect(context.Background(), &rm))
	53	+
	54	+	for _, sm := range rm.ScopeMetrics {
	55	+	for _, m := range sm.Metrics {
	56	+	if m.Name != "gordon.container.managed" {
	57	+	continue
	58	+	}
	59	+	sum, ok := m.Data.(metricdata.Sum[int64])
	60	+	require.True(t, ok)
	61	+	var total int64
	62	+	for _, dp := range sum.DataPoints {
	63	+	total += dp.Value
	64	+	}
	65	+	return total, len(sum.DataPoints), sum.DataPoints
	66	+	}
	67	+	}
	68	+
	69	+	return 0, 0, nil
	70	+	}
	71	+
	72	+	func TestService_ManagedContainersMetric_GlobalSeriesOnDeployReplaceAndRemove(t *testing.T) {
	73	+	runtime := mocks.NewMockContainerRuntime(t)
	74	+	envLoader := mocks.NewMockEnvLoader(t)
	75	+	eventBus := mocks.NewMockEventPublisher(t)
	76	+
	77	+	svc := NewService(runtime, envLoader, eventBus, nil, Config{})
	78	+	ctx := testContext()
	79	+	metrics, reader := setupMetricsTest(t)
	80	+	svc.SetMetrics(metrics)
	81	+	eventBus.EXPECT().Publish(domain.EventContainerDeployed, mock.AnythingOfType("*domain.ContainerEventPayload")).Return(nil).Twice()
	82	+
	83	+	// First deploy increments metric.
	84	+	svc.activateDeployedContainer(ctx, "test.example.com", &domain.Container{ID: "container-1"})
	85	+	value, seriesCount, points := managedMetricState(t, reader)
	86	+	assert.Equal(t, int64(1), value)
	87	+	assert.Equal(t, 1, seriesCount)
	88	+	assert.Equal(t, 0, points[0].Attributes.Len())
	89	+
	90	+	// Replacing an existing domain should not increment.
	91	+	svc.activateDeployedContainer(ctx, "test.example.com", &domain.Container{ID: "container-2"})
	92	+	value, seriesCount, points = managedMetricState(t, reader)
	93	+	assert.Equal(t, int64(1), value)
	94	+	assert.Equal(t, 1, seriesCount)
	95	+	assert.Equal(t, 0, points[0].Attributes.Len())
	96	+
	97	+	runtime.EXPECT().RemoveContainer(mock.Anything, "container-2", true).Return(nil)
	98	+	require.NoError(t, svc.Remove(ctx, "container-2", true))
	99	+
	100	+	value, seriesCount, points = managedMetricState(t, reader)
	101	+	assert.Equal(t, int64(0), value)
	102	+	assert.Equal(t, 1, seriesCount)
	103	+	assert.Equal(t, 0, points[0].Attributes.Len())
	104	+	}
	105	+
	106	+	func TestService_SyncContainers_ManagedContainersMetricTracksDelta(t *testing.T) {
	107	+	runtime := mocks.NewMockContainerRuntime(t)
	108	+	envLoader := mocks.NewMockEnvLoader(t)
	109	+	eventBus := mocks.NewMockEventPublisher(t)
	110	+
	111	+	svc := NewService(runtime, envLoader, eventBus, nil, Config{})
	112	+	ctx := testContext()
	113	+	metrics, reader := setupMetricsTest(t)
	114	+	svc.SetMetrics(metrics)
	115	+
	116	+	runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{
	117	+	{
	118	+	ID: "container-1",
	119	+	Labels: map[string]string{
	120	+	"gordon.domain": "app1.example.com",
	121	+	"gordon.managed": "true",
	122	+	},
	123	+	},
	124	+	{
	125	+	ID: "container-2",
	126	+	Labels: map[string]string{
	127	+	"gordon.domain": "app2.example.com",
	128	+	"gordon.managed": "true",
	129	+	},
	130	+	},
	131	+	}, nil).Once()
	132	+	require.NoError(t, svc.SyncContainers(ctx))
	133	+
	134	+	value, seriesCount, points := managedMetricState(t, reader)
	135	+	assert.Equal(t, int64(2), value)
	136	+	assert.Equal(t, 1, seriesCount)
	137	+	assert.Equal(t, 0, points[0].Attributes.Len())
	138	+
	139	+	// No runtime change: metric should remain stable.
	140	+	runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{
	141	+	{
	142	+	ID: "container-1",
	143	+	Labels: map[string]string{
	144	+	"gordon.domain": "app1.example.com",
	145	+	"gordon.managed": "true",
	146	+	},
	147	+	},
	148	+	{
	149	+	ID: "container-2",
	150	+	Labels: map[string]string{
	151	+	"gordon.domain": "app2.example.com",
	152	+	"gordon.managed": "true",
	153	+	},
	154	+	},
	155	+	}, nil).Once()
	156	+	require.NoError(t, svc.SyncContainers(ctx))
	157	+
	158	+	value, seriesCount, points = managedMetricState(t, reader)
	159	+	assert.Equal(t, int64(2), value)
	160	+	assert.Equal(t, 1, seriesCount)
	161	+	assert.Equal(t, 0, points[0].Attributes.Len())
	162	+
	163	+	// One container removed in runtime: metric should decrement by one.
	164	+	runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{
	165	+	{
	166	+	ID: "container-1",
	167	+	Labels: map[string]string{
	168	+	"gordon.domain": "app1.example.com",
	169	+	"gordon.managed": "true",
	170	+	},
	171	+	},
	172	+	}, nil).Once()
	173	+	require.NoError(t, svc.SyncContainers(ctx))
	174	+
	175	+	value, seriesCount, points = managedMetricState(t, reader)
	176	+	assert.Equal(t, int64(1), value)
	177	+	assert.Equal(t, 1, seriesCount)
	178	+	assert.Equal(t, 0, points[0].Attributes.Len())
	179	+	}
	180	+
23	181		func TestService_Deploy_Success(t *testing.T) {
24	182		runtime := mocks.NewMockContainerRuntime(t)
25	183		envLoader := mocks.NewMockEnvLoader(t)


@@ -11,7 +11,12 @@ import (
11	"github.com/bnema/zerowrap"	11		"github.com/bnema/zerowrap"
12	"github.com/stretchr/testify/assert"	12		"github.com/stretchr/testify/assert"
13	"github.com/stretchr/testify/mock"	13		"github.com/stretchr/testify/mock"
		14	+	"github.com/stretchr/testify/require"
		15	+	"go.opentelemetry.io/otel"
		16	+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
		17	+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
14		18
		19	+	"github.com/bnema/gordon/internal/adapters/out/telemetry"
15	"github.com/bnema/gordon/internal/boundaries/out/mocks"	20		"github.com/bnema/gordon/internal/boundaries/out/mocks"
16	"github.com/bnema/gordon/internal/domain"	21		"github.com/bnema/gordon/internal/domain"
17	)	22		)

@@ -20,6 +25,159 @@ func testContext() context.Context {
20	return zerowrap.WithCtx(context.Background(), zerowrap.Default())	25		return zerowrap.WithCtx(context.Background(), zerowrap.Default())
21	}	26		}
22		27
		28	+	func setupMetricsTest(t testing.T) (telemetry.Metrics, *sdkmetric.ManualReader) {
		29	+	t.Helper()
		30	+
		31	+	prev := otel.GetMeterProvider()
		32	+	t.Cleanup(func() {
		33	+	otel.SetMeterProvider(prev)
		34	+	})
		35	+
		36	+	reader := sdkmetric.NewManualReader()
		37	+	mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))
		38	+	otel.SetMeterProvider(mp)
		39	+	t.Cleanup(func() {
		40	+	_ = mp.Shutdown(context.Background())
		41	+	})
		42	+
		43	+	m, err := telemetry.NewMetrics()
		44	+	require.NoError(t, err)
		45	+	return m, reader
		46	+	}
		47	+
		48	+	func managedMetricState(t testing.T, reader sdkmetric.ManualReader) (int64, int, []metricdata.DataPoint[int64]) {
		49	+	t.Helper()
		50	+
		51	+	var rm metricdata.ResourceMetrics
		52	+	require.NoError(t, reader.Collect(context.Background(), &rm))
		53	+
		54	+	for _, sm := range rm.ScopeMetrics {
		55	+	for _, m := range sm.Metrics {
		56	+	if m.Name != "gordon.container.managed" {
		57	+	continue
		58	+	}
		59	+	sum, ok := m.Data.(metricdata.Sum[int64])
		60	+	require.True(t, ok)
		61	+	var total int64
		62	+	for _, dp := range sum.DataPoints {
		63	+	total += dp.Value
		64	+	}
		65	+	return total, len(sum.DataPoints), sum.DataPoints
		66	+	}
		67	+	}
		68	+
		69	+	return 0, 0, nil
		70	+	}
		71	+
		72	+	func TestService_ManagedContainersMetric_GlobalSeriesOnDeployReplaceAndRemove(t *testing.T) {
		73	+	runtime := mocks.NewMockContainerRuntime(t)
		74	+	envLoader := mocks.NewMockEnvLoader(t)
		75	+	eventBus := mocks.NewMockEventPublisher(t)
		76	+
		77	+	svc := NewService(runtime, envLoader, eventBus, nil, Config{})
		78	+	ctx := testContext()
		79	+	metrics, reader := setupMetricsTest(t)
		80	+	svc.SetMetrics(metrics)
		81	+	eventBus.EXPECT().Publish(domain.EventContainerDeployed, mock.AnythingOfType("*domain.ContainerEventPayload")).Return(nil).Twice()
		82	+
		83	+	// First deploy increments metric.
		84	+	svc.activateDeployedContainer(ctx, "test.example.com", &domain.Container{ID: "container-1"})
		85	+	value, seriesCount, points := managedMetricState(t, reader)
		86	+	assert.Equal(t, int64(1), value)
		87	+	assert.Equal(t, 1, seriesCount)
		88	+	assert.Equal(t, 0, points[0].Attributes.Len())
		89	+
		90	+	// Replacing an existing domain should not increment.
		91	+	svc.activateDeployedContainer(ctx, "test.example.com", &domain.Container{ID: "container-2"})
		92	+	value, seriesCount, points = managedMetricState(t, reader)
		93	+	assert.Equal(t, int64(1), value)
		94	+	assert.Equal(t, 1, seriesCount)
		95	+	assert.Equal(t, 0, points[0].Attributes.Len())
		96	+
		97	+	runtime.EXPECT().RemoveContainer(mock.Anything, "container-2", true).Return(nil)
		98	+	require.NoError(t, svc.Remove(ctx, "container-2", true))
		99	+
		100	+	value, seriesCount, points = managedMetricState(t, reader)
		101	+	assert.Equal(t, int64(0), value)
		102	+	assert.Equal(t, 1, seriesCount)
		103	+	assert.Equal(t, 0, points[0].Attributes.Len())
		104	+	}
		105	+
		106	+	func TestService_SyncContainers_ManagedContainersMetricTracksDelta(t *testing.T) {
		107	+	runtime := mocks.NewMockContainerRuntime(t)
		108	+	envLoader := mocks.NewMockEnvLoader(t)
		109	+	eventBus := mocks.NewMockEventPublisher(t)
		110	+
		111	+	svc := NewService(runtime, envLoader, eventBus, nil, Config{})
		112	+	ctx := testContext()
		113	+	metrics, reader := setupMetricsTest(t)
		114	+	svc.SetMetrics(metrics)
		115	+
		116	+	runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{
		117	+	{
		118	+	ID: "container-1",
		119	+	Labels: map[string]string{
		120	+	"gordon.domain": "app1.example.com",
		121	+	"gordon.managed": "true",
		122	+	},
		123	+	},
		124	+	{
		125	+	ID: "container-2",
		126	+	Labels: map[string]string{
		127	+	"gordon.domain": "app2.example.com",
		128	+	"gordon.managed": "true",
		129	+	},
		130	+	},
		131	+	}, nil).Once()
		132	+	require.NoError(t, svc.SyncContainers(ctx))
		133	+
		134	+	value, seriesCount, points := managedMetricState(t, reader)
		135	+	assert.Equal(t, int64(2), value)
		136	+	assert.Equal(t, 1, seriesCount)
		137	+	assert.Equal(t, 0, points[0].Attributes.Len())
		138	+
		139	+	// No runtime change: metric should remain stable.
		140	+	runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{
		141	+	{
		142	+	ID: "container-1",
		143	+	Labels: map[string]string{
		144	+	"gordon.domain": "app1.example.com",
		145	+	"gordon.managed": "true",
		146	+	},
		147	+	},
		148	+	{
		149	+	ID: "container-2",
		150	+	Labels: map[string]string{
		151	+	"gordon.domain": "app2.example.com",
		152	+	"gordon.managed": "true",
		153	+	},
		154	+	},
		155	+	}, nil).Once()
		156	+	require.NoError(t, svc.SyncContainers(ctx))
		157	+
		158	+	value, seriesCount, points = managedMetricState(t, reader)
		159	+	assert.Equal(t, int64(2), value)
		160	+	assert.Equal(t, 1, seriesCount)
		161	+	assert.Equal(t, 0, points[0].Attributes.Len())
		162	+
		163	+	// One container removed in runtime: metric should decrement by one.
		164	+	runtime.EXPECT().ListContainers(mock.Anything, false).Return([]*domain.Container{
		165	+	{
		166	+	ID: "container-1",
		167	+	Labels: map[string]string{
		168	+	"gordon.domain": "app1.example.com",
		169	+	"gordon.managed": "true",
		170	+	},
		171	+	},
		172	+	}, nil).Once()
		173	+	require.NoError(t, svc.SyncContainers(ctx))
		174	+
		175	+	value, seriesCount, points = managedMetricState(t, reader)
		176	+	assert.Equal(t, int64(1), value)
		177	+	assert.Equal(t, 1, seriesCount)
		178	+	assert.Equal(t, 0, points[0].Attributes.Len())
		179	+	}
		180	+
23	func TestService_Deploy_Success(t *testing.T) {	181		func TestService_Deploy_Success(t *testing.T) {
24	runtime := mocks.NewMockContainerRuntime(t)	182		runtime := mocks.NewMockContainerRuntime(t)
25	envLoader := mocks.NewMockEnvLoader(t)	183		envLoader := mocks.NewMockEnvLoader(t)