feat(backup): export OTel metrics for database and volume backups

Record every backup run (result, duration, size) and expose the last success per target plus the next scheduled run as gauges. The last success is seeded from stored dumps and S3 archives at startup so it survives restarts. Docs list the metrics and example alerts.

commit: 39593e0effdc96037af4725e5d8ae2d6763334b6
author: brice · 5d ago · 2026-10-17 06:51 UTC
parent: 108f11b
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

15 files · +747 -2

M docs/config/backups.md +5 -0


@@ -105,6 +105,10 @@ Objects written by earlier versions under a domain prefix are not read or
105	105		migrated: back them up and re-run the app's volume backups if you need them in
106	106		the current layout.
107	107
	108	+	## Monitoring
	109	+
	110	+	With [telemetry](./telemetry.md) metrics enabled, Gordon exports the result, duration, and size of every backup run, the last success of each target, and the next scheduled run. See [Backups metrics](./telemetry.md#backups) for the metric names and example alerts.
	111	+
108	112		## Notes
109	113
110	114		- Volume backups archive the app's declared volumes that its backup declaration references.

@@ -122,3 +126,4 @@ the current layout.
122	126		- [App Manifest](./apps.md)
123	127		- [CLI Backup Command](../cli/backup.md)
124	128		- [Configuration Reference](./reference.md)
	129	+	- [Telemetry](./telemetry.md)


@@ -105,6 +105,10 @@ Objects written by earlier versions under a domain prefix are not read or
105	migrated: back them up and re-run the app's volume backups if you need them in	105		migrated: back them up and re-run the app's volume backups if you need them in
106	the current layout.	106		the current layout.
107		107
		108	+	## Monitoring
		109	+
		110	+	With [telemetry](./telemetry.md) metrics enabled, Gordon exports the result, duration, and size of every backup run, the last success of each target, and the next scheduled run. See [Backups metrics](./telemetry.md#backups) for the metric names and example alerts.
		111	+
108	## Notes	112		## Notes
109		113
110	- Volume backups archive the app's declared volumes that its backup declaration references.	114		- Volume backups archive the app's declared volumes that its backup declaration references.

@@ -122,3 +126,4 @@ the current layout.
122	- [App Manifest](./apps.md)	126		- [App Manifest](./apps.md)
123	- [CLI Backup Command](../cli/backup.md)	127		- [CLI Backup Command](../cli/backup.md)
124	- [Configuration Reference](./reference.md)	128		- [Configuration Reference](./reference.md)
		129	+	- [Telemetry](./telemetry.md)

M docs/config/telemetry.md +25 -1


@@ -25,7 +25,7 @@ trace_sample_rate = 1.0
25	25		\| `endpoint` \| string \| `""` \| OTLP HTTP endpoint URL (e.g. `http://localhost:4318/api/default`) \|
26	26		\| `auth_token` \| string \| `""` \| Base64-encoded `user:password` for Basic auth \|
27	27		\| `traces` \| bool \| `true` \| Export distributed traces \|
28		-	\| `metrics` \| bool \| `true` \| Export metrics (deploy counters, container lifecycle, registry, events) \|
	28	+	\| `metrics` \| bool \| `true` \| Export metrics (deploy counters, container lifecycle, registry, events, backups) \|
29	29		\| `logs` \| bool \| `true` \| Export Gordon, proxy access, and app container logs to OTLP \|
30	30		\| `trace_sample_rate` \| float \| `1.0` \| Fraction of traces to sample (`0.0` = none, `1.0` = all) \|
31	31

@@ -160,6 +160,30 @@ Attributes: `source` (restarts only: `monitor` or `api`). `gordon.container.mana
160	160
161	161		Attributes: `name`, `reference`
162	162
	163	+	### Backups
	164	+
	165	+	\| Metric \| Type \| Unit \| Description \|
	166	+	\|--------\|------\|------\|-------------\|
	167	+	\| `gordon.backup.runs` \| Counter \| - \| Finished backup runs (scheduled and manual) \|
	168	+	\| `gordon.backup.duration_seconds` \| Histogram \| s \| Duration of each backup run \|
	169	+	\| `gordon.backup.last_success_timestamp_seconds` \| Gauge \| s \| Unix time of the last successful backup per target \|
	170	+	\| `gordon.backup.last_size_bytes` \| Gauge \| By \| Size of the last successful backup per target \|
	171	+	\| `gordon.backup.next_run_timestamp_seconds` \| Gauge \| s \| Unix time of the next scheduled run \|
	172	+
	173	+	Attributes: `kind` (`database` or `volume`), `app`, `service`, `target` (database name or declared volume name), and `result` (`success` or `failure`, runs only). `next_run_timestamp_seconds` only carries `kind`.
	174	+
	175	+	At startup Gordon reads the stored dumps and S3 archives to restore each target's last success, so the gauges stay correct across restarts. A run missed while Gordon is down never appears as a failure: alert on the age of the last success instead.
	176	+
	177	+	With the OpenTelemetry Collector Prometheus exporter, names become `gordon_backup_runs_total`, `gordon_backup_last_success_timestamp_seconds`, and so on. Example alerts:
	178	+
	179	+	```promql
	180	+	# No successful backup of a target for more than 26 hours
	181	+	time() - gordon_backup_last_success_timestamp_seconds > 26 * 3600
	182	+
	183	+	# Gordon stopped exporting backup metrics
	184	+	absent_over_time(gordon_backup_next_run_timestamp_seconds[15m])
	185	+	```
	186	+
163	187		### Event Bus
164	188
165	189		\| Metric \| Type \| Unit \| Description \|


@@ -25,7 +25,7 @@ trace_sample_rate = 1.0
25		\| `endpoint` \| string \| `""` \| OTLP HTTP endpoint URL (e.g. `http://localhost:4318/api/default`) \|	25		\| `endpoint` \| string \| `""` \| OTLP HTTP endpoint URL (e.g. `http://localhost:4318/api/default`) \|
26		\| `auth_token` \| string \| `""` \| Base64-encoded `user:password` for Basic auth \|	26		\| `auth_token` \| string \| `""` \| Base64-encoded `user:password` for Basic auth \|
27		\| `traces` \| bool \| `true` \| Export distributed traces \|	27		\| `traces` \| bool \| `true` \| Export distributed traces \|
28	-	\| `metrics` \| bool \| `true` \| Export metrics (deploy counters, container lifecycle, registry, events) \|	28	+	\| `metrics` \| bool \| `true` \| Export metrics (deploy counters, container lifecycle, registry, events, backups) \|
29		\| `logs` \| bool \| `true` \| Export Gordon, proxy access, and app container logs to OTLP \|	29		\| `logs` \| bool \| `true` \| Export Gordon, proxy access, and app container logs to OTLP \|
30		\| `trace_sample_rate` \| float \| `1.0` \| Fraction of traces to sample (`0.0` = none, `1.0` = all) \|	30		\| `trace_sample_rate` \| float \| `1.0` \| Fraction of traces to sample (`0.0` = none, `1.0` = all) \|
31			31

@@ -160,6 +160,30 @@ Attributes: `source` (restarts only: `monitor` or `api`). `gordon.container.mana
160			160
161		Attributes: `name`, `reference`	161		Attributes: `name`, `reference`
162			162
			163	+	### Backups
			164	+
			165	+	\| Metric \| Type \| Unit \| Description \|
			166	+	\|--------\|------\|------\|-------------\|
			167	+	\| `gordon.backup.runs` \| Counter \| - \| Finished backup runs (scheduled and manual) \|
			168	+	\| `gordon.backup.duration_seconds` \| Histogram \| s \| Duration of each backup run \|
			169	+	\| `gordon.backup.last_success_timestamp_seconds` \| Gauge \| s \| Unix time of the last successful backup per target \|
			170	+	\| `gordon.backup.last_size_bytes` \| Gauge \| By \| Size of the last successful backup per target \|
			171	+	\| `gordon.backup.next_run_timestamp_seconds` \| Gauge \| s \| Unix time of the next scheduled run \|
			172	+
			173	+	Attributes: `kind` (`database` or `volume`), `app`, `service`, `target` (database name or declared volume name), and `result` (`success` or `failure`, runs only). `next_run_timestamp_seconds` only carries `kind`.
			174	+
			175	+	At startup Gordon reads the stored dumps and S3 archives to restore each target's last success, so the gauges stay correct across restarts. A run missed while Gordon is down never appears as a failure: alert on the age of the last success instead.
			176	+
			177	+	With the OpenTelemetry Collector Prometheus exporter, names become `gordon_backup_runs_total`, `gordon_backup_last_success_timestamp_seconds`, and so on. Example alerts:
			178	+
			179	+	```promql
			180	+	# No successful backup of a target for more than 26 hours
			181	+	time() - gordon_backup_last_success_timestamp_seconds > 26 * 3600
			182	+
			183	+	# Gordon stopped exporting backup metrics
			184	+	absent_over_time(gordon_backup_next_run_timestamp_seconds[15m])
			185	+	```
			186	+
163		### Event Bus	187		### Event Bus
164			188
165		\| Metric \| Type \| Unit \| Description \|	189		\| Metric \| Type \| Unit \| Description \|

A internal/adapters/out/telemetry/backup_metrics.go +145 -0

@@ -0,0 +1,145 @@
	1	+	package telemetry
	2	+
	3	+	import (
	4	+	"context"
	5	+	"sync"
	6	+	"time"
	7	+
	8	+	"go.opentelemetry.io/otel/attribute"
	9	+	"go.opentelemetry.io/otel/metric"
	10	+
	11	+	"github.com/bnema/gordon/internal/boundaries/out"
	12	+	"github.com/bnema/gordon/internal/domain"
	13	+	)
	14	+
	15	+	var _ out.BackupMetrics = (*Metrics)(nil)
	16	+
	17	+	// backupTargetKey identifies one backed-up resource.
	18	+	type backupTargetKey struct {
	19	+	kind domain.BackupKind
	20	+	app string
	21	+	service string
	22	+	target string
	23	+	}
	24	+
	25	+	func (k backupTargetKey) attrs() []attribute.KeyValue {
	26	+	return []attribute.KeyValue{
	27	+	attribute.String("kind", string(k.kind)),
	28	+	attribute.String("app", k.app),
	29	+	attribute.String("service", k.service),
	30	+	attribute.String("target", k.target),
	31	+	}
	32	+	}
	33	+
	34	+	type backupLastSuccess struct {
	35	+	at time.Time
	36	+	size int64
	37	+	}
	38	+
	39	+	// backupState holds the last success per target, exported by observable
	40	+	// gauges at each collection.
	41	+	type backupState struct {
	42	+	mu sync.Mutex
	43	+	last map[backupTargetKey]backupLastSuccess
	44	+	nextRun map[domain.BackupKind]func() time.Time
	45	+	}
	46	+
	47	+	func newBackupState() *backupState {
	48	+	return &backupState{
	49	+	last: map[backupTargetKey]backupLastSuccess{},
	50	+	nextRun: map[domain.BackupKind]func() time.Time{},
	51	+	}
	52	+	}
	53	+
	54	+	func keyOf(run domain.BackupRunRecord) backupTargetKey {
	55	+	return backupTargetKey{kind: run.Kind, app: run.App, service: run.Service, target: run.Target}
	56	+	}
	57	+
	58	+	// remember stores a success unless a newer one is already known.
	59	+	func (s *backupState) remember(run domain.BackupRunRecord) {
	60	+	if !run.Succeeded \|\| run.FinishedAt.IsZero() {
	61	+	return
	62	+	}
	63	+	key := keyOf(run)
	64	+	s.mu.Lock()
	65	+	defer s.mu.Unlock()
	66	+	if prev, ok := s.last[key]; ok && !run.FinishedAt.After(prev.at) {
	67	+	return
	68	+	}
	69	+	s.last[key] = backupLastSuccess{at: run.FinishedAt, size: run.SizeBytes}
	70	+	}
	71	+
	72	+	// registerBackupInstruments creates the backup counters, histogram, and
	73	+	// observable gauges.
	74	+	func (m *Metrics) registerBackupInstruments() error {
	75	+	var err error
	76	+	if m.BackupRuns, err = m.meter.Int64Counter("gordon.backup.runs",
	77	+	metric.WithDescription("Finished backup runs by result")); err != nil {
	78	+	return err
	79	+	}
	80	+	if m.BackupDuration, err = m.meter.Float64Histogram("gordon.backup.duration_seconds",
	81	+	metric.WithDescription("Backup run duration in seconds"),
	82	+	metric.WithUnit("s"),
	83	+	metric.WithExplicitBucketBoundaries(1, 5, 15, 30, 60, 120, 300, 600, 1800, 3600)); err != nil {
	84	+	return err
	85	+	}
	86	+	lastSuccess, err := m.meter.Float64ObservableGauge("gordon.backup.last_success_timestamp_seconds",
	87	+	metric.WithDescription("Unix time of the last successful backup per target"),
	88	+	metric.WithUnit("s"))
	89	+	if err != nil {
	90	+	return err
	91	+	}
	92	+	lastSize, err := m.meter.Float64ObservableGauge("gordon.backup.last_size_bytes",
	93	+	metric.WithDescription("Size of the last successful backup per target"),
	94	+	metric.WithUnit("By"))
	95	+	if err != nil {
	96	+	return err
	97	+	}
	98	+	nextRun, err := m.meter.Float64ObservableGauge("gordon.backup.next_run_timestamp_seconds",
	99	+	metric.WithDescription("Unix time of the next scheduled backup run"),
	100	+	metric.WithUnit("s"))
	101	+	if err != nil {
	102	+	return err
	103	+	}
	104	+	_, err = m.meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
	105	+	m.backups.mu.Lock()
	106	+	defer m.backups.mu.Unlock()
	107	+	for key, last := range m.backups.last {
	108	+	attrs := metric.WithAttributes(key.attrs()...)
	109	+	o.ObserveFloat64(lastSuccess, float64(last.at.Unix()), attrs)
	110	+	o.ObserveFloat64(lastSize, float64(last.size), attrs)
	111	+	}
	112	+	for kind, next := range m.backups.nextRun {
	113	+	if at := next(); !at.IsZero() {
	114	+	o.ObserveFloat64(nextRun, float64(at.Unix()), metric.WithAttributes(attribute.String("kind", string(kind))))
	115	+	}
	116	+	}
	117	+	return nil
	118	+	}, lastSuccess, lastSize, nextRun)
	119	+	return err
	120	+	}
	121	+
	122	+	// RecordBackupRun implements out.BackupMetrics.
	123	+	func (m *Metrics) RecordBackupRun(ctx context.Context, run domain.BackupRunRecord) {
	124	+	result := "failure"
	125	+	if run.Succeeded {
	126	+	result = "success"
	127	+	}
	128	+	attrs := keyOf(run).attrs()
	129	+	m.BackupRuns.Add(ctx, 1, metric.WithAttributes(append(attrs, attribute.String("result", result))...))
	130	+	m.BackupDuration.Record(ctx, run.Duration.Seconds(), metric.WithAttributes(attrs...))
	131	+	m.backups.remember(run)
	132	+	}
	133	+
	134	+	// ObserveStoredBackup implements out.BackupMetrics.
	135	+	func (m *Metrics) ObserveStoredBackup(run domain.BackupRunRecord) {
	136	+	m.backups.remember(run)
	137	+	}
	138	+
	139	+	// ObserveBackupNextRun exports the next planned run of one backup kind.
	140	+	// next is called at each collection; a zero time is not exported.
	141	+	func (m *Metrics) ObserveBackupNextRun(kind domain.BackupKind, next func() time.Time) {
	142	+	m.backups.mu.Lock()
	143	+	defer m.backups.mu.Unlock()
	144	+	m.backups.nextRun[kind] = next
	145	+	}

@@ -0,0 +1,145 @@
	1	+	package telemetry
	2	+
	3	+	import (
	4	+	"context"
	5	+	"sync"
	6	+	"time"
	7	+
	8	+	"go.opentelemetry.io/otel/attribute"
	9	+	"go.opentelemetry.io/otel/metric"
	10	+
	11	+	"github.com/bnema/gordon/internal/boundaries/out"
	12	+	"github.com/bnema/gordon/internal/domain"
	13	+	)
	14	+
	15	+	var _ out.BackupMetrics = (*Metrics)(nil)
	16	+
	17	+	// backupTargetKey identifies one backed-up resource.
	18	+	type backupTargetKey struct {
	19	+	kind domain.BackupKind
	20	+	app string
	21	+	service string
	22	+	target string
	23	+	}
	24	+
	25	+	func (k backupTargetKey) attrs() []attribute.KeyValue {
	26	+	return []attribute.KeyValue{
	27	+	attribute.String("kind", string(k.kind)),
	28	+	attribute.String("app", k.app),
	29	+	attribute.String("service", k.service),
	30	+	attribute.String("target", k.target),
	31	+	}
	32	+	}
	33	+
	34	+	type backupLastSuccess struct {
	35	+	at time.Time
	36	+	size int64
	37	+	}
	38	+
	39	+	// backupState holds the last success per target, exported by observable
	40	+	// gauges at each collection.
	41	+	type backupState struct {
	42	+	mu sync.Mutex
	43	+	last map[backupTargetKey]backupLastSuccess
	44	+	nextRun map[domain.BackupKind]func() time.Time
	45	+	}
	46	+
	47	+	func newBackupState() *backupState {
	48	+	return &backupState{
	49	+	last: map[backupTargetKey]backupLastSuccess{},
	50	+	nextRun: map[domain.BackupKind]func() time.Time{},
	51	+	}
	52	+	}
	53	+
	54	+	func keyOf(run domain.BackupRunRecord) backupTargetKey {
	55	+	return backupTargetKey{kind: run.Kind, app: run.App, service: run.Service, target: run.Target}
	56	+	}
	57	+
	58	+	// remember stores a success unless a newer one is already known.
	59	+	func (s *backupState) remember(run domain.BackupRunRecord) {
	60	+	if !run.Succeeded \|\| run.FinishedAt.IsZero() {
	61	+	return
	62	+	}
	63	+	key := keyOf(run)
	64	+	s.mu.Lock()
	65	+	defer s.mu.Unlock()
	66	+	if prev, ok := s.last[key]; ok && !run.FinishedAt.After(prev.at) {
	67	+	return
	68	+	}
	69	+	s.last[key] = backupLastSuccess{at: run.FinishedAt, size: run.SizeBytes}
	70	+	}
	71	+
	72	+	// registerBackupInstruments creates the backup counters, histogram, and
	73	+	// observable gauges.
	74	+	func (m *Metrics) registerBackupInstruments() error {
	75	+	var err error
	76	+	if m.BackupRuns, err = m.meter.Int64Counter("gordon.backup.runs",
	77	+	metric.WithDescription("Finished backup runs by result")); err != nil {
	78	+	return err
	79	+	}
	80	+	if m.BackupDuration, err = m.meter.Float64Histogram("gordon.backup.duration_seconds",
	81	+	metric.WithDescription("Backup run duration in seconds"),
	82	+	metric.WithUnit("s"),
	83	+	metric.WithExplicitBucketBoundaries(1, 5, 15, 30, 60, 120, 300, 600, 1800, 3600)); err != nil {
	84	+	return err
	85	+	}
	86	+	lastSuccess, err := m.meter.Float64ObservableGauge("gordon.backup.last_success_timestamp_seconds",
	87	+	metric.WithDescription("Unix time of the last successful backup per target"),
	88	+	metric.WithUnit("s"))
	89	+	if err != nil {
	90	+	return err
	91	+	}
	92	+	lastSize, err := m.meter.Float64ObservableGauge("gordon.backup.last_size_bytes",
	93	+	metric.WithDescription("Size of the last successful backup per target"),
	94	+	metric.WithUnit("By"))
	95	+	if err != nil {
	96	+	return err
	97	+	}
	98	+	nextRun, err := m.meter.Float64ObservableGauge("gordon.backup.next_run_timestamp_seconds",
	99	+	metric.WithDescription("Unix time of the next scheduled backup run"),
	100	+	metric.WithUnit("s"))
	101	+	if err != nil {
	102	+	return err
	103	+	}
	104	+	_, err = m.meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
	105	+	m.backups.mu.Lock()
	106	+	defer m.backups.mu.Unlock()
	107	+	for key, last := range m.backups.last {
	108	+	attrs := metric.WithAttributes(key.attrs()...)
	109	+	o.ObserveFloat64(lastSuccess, float64(last.at.Unix()), attrs)
	110	+	o.ObserveFloat64(lastSize, float64(last.size), attrs)
	111	+	}
	112	+	for kind, next := range m.backups.nextRun {
	113	+	if at := next(); !at.IsZero() {
	114	+	o.ObserveFloat64(nextRun, float64(at.Unix()), metric.WithAttributes(attribute.String("kind", string(kind))))
	115	+	}
	116	+	}
	117	+	return nil
	118	+	}, lastSuccess, lastSize, nextRun)
	119	+	return err
	120	+	}
	121	+
	122	+	// RecordBackupRun implements out.BackupMetrics.
	123	+	func (m *Metrics) RecordBackupRun(ctx context.Context, run domain.BackupRunRecord) {
	124	+	result := "failure"
	125	+	if run.Succeeded {
	126	+	result = "success"
	127	+	}
	128	+	attrs := keyOf(run).attrs()
	129	+	m.BackupRuns.Add(ctx, 1, metric.WithAttributes(append(attrs, attribute.String("result", result))...))
	130	+	m.BackupDuration.Record(ctx, run.Duration.Seconds(), metric.WithAttributes(attrs...))
	131	+	m.backups.remember(run)
	132	+	}
	133	+
	134	+	// ObserveStoredBackup implements out.BackupMetrics.
	135	+	func (m *Metrics) ObserveStoredBackup(run domain.BackupRunRecord) {
	136	+	m.backups.remember(run)
	137	+	}
	138	+
	139	+	// ObserveBackupNextRun exports the next planned run of one backup kind.
	140	+	// next is called at each collection; a zero time is not exported.
	141	+	func (m *Metrics) ObserveBackupNextRun(kind domain.BackupKind, next func() time.Time) {
	142	+	m.backups.mu.Lock()
	143	+	defer m.backups.mu.Unlock()
	144	+	m.backups.nextRun[kind] = next
	145	+	}

A internal/adapters/out/telemetry/backup_metrics_test.go +135 -0

@@ -0,0 +1,135 @@
	1	+	package telemetry
	2	+
	3	+	import (
	4	+	"context"
	5	+	"testing"
	6	+	"time"
	7	+
	8	+	"github.com/stretchr/testify/assert"
	9	+	"github.com/stretchr/testify/require"
	10	+	"go.opentelemetry.io/otel"
	11	+	"go.opentelemetry.io/otel/attribute"
	12	+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
	13	+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
	14	+
	15	+	"github.com/bnema/gordon/internal/domain"
	16	+	)
	17	+
	18	+	func newTestMetrics(t testing.T) (Metrics, *sdkmetric.ManualReader) {
	19	+	t.Helper()
	20	+	reader := sdkmetric.NewManualReader()
	21	+	prev := otel.GetMeterProvider()
	22	+	otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)))
	23	+	t.Cleanup(func() { otel.SetMeterProvider(prev) })
	24	+	m, err := NewMetrics()
	25	+	require.NoError(t, err)
	26	+	return m, reader
	27	+	}
	28	+
	29	+	func collect(t testing.T, reader sdkmetric.ManualReader) map[string]metricdata.Aggregation {
	30	+	t.Helper()
	31	+	var rm metricdata.ResourceMetrics
	32	+	require.NoError(t, reader.Collect(context.Background(), &rm))
	33	+	out := map[string]metricdata.Aggregation{}
	34	+	for _, sm := range rm.ScopeMetrics {
	35	+	for _, m := range sm.Metrics {
	36	+	out[m.Name] = m.Data
	37	+	}
	38	+	}
	39	+	return out
	40	+	}
	41	+
	42	+	func attrValue(set attribute.Set, key string) string {
	43	+	v, _ := set.Value(attribute.Key(key))
	44	+	return v.AsString()
	45	+	}
	46	+
	47	+	// gaugeByTarget indexes float gauge points by their target attribute.
	48	+	func gaugeByTarget(t *testing.T, data metricdata.Aggregation) map[string]float64 {
	49	+	t.Helper()
	50	+	gauge, ok := data.(metricdata.Gauge[float64])
	51	+	require.True(t, ok, "expected a float64 gauge, got %T", data)
	52	+	out := map[string]float64{}
	53	+	for _, dp := range gauge.DataPoints {
	54	+	out[attrValue(dp.Attributes, "target")] = dp.Value
	55	+	}
	56	+	return out
	57	+	}
	58	+
	59	+	func TestRecordBackupRun_CountsResultAndTracksLastSuccess(t *testing.T) {
	60	+	m, reader := newTestMetrics(t)
	61	+	ctx := context.Background()
	62	+	done := time.Date(2026, 10, 18, 1, 30, 0, 0, time.UTC)
	63	+
	64	+	m.RecordBackupRun(ctx, domain.BackupRunRecord{
	65	+	Kind: domain.BackupKindVolume, App: "forgejo", Service: "forgejo", Target: "forgejo-data",
	66	+	Succeeded: true, Duration: 40 * time.Second, SizeBytes: 1024, FinishedAt: done,
	67	+	})
	68	+	m.RecordBackupRun(ctx, domain.BackupRunRecord{
	69	+	Kind: domain.BackupKindVolume, App: "forgejo", Service: "forgejo", Target: "forgejo-data",
	70	+	Succeeded: false, Duration: time.Second, FinishedAt: done.Add(time.Hour),
	71	+	})
	72	+
	73	+	data := collect(t, reader)
	74	+
	75	+	runs, ok := data["gordon.backup.runs"].(metricdata.Sum[int64])
	76	+	require.True(t, ok)
	77	+	results := map[string]int64{}
	78	+	for _, dp := range runs.DataPoints {
	79	+	assert.Equal(t, "volume", attrValue(dp.Attributes, "kind"))
	80	+	assert.Equal(t, "forgejo", attrValue(dp.Attributes, "app"))
	81	+	assert.Equal(t, "forgejo", attrValue(dp.Attributes, "service"))
	82	+	assert.Equal(t, "forgejo-data", attrValue(dp.Attributes, "target"))
	83	+	results[attrValue(dp.Attributes, "result")] = dp.Value
	84	+	}
	85	+	assert.Equal(t, map[string]int64{"success": 1, "failure": 1}, results)
	86	+
	87	+	hist, ok := data["gordon.backup.duration_seconds"].(metricdata.Histogram[float64])
	88	+	require.True(t, ok)
	89	+	var count uint64
	90	+	for _, dp := range hist.DataPoints {
	91	+	count += dp.Count
	92	+	}
	93	+	assert.Equal(t, uint64(2), count)
	94	+
	95	+	// A failure never moves the last-success timestamp or size.
	96	+	assert.Equal(t, map[string]float64{"forgejo-data": float64(done.Unix())},
	97	+	gaugeByTarget(t, data["gordon.backup.last_success_timestamp_seconds"]))
	98	+	assert.Equal(t, map[string]float64{"forgejo-data": 1024},
	99	+	gaugeByTarget(t, data["gordon.backup.last_size_bytes"]))
	100	+	}
	101	+
	102	+	func TestObserveStoredBackup_KeepsTheNewestSuccess(t *testing.T) {
	103	+	m, reader := newTestMetrics(t)
	104	+	newer := time.Date(2026, 10, 17, 1, 30, 0, 0, time.UTC)
	105	+	older := newer.Add(-24 * time.Hour)
	106	+	rec := domain.BackupRunRecord{Kind: domain.BackupKindDatabase, App: "shop", Service: "db", Target: "orders", Succeeded: true}
	107	+
	108	+	rec.FinishedAt, rec.SizeBytes = newer, 200
	109	+	m.ObserveStoredBackup(rec)
	110	+	rec.FinishedAt, rec.SizeBytes = older, 100
	111	+	m.ObserveStoredBackup(rec)
	112	+
	113	+	data := collect(t, reader)
	114	+	assert.Equal(t, map[string]float64{"orders": float64(newer.Unix())},
	115	+	gaugeByTarget(t, data["gordon.backup.last_success_timestamp_seconds"]))
	116	+	assert.Equal(t, map[string]float64{"orders": 200},
	117	+	gaugeByTarget(t, data["gordon.backup.last_size_bytes"]))
	118	+
	119	+	// Seeding never counts a run.
	120	+	_, counted := data["gordon.backup.runs"]
	121	+	assert.False(t, counted)
	122	+	}
	123	+
	124	+	func TestObserveBackupNextRun(t *testing.T) {
	125	+	m, reader := newTestMetrics(t)
	126	+	next := time.Date(2026, 10, 18, 1, 30, 0, 0, time.UTC)
	127	+	m.ObserveBackupNextRun(domain.BackupKindVolume, func() time.Time { return next })
	128	+	m.ObserveBackupNextRun(domain.BackupKindDatabase, func() time.Time { return time.Time{} })
	129	+
	130	+	gauge, ok := collect(t, reader)["gordon.backup.next_run_timestamp_seconds"].(metricdata.Gauge[float64])
	131	+	require.True(t, ok)
	132	+	require.Len(t, gauge.DataPoints, 1, "an unknown next run is not exported")
	133	+	assert.Equal(t, "volume", attrValue(gauge.DataPoints[0].Attributes, "kind"))
	134	+	assert.Equal(t, float64(next.Unix()), gauge.DataPoints[0].Value)
	135	+	}

@@ -0,0 +1,135 @@
	1	+	package telemetry
	2	+
	3	+	import (
	4	+	"context"
	5	+	"testing"
	6	+	"time"
	7	+
	8	+	"github.com/stretchr/testify/assert"
	9	+	"github.com/stretchr/testify/require"
	10	+	"go.opentelemetry.io/otel"
	11	+	"go.opentelemetry.io/otel/attribute"
	12	+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
	13	+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
	14	+
	15	+	"github.com/bnema/gordon/internal/domain"
	16	+	)
	17	+
	18	+	func newTestMetrics(t testing.T) (Metrics, *sdkmetric.ManualReader) {
	19	+	t.Helper()
	20	+	reader := sdkmetric.NewManualReader()
	21	+	prev := otel.GetMeterProvider()
	22	+	otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)))
	23	+	t.Cleanup(func() { otel.SetMeterProvider(prev) })
	24	+	m, err := NewMetrics()
	25	+	require.NoError(t, err)
	26	+	return m, reader
	27	+	}
	28	+
	29	+	func collect(t testing.T, reader sdkmetric.ManualReader) map[string]metricdata.Aggregation {
	30	+	t.Helper()
	31	+	var rm metricdata.ResourceMetrics
	32	+	require.NoError(t, reader.Collect(context.Background(), &rm))
	33	+	out := map[string]metricdata.Aggregation{}
	34	+	for _, sm := range rm.ScopeMetrics {
	35	+	for _, m := range sm.Metrics {
	36	+	out[m.Name] = m.Data
	37	+	}
	38	+	}
	39	+	return out
	40	+	}
	41	+
	42	+	func attrValue(set attribute.Set, key string) string {
	43	+	v, _ := set.Value(attribute.Key(key))
	44	+	return v.AsString()
	45	+	}
	46	+
	47	+	// gaugeByTarget indexes float gauge points by their target attribute.
	48	+	func gaugeByTarget(t *testing.T, data metricdata.Aggregation) map[string]float64 {
	49	+	t.Helper()
	50	+	gauge, ok := data.(metricdata.Gauge[float64])
	51	+	require.True(t, ok, "expected a float64 gauge, got %T", data)
	52	+	out := map[string]float64{}
	53	+	for _, dp := range gauge.DataPoints {
	54	+	out[attrValue(dp.Attributes, "target")] = dp.Value
	55	+	}
	56	+	return out
	57	+	}
	58	+
	59	+	func TestRecordBackupRun_CountsResultAndTracksLastSuccess(t *testing.T) {
	60	+	m, reader := newTestMetrics(t)
	61	+	ctx := context.Background()
	62	+	done := time.Date(2026, 10, 18, 1, 30, 0, 0, time.UTC)
	63	+
	64	+	m.RecordBackupRun(ctx, domain.BackupRunRecord{
	65	+	Kind: domain.BackupKindVolume, App: "forgejo", Service: "forgejo", Target: "forgejo-data",
	66	+	Succeeded: true, Duration: 40 * time.Second, SizeBytes: 1024, FinishedAt: done,
	67	+	})
	68	+	m.RecordBackupRun(ctx, domain.BackupRunRecord{
	69	+	Kind: domain.BackupKindVolume, App: "forgejo", Service: "forgejo", Target: "forgejo-data",
	70	+	Succeeded: false, Duration: time.Second, FinishedAt: done.Add(time.Hour),
	71	+	})
	72	+
	73	+	data := collect(t, reader)
	74	+
	75	+	runs, ok := data["gordon.backup.runs"].(metricdata.Sum[int64])
	76	+	require.True(t, ok)
	77	+	results := map[string]int64{}
	78	+	for _, dp := range runs.DataPoints {
	79	+	assert.Equal(t, "volume", attrValue(dp.Attributes, "kind"))
	80	+	assert.Equal(t, "forgejo", attrValue(dp.Attributes, "app"))
	81	+	assert.Equal(t, "forgejo", attrValue(dp.Attributes, "service"))
	82	+	assert.Equal(t, "forgejo-data", attrValue(dp.Attributes, "target"))
	83	+	results[attrValue(dp.Attributes, "result")] = dp.Value
	84	+	}
	85	+	assert.Equal(t, map[string]int64{"success": 1, "failure": 1}, results)
	86	+
	87	+	hist, ok := data["gordon.backup.duration_seconds"].(metricdata.Histogram[float64])
	88	+	require.True(t, ok)
	89	+	var count uint64
	90	+	for _, dp := range hist.DataPoints {
	91	+	count += dp.Count
	92	+	}
	93	+	assert.Equal(t, uint64(2), count)
	94	+
	95	+	// A failure never moves the last-success timestamp or size.
	96	+	assert.Equal(t, map[string]float64{"forgejo-data": float64(done.Unix())},
	97	+	gaugeByTarget(t, data["gordon.backup.last_success_timestamp_seconds"]))
	98	+	assert.Equal(t, map[string]float64{"forgejo-data": 1024},
	99	+	gaugeByTarget(t, data["gordon.backup.last_size_bytes"]))
	100	+	}
	101	+
	102	+	func TestObserveStoredBackup_KeepsTheNewestSuccess(t *testing.T) {
	103	+	m, reader := newTestMetrics(t)
	104	+	newer := time.Date(2026, 10, 17, 1, 30, 0, 0, time.UTC)
	105	+	older := newer.Add(-24 * time.Hour)
	106	+	rec := domain.BackupRunRecord{Kind: domain.BackupKindDatabase, App: "shop", Service: "db", Target: "orders", Succeeded: true}
	107	+
	108	+	rec.FinishedAt, rec.SizeBytes = newer, 200
	109	+	m.ObserveStoredBackup(rec)
	110	+	rec.FinishedAt, rec.SizeBytes = older, 100
	111	+	m.ObserveStoredBackup(rec)
	112	+
	113	+	data := collect(t, reader)
	114	+	assert.Equal(t, map[string]float64{"orders": float64(newer.Unix())},
	115	+	gaugeByTarget(t, data["gordon.backup.last_success_timestamp_seconds"]))
	116	+	assert.Equal(t, map[string]float64{"orders": 200},
	117	+	gaugeByTarget(t, data["gordon.backup.last_size_bytes"]))
	118	+
	119	+	// Seeding never counts a run.
	120	+	_, counted := data["gordon.backup.runs"]
	121	+	assert.False(t, counted)
	122	+	}
	123	+
	124	+	func TestObserveBackupNextRun(t *testing.T) {
	125	+	m, reader := newTestMetrics(t)
	126	+	next := time.Date(2026, 10, 18, 1, 30, 0, 0, time.UTC)
	127	+	m.ObserveBackupNextRun(domain.BackupKindVolume, func() time.Time { return next })
	128	+	m.ObserveBackupNextRun(domain.BackupKindDatabase, func() time.Time { return time.Time{} })
	129	+
	130	+	gauge, ok := collect(t, reader)["gordon.backup.next_run_timestamp_seconds"].(metricdata.Gauge[float64])
	131	+	require.True(t, ok)
	132	+	require.Len(t, gauge.DataPoints, 1, "an unknown next run is not exported")
	133	+	assert.Equal(t, "volume", attrValue(gauge.DataPoints[0].Attributes, "kind"))
	134	+	assert.Equal(t, float64(next.Unix()), gauge.DataPoints[0].Value)
	135	+	}

M internal/adapters/out/telemetry/metrics.go +9 -1


@@ -33,6 +33,11 @@ type Metrics struct {
33	33		// Events
34	34		EventsProcessed metric.Int64Counter
35	35		EventsDropped metric.Int64Counter
	36	+
	37	+	// Backups
	38	+	BackupRuns metric.Int64Counter
	39	+	BackupDuration metric.Float64Histogram
	40	+	backups *backupState
36	41		}
37	42
38	43		// NewMetrics creates and registers all Gordon metric instruments.

@@ -40,7 +45,7 @@ type Metrics struct {
40	45		// (OTel returns noop instruments when no MeterProvider is set).
41	46		func NewMetrics() (*Metrics, error) {
42	47		meter := otel.Meter("gordon")
43		-	m := &Metrics{meter: meter}
	48	+	m := &Metrics{meter: meter, backups: newBackupState()}
44	49		var err error
45	50
46	51		if m.DeployTotal, err = meter.Int64Counter("gordon.deploy.total",

@@ -82,6 +87,9 @@ func NewMetrics() (*Metrics, error) {
82	87		metric.WithDescription("Total events dropped")); err != nil {
83	88		return nil, err
84	89		}
	90	+	if err := m.registerBackupInstruments(); err != nil {
	91	+	return nil, err
	92	+	}
85	93
86	94		return m, nil
87	95		}

 	// Events
 	EventsProcessed metric.Int64Counter
 	EventsDropped   metric.Int64Counter
+	// Backups
+	BackupRuns     metric.Int64Counter
+	BackupDuration metric.Float64Histogram
+	backups        *backupState
 }
 // NewMetrics creates and registers all Gordon metric instruments.
 // (OTel returns noop instruments when no MeterProvider is set).
 func NewMetrics() (*Metrics, error) {
 	meter := otel.Meter("gordon")
-	m := &Metrics{meter: meter}
+	m := &Metrics{meter: meter, backups: newBackupState()}
 	var err error
 	if m.DeployTotal, err = meter.Int64Counter("gordon.deploy.total",
 		metric.WithDescription("Total events dropped")); err != nil {
 		return nil, err
 	}
+	if err := m.registerBackupInstruments(); err != nil {
+		return nil, err
+	}
 	return m, nil
 }

A internal/app/backup_metrics_wiring.go +71 -0

@@ -0,0 +1,71 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+	"time"
	6	+
	7	+	"github.com/bnema/zerowrap"
	8	+
	9	+	"github.com/bnema/gordon/internal/adapters/out/telemetry"
	10	+	"github.com/bnema/gordon/internal/domain"
	11	+	cronSvc "github.com/bnema/gordon/internal/usecase/cron"
	12	+	)
	13	+
	14	+	// nextRunObserver exports the next planned run of one backup kind.
	15	+	type nextRunObserver interface {
	16	+	ObserveBackupNextRun(kind domain.BackupKind, next func() time.Time)
	17	+	}
	18	+
	19	+	// nextRunObserverPort converts a possibly nil metrics adapter into an
	20	+	// interface that is nil when metrics are disabled. A typed nil must not
	21	+	// leak into the interface.
	22	+	func nextRunObserverPort(metrics *telemetry.Metrics) nextRunObserver {
	23	+	if metrics == nil {
	24	+	return nil
	25	+	}
	26	+	return metrics
	27	+	}
	28	+
	29	+	// wireBackupMetrics injects backup metrics into the backup services. It
	30	+	// must run before any backup can start, so no run reads the field while
	31	+	// it is written.
	32	+	func wireBackupMetrics(svc *services) {
	33	+	if svc == nil \|\| svc.metrics == nil {
	34	+	return
	35	+	}
	36	+	if svc.backupSvc != nil {
	37	+	svc.backupSvc.WithMetrics(svc.metrics)
	38	+	}
	39	+	if svc.volumeBackupSvc != nil {
	40	+	svc.volumeBackupSvc.WithMetrics(svc.metrics)
	41	+	}
	42	+	}
	43	+
	44	+	// seedBackupMetrics reports the last success of every target from
	45	+	// storage, so the metric is correct right after a restart. It only feeds
	46	+	// the metrics state (mutex-guarded), so it may run alongside backups.
	47	+	// Failures are logged, never fatal.
	48	+	func seedBackupMetrics(ctx context.Context, svc *services, log zerowrap.Logger) {
	49	+	if svc == nil \|\| svc.metrics == nil {
	50	+	return
	51	+	}
	52	+	if svc.backupSvc != nil {
	53	+	if err := svc.backupSvc.SeedMetrics(ctx); err != nil {
	54	+	log.Warn().Err(err).Msg("failed to seed database backup metrics")
	55	+	}
	56	+	}
	57	+	if svc.volumeBackupSvc != nil {
	58	+	if err := svc.volumeBackupSvc.SeedMetrics(ctx); err != nil {
	59	+	log.Warn().Err(err).Msg("failed to seed volume backup metrics")
	60	+	}
	61	+	}
	62	+	}
	63	+
	64	+	// observeSchedulerNextRun exports the scheduler entry's next run, read at
	65	+	// each metrics collection.
	66	+	func observeSchedulerNextRun(obs nextRunObserver, kind domain.BackupKind, scheduler *cronSvc.Scheduler, id string) {
	67	+	if obs == nil \|\| scheduler == nil {
	68	+	return
	69	+	}
	70	+	obs.ObserveBackupNextRun(kind, func() time.Time { return nextRunOf(scheduler, id) })
	71	+	}

@@ -0,0 +1,71 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+	"time"
	6	+
	7	+	"github.com/bnema/zerowrap"
	8	+
	9	+	"github.com/bnema/gordon/internal/adapters/out/telemetry"
	10	+	"github.com/bnema/gordon/internal/domain"
	11	+	cronSvc "github.com/bnema/gordon/internal/usecase/cron"
	12	+	)
	13	+
	14	+	// nextRunObserver exports the next planned run of one backup kind.
	15	+	type nextRunObserver interface {
	16	+	ObserveBackupNextRun(kind domain.BackupKind, next func() time.Time)
	17	+	}
	18	+
	19	+	// nextRunObserverPort converts a possibly nil metrics adapter into an
	20	+	// interface that is nil when metrics are disabled. A typed nil must not
	21	+	// leak into the interface.
	22	+	func nextRunObserverPort(metrics *telemetry.Metrics) nextRunObserver {
	23	+	if metrics == nil {
	24	+	return nil
	25	+	}
	26	+	return metrics
	27	+	}
	28	+
	29	+	// wireBackupMetrics injects backup metrics into the backup services. It
	30	+	// must run before any backup can start, so no run reads the field while
	31	+	// it is written.
	32	+	func wireBackupMetrics(svc *services) {
	33	+	if svc == nil \|\| svc.metrics == nil {
	34	+	return
	35	+	}
	36	+	if svc.backupSvc != nil {
	37	+	svc.backupSvc.WithMetrics(svc.metrics)
	38	+	}
	39	+	if svc.volumeBackupSvc != nil {
	40	+	svc.volumeBackupSvc.WithMetrics(svc.metrics)
	41	+	}
	42	+	}
	43	+
	44	+	// seedBackupMetrics reports the last success of every target from
	45	+	// storage, so the metric is correct right after a restart. It only feeds
	46	+	// the metrics state (mutex-guarded), so it may run alongside backups.
	47	+	// Failures are logged, never fatal.
	48	+	func seedBackupMetrics(ctx context.Context, svc *services, log zerowrap.Logger) {
	49	+	if svc == nil \|\| svc.metrics == nil {
	50	+	return
	51	+	}
	52	+	if svc.backupSvc != nil {
	53	+	if err := svc.backupSvc.SeedMetrics(ctx); err != nil {
	54	+	log.Warn().Err(err).Msg("failed to seed database backup metrics")
	55	+	}
	56	+	}
	57	+	if svc.volumeBackupSvc != nil {
	58	+	if err := svc.volumeBackupSvc.SeedMetrics(ctx); err != nil {
	59	+	log.Warn().Err(err).Msg("failed to seed volume backup metrics")
	60	+	}
	61	+	}
	62	+	}
	63	+
	64	+	// observeSchedulerNextRun exports the scheduler entry's next run, read at
	65	+	// each metrics collection.
	66	+	func observeSchedulerNextRun(obs nextRunObserver, kind domain.BackupKind, scheduler *cronSvc.Scheduler, id string) {
	67	+	if obs == nil \|\| scheduler == nil {
	68	+	return
	69	+	}
	70	+	obs.ObserveBackupNextRun(kind, func() time.Time { return nextRunOf(scheduler, id) })
	71	+	}

A internal/app/backup_metrics_wiring_test.go +49 -0

@@ -0,0 +1,49 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+	"testing"
	6	+	"time"
	7	+
	8	+	"github.com/bnema/zerowrap"
	9	+	"github.com/stretchr/testify/assert"
	10	+	"github.com/stretchr/testify/require"
	11	+
	12	+	"github.com/bnema/gordon/internal/domain"
	13	+	cronSvc "github.com/bnema/gordon/internal/usecase/cron"
	14	+	)
	15	+
	16	+	type fakeNextRunObserver struct {
	17	+	got map[domain.BackupKind]func() time.Time
	18	+	}
	19	+
	20	+	func (f *fakeNextRunObserver) ObserveBackupNextRun(kind domain.BackupKind, next func() time.Time) {
	21	+	if f.got == nil {
	22	+	f.got = map[domain.BackupKind]func() time.Time{}
	23	+	}
	24	+	f.got[kind] = next
	25	+	}
	26	+
	27	+	func TestObserveSchedulerNextRun_ReadsTheLiveEntry(t *testing.T) {
	28	+	scheduler := cronSvc.NewScheduler(zerowrap.Default())
	29	+	require.NoError(t, scheduler.Add("volume-backup-scheduler", "Volume Backups",
	30	+	domain.CronSchedule{Preset: domain.ScheduleDaily}, func(context.Context) error { return nil }))
	31	+
	32	+	obs := &fakeNextRunObserver{}
	33	+	observeSchedulerNextRun(obs, domain.BackupKindVolume, scheduler, "volume-backup-scheduler")
	34	+
	35	+	require.Contains(t, obs.got, domain.BackupKindVolume)
	36	+	next := obs.got[domain.BackupKindVolume]()
	37	+	assert.Equal(t, nextRunOf(scheduler, "volume-backup-scheduler"), next)
	38	+	assert.True(t, next.After(time.Now()))
	39	+	}
	40	+
	41	+	func TestObserveSchedulerNextRun_NilSafe(t *testing.T) {
	42	+	scheduler := cronSvc.NewScheduler(zerowrap.Default())
	43	+	assert.NotPanics(t, func() {
	44	+	observeSchedulerNextRun(nil, domain.BackupKindVolume, scheduler, "x")
	45	+	})
	46	+	obs := &fakeNextRunObserver{}
	47	+	observeSchedulerNextRun(obs, domain.BackupKindVolume, nil, "x")
	48	+	assert.Empty(t, obs.got)
	49	+	}

@@ -0,0 +1,49 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+	"testing"
	6	+	"time"
	7	+
	8	+	"github.com/bnema/zerowrap"
	9	+	"github.com/stretchr/testify/assert"
	10	+	"github.com/stretchr/testify/require"
	11	+
	12	+	"github.com/bnema/gordon/internal/domain"
	13	+	cronSvc "github.com/bnema/gordon/internal/usecase/cron"
	14	+	)
	15	+
	16	+	type fakeNextRunObserver struct {
	17	+	got map[domain.BackupKind]func() time.Time
	18	+	}
	19	+
	20	+	func (f *fakeNextRunObserver) ObserveBackupNextRun(kind domain.BackupKind, next func() time.Time) {
	21	+	if f.got == nil {
	22	+	f.got = map[domain.BackupKind]func() time.Time{}
	23	+	}
	24	+	f.got[kind] = next
	25	+	}
	26	+
	27	+	func TestObserveSchedulerNextRun_ReadsTheLiveEntry(t *testing.T) {
	28	+	scheduler := cronSvc.NewScheduler(zerowrap.Default())
	29	+	require.NoError(t, scheduler.Add("volume-backup-scheduler", "Volume Backups",
	30	+	domain.CronSchedule{Preset: domain.ScheduleDaily}, func(context.Context) error { return nil }))
	31	+
	32	+	obs := &fakeNextRunObserver{}
	33	+	observeSchedulerNextRun(obs, domain.BackupKindVolume, scheduler, "volume-backup-scheduler")
	34	+
	35	+	require.Contains(t, obs.got, domain.BackupKindVolume)
	36	+	next := obs.got[domain.BackupKindVolume]()
	37	+	assert.Equal(t, nextRunOf(scheduler, "volume-backup-scheduler"), next)
	38	+	assert.True(t, next.After(time.Now()))
	39	+	}
	40	+
	41	+	func TestObserveSchedulerNextRun_NilSafe(t *testing.T) {
	42	+	scheduler := cronSvc.NewScheduler(zerowrap.Default())
	43	+	assert.NotPanics(t, func() {
	44	+	observeSchedulerNextRun(nil, domain.BackupKindVolume, scheduler, "x")
	45	+	})
	46	+	obs := &fakeNextRunObserver{}
	47	+	observeSchedulerNextRun(obs, domain.BackupKindVolume, nil, "x")
	48	+	assert.Empty(t, obs.got)
	49	+	}

M internal/app/run.go +3 -0


@@ -296,6 +296,9 @@ func runServers(ctx context.Context, v viper.Viper, cfg Config, svc services,
296	296
297	297		startPublicTLSRuntimeWithWarning(ctx, svc.publicTLSRuntime, log)
298	298
	299	+	// Seeding lists stored backups (S3 for volumes); never delay startup.
	300	+	go seedBackupMetrics(ctx, svc, log)
	301	+
299	302		schedulerCleanup, err := startOptionalSchedulers(ctx, cfg, svc, log, v)
300	303		if err != nil {
301	304		cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)


@@ -296,6 +296,9 @@ func runServers(ctx context.Context, v viper.Viper, cfg Config, svc services,
296		296
297	startPublicTLSRuntimeWithWarning(ctx, svc.publicTLSRuntime, log)	297		startPublicTLSRuntimeWithWarning(ctx, svc.publicTLSRuntime, log)
298		298
		299	+	// Seeding lists stored backups (S3 for volumes); never delay startup.
		300	+	go seedBackupMetrics(ctx, svc, log)
		301	+
299	schedulerCleanup, err := startOptionalSchedulers(ctx, cfg, svc, log, v)	302		schedulerCleanup, err := startOptionalSchedulers(ctx, cfg, svc, log, v)
300	if err != nil {	303		if err != nil {
301	cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)	304		cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)

M internal/app/schedulers.go +2 -0


@@ -27,6 +27,7 @@ func startOptionalSchedulers(ctx context.Context, cfg Config, svc *services, log
27	27		}
28	28		if backupScheduler != nil {
29	29		schedulers = append(schedulers, backupScheduler)
	30	+	observeSchedulerNextRun(nextRunObserverPort(svc.metrics), domain.BackupKindDatabase, backupScheduler, "backup-scheduler")
30	31		}
31	32
32	33		volumeBackupScheduler, err := startVolumeBackupScheduler(ctx, cfg, svc, log)

@@ -36,6 +37,7 @@ func startOptionalSchedulers(ctx context.Context, cfg Config, svc *services, log
36	37		}
37	38		if volumeBackupScheduler != nil {
38	39		schedulers = append(schedulers, volumeBackupScheduler)
	40	+	observeSchedulerNextRun(nextRunObserverPort(svc.metrics), domain.BackupKindVolume, volumeBackupScheduler, "volume-backup-scheduler")
39	41		}
40	42
41	43		imageScheduler, err := startImagePruneScheduler(ctx, cfg, svc, log, func() int {


@@ -27,6 +27,7 @@ func startOptionalSchedulers(ctx context.Context, cfg Config, svc *services, log
27	}	27		}
28	if backupScheduler != nil {	28		if backupScheduler != nil {
29	schedulers = append(schedulers, backupScheduler)	29		schedulers = append(schedulers, backupScheduler)
		30	+	observeSchedulerNextRun(nextRunObserverPort(svc.metrics), domain.BackupKindDatabase, backupScheduler, "backup-scheduler")
30	}	31		}
31		32
32	volumeBackupScheduler, err := startVolumeBackupScheduler(ctx, cfg, svc, log)	33		volumeBackupScheduler, err := startVolumeBackupScheduler(ctx, cfg, svc, log)

@@ -36,6 +37,7 @@ func startOptionalSchedulers(ctx context.Context, cfg Config, svc *services, log
36	}	37		}
37	if volumeBackupScheduler != nil {	38		if volumeBackupScheduler != nil {
38	schedulers = append(schedulers, volumeBackupScheduler)	39		schedulers = append(schedulers, volumeBackupScheduler)
		40	+	observeSchedulerNextRun(nextRunObserverPort(svc.metrics), domain.BackupKindVolume, volumeBackupScheduler, "volume-backup-scheduler")
39	}	41		}
40		42
41	imageScheduler, err := startImagePruneScheduler(ctx, cfg, svc, log, func() int {	43		imageScheduler, err := startImagePruneScheduler(ctx, cfg, svc, log, func() int {

M internal/app/services.go +1 -0


@@ -483,6 +483,7 @@ func injectTelemetryMetrics(cfg Config, svc *services, log zerowrap.Logger) {
483	483		svc.registrySvc.SetMetrics(gordonMetrics)
484	484		svc.eventBus.SetMetrics(gordonMetrics)
485	485		svc.metrics = gordonMetrics
	486	+	wireBackupMetrics(svc)
486	487		}
487	488
488	489		// observeManagedContainers exports the managed-container gauge from app


@@ -483,6 +483,7 @@ func injectTelemetryMetrics(cfg Config, svc *services, log zerowrap.Logger) {
483	svc.registrySvc.SetMetrics(gordonMetrics)	483		svc.registrySvc.SetMetrics(gordonMetrics)
484	svc.eventBus.SetMetrics(gordonMetrics)	484		svc.eventBus.SetMetrics(gordonMetrics)
485	svc.metrics = gordonMetrics	485		svc.metrics = gordonMetrics
		486	+	wireBackupMetrics(svc)
486	}	487		}
487		488
488	// observeManagedContainers exports the managed-container gauge from app	489		// observeManagedContainers exports the managed-container gauge from app

A internal/boundaries/out/backup_metrics.go +19 -0

@@ -0,0 +1,19 @@
	1	+	package out
	2	+
	3	+	import (
	4	+	"context"
	5	+
	6	+	"github.com/bnema/gordon/internal/domain"
	7	+	)
	8	+
	9	+	// BackupMetrics records backup outcomes.
	10	+	// Implementations must be safe for concurrent use.
	11	+	type BackupMetrics interface {
	12	+	// RecordBackupRun counts one finished backup run. A successful run
	13	+	// also becomes the target's last success.
	14	+	RecordBackupRun(ctx context.Context, run domain.BackupRunRecord)
	15	+	// ObserveStoredBackup reports an existing stored backup as a success
	16	+	// without counting a run, so the last success survives a restart.
	17	+	// Older records never replace a newer one.
	18	+	ObserveStoredBackup(run domain.BackupRunRecord)
	19	+	}

@@ -0,0 +1,19 @@
	1	+	package out
	2	+
	3	+	import (
	4	+	"context"
	5	+
	6	+	"github.com/bnema/gordon/internal/domain"
	7	+	)
	8	+
	9	+	// BackupMetrics records backup outcomes.
	10	+	// Implementations must be safe for concurrent use.
	11	+	type BackupMetrics interface {
	12	+	// RecordBackupRun counts one finished backup run. A successful run
	13	+	// also becomes the target's last success.
	14	+	RecordBackupRun(ctx context.Context, run domain.BackupRunRecord)
	15	+	// ObserveStoredBackup reports an existing stored backup as a success
	16	+	// without counting a run, so the last success survives a restart.
	17	+	// Older records never replace a newer one.
	18	+	ObserveStoredBackup(run domain.BackupRunRecord)
	19	+	}

M internal/domain/backup.go +22 -0


@@ -85,6 +85,28 @@ type BackupResult struct {
85	85		Duration time.Duration
86	86		}
87	87
	88	+	// BackupKind tells database dumps and volume archives apart in metrics.
	89	+	type BackupKind string
	90	+
	91	+	const (
	92	+	BackupKindDatabase BackupKind = "database"
	93	+	BackupKindVolume BackupKind = "volume"
	94	+	)
	95	+
	96	+	// BackupRunRecord is the outcome of one backup, as reported to metrics.
	97	+	// Target is the database name for a dump and the declared volume name for
	98	+	// an archive.
	99	+	type BackupRunRecord struct {
	100	+	Kind BackupKind
	101	+	App string
	102	+	Service string
	103	+	Target string
	104	+	Succeeded bool
	105	+	Duration time.Duration
	106	+	SizeBytes int64
	107	+	FinishedAt time.Time
	108	+	}
	109	+
88	110		// DatabaseBackupJob represents a logical database backup operation.
89	111		type DatabaseBackupJob = BackupJob
90	112


@@ -85,6 +85,28 @@ type BackupResult struct {
85	Duration time.Duration	85		Duration time.Duration
86	}	86		}
87		87
		88	+	// BackupKind tells database dumps and volume archives apart in metrics.
		89	+	type BackupKind string
		90	+
		91	+	const (
		92	+	BackupKindDatabase BackupKind = "database"
		93	+	BackupKindVolume BackupKind = "volume"
		94	+	)
		95	+
		96	+	// BackupRunRecord is the outcome of one backup, as reported to metrics.
		97	+	// Target is the database name for a dump and the declared volume name for
		98	+	// an archive.
		99	+	type BackupRunRecord struct {
		100	+	Kind BackupKind
		101	+	App string
		102	+	Service string
		103	+	Target string
		104	+	Succeeded bool
		105	+	Duration time.Duration
		106	+	SizeBytes int64
		107	+	FinishedAt time.Time
		108	+	}
		109	+
88	// DatabaseBackupJob represents a logical database backup operation.	110		// DatabaseBackupJob represents a logical database backup operation.
89	type DatabaseBackupJob = BackupJob	111		type DatabaseBackupJob = BackupJob
90		112

A internal/usecase/backup/metrics_test.go +177 -0

@@ -0,0 +1,177 @@
	1	+	package backup
	2	+
	3	+	import (
	4	+	"context"
	5	+	"errors"
	6	+	"io"
	7	+	"strings"
	8	+	"sync"
	9	+	"testing"
	10	+	"time"
	11	+
	12	+	"github.com/stretchr/testify/assert"
	13	+	"github.com/stretchr/testify/mock"
	14	+	"github.com/stretchr/testify/require"
	15	+
	16	+	"github.com/bnema/gordon/internal/boundaries/out"
	17	+	outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
	18	+	"github.com/bnema/gordon/internal/domain"
	19	+	)
	20	+
	21	+	// recordingMetrics is a concurrency-safe out.BackupMetrics fake.
	22	+	type recordingMetrics struct {
	23	+	mu sync.Mutex
	24	+	runs []domain.BackupRunRecord
	25	+	stored []domain.BackupRunRecord
	26	+	}
	27	+
	28	+	func (r *recordingMetrics) RecordBackupRun(_ context.Context, run domain.BackupRunRecord) {
	29	+	r.mu.Lock()
	30	+	defer r.mu.Unlock()
	31	+	r.runs = append(r.runs, run)
	32	+	}
	33	+
	34	+	func (r *recordingMetrics) ObserveStoredBackup(run domain.BackupRunRecord) {
	35	+	r.mu.Lock()
	36	+	defer r.mu.Unlock()
	37	+	r.stored = append(r.stored, run)
	38	+	}
	39	+
	40	+	var _ out.BackupMetrics = (*recordingMetrics)(nil)
	41	+
	42	+	func TestVolumeService_RecordsRunMetrics(t *testing.T) {
	43	+	ctx := context.Background()
	44	+	state := outmocks.NewMockAppStateReader(t)
	45	+	state.EXPECT().ListApps(mock.Anything).Return([]string{"shop"}, nil).Once()
	46	+	state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once()
	47	+	state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{
	48	+	"api": volumeSpec("api", "data", "/data"),
	49	+	}, "ctr-api"), true, nil).Once()
	50	+
	51	+	metrics := &recordingMetrics{}
	52	+	svc := volumeTestService(t, state, &fakeVolumeArchiveExporter{}, &fakeVolumeBackupStorage{}, true).WithMetrics(metrics)
	53	+
	54	+	require.NoError(t, svc.RunVolumeBackupsForSchedule(ctx, domain.ScheduleDaily))
	55	+	require.Len(t, metrics.runs, 1)
	56	+	run := metrics.runs[0]
	57	+	assert.Equal(t, domain.BackupKindVolume, run.Kind)
	58	+	assert.Equal(t, "shop", run.App)
	59	+	assert.Equal(t, "api", run.Service)
	60	+	assert.Equal(t, "data", run.Target)
	61	+	assert.True(t, run.Succeeded)
	62	+	assert.Equal(t, int64(len("archive")), run.SizeBytes)
	63	+	assert.False(t, run.FinishedAt.IsZero())
	64	+	}
	65	+
	66	+	func TestVolumeService_RecordsFailedRun(t *testing.T) {
	67	+	ctx := context.Background()
	68	+	state := outmocks.NewMockAppStateReader(t)
	69	+	state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once()
	70	+	state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{
	71	+	"api": volumeSpec("api", "data", "/data"),
	72	+	}, "ctr-api"), true, nil).Once()
	73	+
	74	+	metrics := &recordingMetrics{}
	75	+	exporter := &fakeVolumeArchiveExporter{err: errors.New("helper failed")}
	76	+	svc := volumeTestService(t, state, exporter, &fakeVolumeBackupStorage{}, true).WithMetrics(metrics)
	77	+
	78	+	_, err := svc.RunVolumeBackups(ctx, "shop", "api", "data")
	79	+	require.Error(t, err)
	80	+	require.Len(t, metrics.runs, 1)
	81	+	assert.False(t, metrics.runs[0].Succeeded)
	82	+	assert.Equal(t, "data", metrics.runs[0].Target)
	83	+	}
	84	+
	85	+	func TestVolumeService_SeedMetricsFromStoredArchives(t *testing.T) {
	86	+	ctx := context.Background()
	87	+	started := time.Date(2026, 10, 17, 1, 30, 0, 0, time.UTC)
	88	+	storage := &fakeVolumeBackupStorage{jobs: []domain.VolumeBackupJob{
	89	+	{App: "shop", Service: "api", VolumeName: "data", Status: domain.BackupStatusCompleted, StartedAt: started, SizeBytes: 42},
	90	+	}}
	91	+	metrics := &recordingMetrics{}
	92	+	svc := volumeTestService(t, outmocks.NewMockAppStateReader(t), &fakeVolumeArchiveExporter{}, storage, true).WithMetrics(metrics)
	93	+
	94	+	require.NoError(t, svc.SeedMetrics(ctx))
	95	+	require.Len(t, metrics.stored, 1)
	96	+	assert.Equal(t, domain.BackupRunRecord{
	97	+	Kind: domain.BackupKindVolume, App: "shop", Service: "api", Target: "data",
	98	+	Succeeded: true, SizeBytes: 42, FinishedAt: started,
	99	+	}, metrics.stored[0])
	100	+	assert.Empty(t, metrics.runs)
	101	+	}
	102	+
	103	+	func TestService_RecordsRunMetrics(t *testing.T) {
	104	+	ctx := context.Background()
	105	+	state := outmocks.NewMockAppStateReader(t)
	106	+	state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once()
	107	+	state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{
	108	+	"api": appSpecWithDatabase("api", "orders", "daily"),
	109	+	}, "ctr-api"), true, nil).Once()
	110	+
	111	+	runtime, storage, svc := backupTestService(t, state)
	112	+	metrics := &recordingMetrics{}
	113	+	svc.WithMetrics(metrics)
	114	+	runtime.EXPECT().ExecInContainer(mock.Anything, "ctr-api", mock.Anything).
	115	+	Return(&out.ExecResult{ExitCode: 1, Stderr: []byte("boom")}, nil).Once()
	116	+	runtime.EXPECT().ExecInContainer(mock.Anything, "ctr-api", mock.MatchedBy(func(cmd []string) bool {
	117	+	return strings.HasPrefix(cmd[2], "rm -f ")
	118	+	})).Return(&out.ExecResult{ExitCode: 0}, nil).Once()
	119	+
	120	+	_, err := svc.RunBackup(ctx, "shop", "api", "orders")
	121	+	require.Error(t, err)
	122	+	require.Len(t, metrics.runs, 1)
	123	+	assert.Equal(t, domain.BackupRunRecord{
	124	+	Kind: domain.BackupKindDatabase, App: "shop", Service: "api", Target: "orders",
	125	+	Succeeded: false, Duration: metrics.runs[0].Duration, FinishedAt: metrics.runs[0].FinishedAt,
	126	+	}, metrics.runs[0])
	127	+	_ = storage
	128	+	}
	129	+
	130	+	func TestService_RecordsSuccessfulRun(t *testing.T) {
	131	+	ctx := context.Background()
	132	+	state := outmocks.NewMockAppStateReader(t)
	133	+	state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once()
	134	+	state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{
	135	+	"api": appSpecWithDatabase("api", "orders", "daily"),
	136	+	}, "ctr-api"), true, nil).Once()
	137	+
	138	+	runtime, storage, svc := backupTestService(t, state)
	139	+	metrics := &recordingMetrics{}
	140	+	svc.WithMetrics(metrics)
	141	+	runtime.EXPECT().ExecInContainer(mock.Anything, "ctr-api", mock.Anything).Return(&out.ExecResult{ExitCode: 0}, nil).Twice()
	142	+	runtime.EXPECT().CopyFromContainer(mock.Anything, "ctr-api", mock.Anything).
	143	+	Return(io.NopCloser(strings.NewReader("dump")), nil).Once()
	144	+	storage.EXPECT().Store(mock.Anything, "shop", "api", "orders", domain.BackupSchedule(""), mock.Anything, mock.Anything).
	145	+	RunAndReturn(func(_ context.Context, _, _, _ string, _ domain.BackupSchedule, _ time.Time, data io.Reader) (string, error) {
	146	+	_, err := io.Copy(io.Discard, data)
	147	+	return "/b/file.bak", err
	148	+	}).Once()
	149	+
	150	+	_, err := svc.RunBackup(ctx, "shop", "api", "orders")
	151	+	require.NoError(t, err)
	152	+	require.Len(t, metrics.runs, 1)
	153	+	assert.True(t, metrics.runs[0].Succeeded)
	154	+	assert.Equal(t, int64(len("dump")), metrics.runs[0].SizeBytes)
	155	+	}
	156	+
	157	+	func TestService_SeedMetricsFromStoredDumps(t *testing.T) {
	158	+	ctx := context.Background()
	159	+	state := outmocks.NewMockAppStateReader(t)
	160	+	state.EXPECT().ListApps(mock.Anything).Return([]string{"shop"}, nil).Once()
	161	+	started := time.Date(2026, 10, 17, 2, 0, 0, 0, time.UTC)
	162	+
	163	+	_, storage, svc := backupTestService(t, state)
	164	+	metrics := &recordingMetrics{}
	165	+	svc.WithMetrics(metrics)
	166	+	storage.EXPECT().List(mock.Anything, "shop", mock.Anything).Return([]domain.BackupJob{{
	167	+	App: "shop", Service: "api", DBName: "orders", Status: domain.BackupStatusCompleted,
	168	+	StartedAt: started, SizeBytes: 7,
	169	+	}}, nil).Once()
	170	+
	171	+	require.NoError(t, svc.SeedMetrics(ctx))
	172	+	require.Len(t, metrics.stored, 1)
	173	+	assert.Equal(t, domain.BackupRunRecord{
	174	+	Kind: domain.BackupKindDatabase, App: "shop", Service: "api", Target: "orders",
	175	+	Succeeded: true, SizeBytes: 7, FinishedAt: started,
	176	+	}, metrics.stored[0])
	177	+	}

@@ -0,0 +1,177 @@
	1	+	package backup
	2	+
	3	+	import (
	4	+	"context"
	5	+	"errors"
	6	+	"io"
	7	+	"strings"
	8	+	"sync"
	9	+	"testing"
	10	+	"time"
	11	+
	12	+	"github.com/stretchr/testify/assert"
	13	+	"github.com/stretchr/testify/mock"
	14	+	"github.com/stretchr/testify/require"
	15	+
	16	+	"github.com/bnema/gordon/internal/boundaries/out"
	17	+	outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
	18	+	"github.com/bnema/gordon/internal/domain"
	19	+	)
	20	+
	21	+	// recordingMetrics is a concurrency-safe out.BackupMetrics fake.
	22	+	type recordingMetrics struct {
	23	+	mu sync.Mutex
	24	+	runs []domain.BackupRunRecord
	25	+	stored []domain.BackupRunRecord
	26	+	}
	27	+
	28	+	func (r *recordingMetrics) RecordBackupRun(_ context.Context, run domain.BackupRunRecord) {
	29	+	r.mu.Lock()
	30	+	defer r.mu.Unlock()
	31	+	r.runs = append(r.runs, run)
	32	+	}
	33	+
	34	+	func (r *recordingMetrics) ObserveStoredBackup(run domain.BackupRunRecord) {
	35	+	r.mu.Lock()
	36	+	defer r.mu.Unlock()
	37	+	r.stored = append(r.stored, run)
	38	+	}
	39	+
	40	+	var _ out.BackupMetrics = (*recordingMetrics)(nil)
	41	+
	42	+	func TestVolumeService_RecordsRunMetrics(t *testing.T) {
	43	+	ctx := context.Background()
	44	+	state := outmocks.NewMockAppStateReader(t)
	45	+	state.EXPECT().ListApps(mock.Anything).Return([]string{"shop"}, nil).Once()
	46	+	state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once()
	47	+	state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{
	48	+	"api": volumeSpec("api", "data", "/data"),
	49	+	}, "ctr-api"), true, nil).Once()
	50	+
	51	+	metrics := &recordingMetrics{}
	52	+	svc := volumeTestService(t, state, &fakeVolumeArchiveExporter{}, &fakeVolumeBackupStorage{}, true).WithMetrics(metrics)
	53	+
	54	+	require.NoError(t, svc.RunVolumeBackupsForSchedule(ctx, domain.ScheduleDaily))
	55	+	require.Len(t, metrics.runs, 1)
	56	+	run := metrics.runs[0]
	57	+	assert.Equal(t, domain.BackupKindVolume, run.Kind)
	58	+	assert.Equal(t, "shop", run.App)
	59	+	assert.Equal(t, "api", run.Service)
	60	+	assert.Equal(t, "data", run.Target)
	61	+	assert.True(t, run.Succeeded)
	62	+	assert.Equal(t, int64(len("archive")), run.SizeBytes)
	63	+	assert.False(t, run.FinishedAt.IsZero())
	64	+	}
	65	+
	66	+	func TestVolumeService_RecordsFailedRun(t *testing.T) {
	67	+	ctx := context.Background()
	68	+	state := outmocks.NewMockAppStateReader(t)
	69	+	state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once()
	70	+	state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{
	71	+	"api": volumeSpec("api", "data", "/data"),
	72	+	}, "ctr-api"), true, nil).Once()
	73	+
	74	+	metrics := &recordingMetrics{}
	75	+	exporter := &fakeVolumeArchiveExporter{err: errors.New("helper failed")}
	76	+	svc := volumeTestService(t, state, exporter, &fakeVolumeBackupStorage{}, true).WithMetrics(metrics)
	77	+
	78	+	_, err := svc.RunVolumeBackups(ctx, "shop", "api", "data")
	79	+	require.Error(t, err)
	80	+	require.Len(t, metrics.runs, 1)
	81	+	assert.False(t, metrics.runs[0].Succeeded)
	82	+	assert.Equal(t, "data", metrics.runs[0].Target)
	83	+	}
	84	+
	85	+	func TestVolumeService_SeedMetricsFromStoredArchives(t *testing.T) {
	86	+	ctx := context.Background()
	87	+	started := time.Date(2026, 10, 17, 1, 30, 0, 0, time.UTC)
	88	+	storage := &fakeVolumeBackupStorage{jobs: []domain.VolumeBackupJob{
	89	+	{App: "shop", Service: "api", VolumeName: "data", Status: domain.BackupStatusCompleted, StartedAt: started, SizeBytes: 42},
	90	+	}}
	91	+	metrics := &recordingMetrics{}
	92	+	svc := volumeTestService(t, outmocks.NewMockAppStateReader(t), &fakeVolumeArchiveExporter{}, storage, true).WithMetrics(metrics)
	93	+
	94	+	require.NoError(t, svc.SeedMetrics(ctx))
	95	+	require.Len(t, metrics.stored, 1)
	96	+	assert.Equal(t, domain.BackupRunRecord{
	97	+	Kind: domain.BackupKindVolume, App: "shop", Service: "api", Target: "data",
	98	+	Succeeded: true, SizeBytes: 42, FinishedAt: started,
	99	+	}, metrics.stored[0])
	100	+	assert.Empty(t, metrics.runs)
	101	+	}
	102	+
	103	+	func TestService_RecordsRunMetrics(t *testing.T) {
	104	+	ctx := context.Background()
	105	+	state := outmocks.NewMockAppStateReader(t)
	106	+	state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once()
	107	+	state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{
	108	+	"api": appSpecWithDatabase("api", "orders", "daily"),
	109	+	}, "ctr-api"), true, nil).Once()
	110	+
	111	+	runtime, storage, svc := backupTestService(t, state)
	112	+	metrics := &recordingMetrics{}
	113	+	svc.WithMetrics(metrics)
	114	+	runtime.EXPECT().ExecInContainer(mock.Anything, "ctr-api", mock.Anything).
	115	+	Return(&out.ExecResult{ExitCode: 1, Stderr: []byte("boom")}, nil).Once()
	116	+	runtime.EXPECT().ExecInContainer(mock.Anything, "ctr-api", mock.MatchedBy(func(cmd []string) bool {
	117	+	return strings.HasPrefix(cmd[2], "rm -f ")
	118	+	})).Return(&out.ExecResult{ExitCode: 0}, nil).Once()
	119	+
	120	+	_, err := svc.RunBackup(ctx, "shop", "api", "orders")
	121	+	require.Error(t, err)
	122	+	require.Len(t, metrics.runs, 1)
	123	+	assert.Equal(t, domain.BackupRunRecord{
	124	+	Kind: domain.BackupKindDatabase, App: "shop", Service: "api", Target: "orders",
	125	+	Succeeded: false, Duration: metrics.runs[0].Duration, FinishedAt: metrics.runs[0].FinishedAt,
	126	+	}, metrics.runs[0])
	127	+	_ = storage
	128	+	}
	129	+
	130	+	func TestService_RecordsSuccessfulRun(t *testing.T) {
	131	+	ctx := context.Background()
	132	+	state := outmocks.NewMockAppStateReader(t)
	133	+	state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once()
	134	+	state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{
	135	+	"api": appSpecWithDatabase("api", "orders", "daily"),
	136	+	}, "ctr-api"), true, nil).Once()
	137	+
	138	+	runtime, storage, svc := backupTestService(t, state)
	139	+	metrics := &recordingMetrics{}
	140	+	svc.WithMetrics(metrics)
	141	+	runtime.EXPECT().ExecInContainer(mock.Anything, "ctr-api", mock.Anything).Return(&out.ExecResult{ExitCode: 0}, nil).Twice()
	142	+	runtime.EXPECT().CopyFromContainer(mock.Anything, "ctr-api", mock.Anything).
	143	+	Return(io.NopCloser(strings.NewReader("dump")), nil).Once()
	144	+	storage.EXPECT().Store(mock.Anything, "shop", "api", "orders", domain.BackupSchedule(""), mock.Anything, mock.Anything).
	145	+	RunAndReturn(func(_ context.Context, _, _, _ string, _ domain.BackupSchedule, _ time.Time, data io.Reader) (string, error) {
	146	+	_, err := io.Copy(io.Discard, data)
	147	+	return "/b/file.bak", err
	148	+	}).Once()
	149	+
	150	+	_, err := svc.RunBackup(ctx, "shop", "api", "orders")
	151	+	require.NoError(t, err)
	152	+	require.Len(t, metrics.runs, 1)
	153	+	assert.True(t, metrics.runs[0].Succeeded)
	154	+	assert.Equal(t, int64(len("dump")), metrics.runs[0].SizeBytes)
	155	+	}
	156	+
	157	+	func TestService_SeedMetricsFromStoredDumps(t *testing.T) {
	158	+	ctx := context.Background()
	159	+	state := outmocks.NewMockAppStateReader(t)
	160	+	state.EXPECT().ListApps(mock.Anything).Return([]string{"shop"}, nil).Once()
	161	+	started := time.Date(2026, 10, 17, 2, 0, 0, 0, time.UTC)
	162	+
	163	+	_, storage, svc := backupTestService(t, state)
	164	+	metrics := &recordingMetrics{}
	165	+	svc.WithMetrics(metrics)
	166	+	storage.EXPECT().List(mock.Anything, "shop", mock.Anything).Return([]domain.BackupJob{{
	167	+	App: "shop", Service: "api", DBName: "orders", Status: domain.BackupStatusCompleted,
	168	+	StartedAt: started, SizeBytes: 7,
	169	+	}}, nil).Once()
	170	+
	171	+	require.NoError(t, svc.SeedMetrics(ctx))
	172	+	require.Len(t, metrics.stored, 1)
	173	+	assert.Equal(t, domain.BackupRunRecord{
	174	+	Kind: domain.BackupKindDatabase, App: "shop", Service: "api", Target: "orders",
	175	+	Succeeded: true, SizeBytes: 7, FinishedAt: started,
	176	+	}, metrics.stored[0])
	177	+	}

M internal/usecase/backup/service.go +44 -0


@@ -28,6 +28,7 @@ type Service struct {
28	28		config domain.BackupConfig
29	29		log zerowrap.Logger
30	30		state out.AppStateReader
	31	+	metrics out.BackupMetrics
31	32		}
32	33
33	34		// NewService creates a backup service.

@@ -52,6 +53,32 @@ func (s Service) WithAppState(state out.AppStateReader) Service {
52	53		return s
53	54		}
54	55
	56	+	// WithMetrics wires the backup metrics recorder. Without it no metric is
	57	+	// recorded.
	58	+	func (s Service) WithMetrics(metrics out.BackupMetrics) Service {
	59	+	s.metrics = metrics
	60	+	return s
	61	+	}
	62	+
	63	+	// SeedMetrics reports the newest stored dump of every target as its last
	64	+	// success, so the metric survives a restart. It counts no run.
	65	+	func (s *Service) SeedMetrics(ctx context.Context) error {
	66	+	if s.metrics == nil {
	67	+	return nil
	68	+	}
	69	+	jobs, err := s.ListBackups(ctx, "")
	70	+	if err != nil {
	71	+	return fmt.Errorf("seed backup metrics: %w", err)
	72	+	}
	73	+	for _, job := range jobs {
	74	+	s.metrics.ObserveStoredBackup(domain.BackupRunRecord{
	75	+	Kind: domain.BackupKindDatabase, App: job.App, Service: job.Service, Target: job.DBName,
	76	+	Succeeded: true, SizeBytes: job.SizeBytes, FinishedAt: job.StartedAt,
	77	+	})
	78	+	}
	79	+	return nil
	80	+	}
	81	+
55	82		// Targets returns every declared database target of one app, ordered by
56	83		// service then database.
57	84		func (s *Service) Targets(ctx context.Context, app string) ([]domain.DatabaseTarget, error) {

@@ -212,7 +239,24 @@ func (s *Service) Status(ctx context.Context) ([]domain.BackupJob, error) {
212	239
213	240		// runTarget runs one declared database backup through the service
214	241		// container that hosts it.
	242	+	// runTarget dumps one target and records the outcome in metrics.
215	243		func (s Service) runTarget(ctx context.Context, target domain.DatabaseTarget, schedule domain.BackupSchedule) (domain.BackupResult, error) {
	244	+	started := time.Now()
	245	+	result, err := s.dumpTarget(ctx, target, schedule)
	246	+	if s.metrics != nil {
	247	+	run := domain.BackupRunRecord{
	248	+	Kind: domain.BackupKindDatabase, App: target.App, Service: target.Service, Target: target.Database,
	249	+	Succeeded: err == nil, Duration: time.Since(started), FinishedAt: time.Now().UTC(),
	250	+	}
	251	+	if result != nil {
	252	+	run.SizeBytes = result.Job.SizeBytes
	253	+	}
	254	+	s.metrics.RecordBackupRun(ctx, run)
	255	+	}
	256	+	return result, err
	257	+	}
	258	+
	259	+	func (s Service) dumpTarget(ctx context.Context, target domain.DatabaseTarget, schedule domain.BackupSchedule) (domain.BackupResult, error) {
216	260		if target.ContainerID == "" {
217	261		return nil, fmt.Errorf("backup: app %q service %q has no active container", target.App, target.Service)
218	262		}


@@ -28,6 +28,7 @@ type Service struct {
28	config domain.BackupConfig	28		config domain.BackupConfig
29	log zerowrap.Logger	29		log zerowrap.Logger
30	state out.AppStateReader	30		state out.AppStateReader
		31	+	metrics out.BackupMetrics
31	}	32		}
32		33
33	// NewService creates a backup service.	34		// NewService creates a backup service.

@@ -52,6 +53,32 @@ func (s Service) WithAppState(state out.AppStateReader) Service {
52	return s	53		return s
53	}	54		}
54		55
		56	+	// WithMetrics wires the backup metrics recorder. Without it no metric is
		57	+	// recorded.
		58	+	func (s Service) WithMetrics(metrics out.BackupMetrics) Service {
		59	+	s.metrics = metrics
		60	+	return s
		61	+	}
		62	+
		63	+	// SeedMetrics reports the newest stored dump of every target as its last
		64	+	// success, so the metric survives a restart. It counts no run.
		65	+	func (s *Service) SeedMetrics(ctx context.Context) error {
		66	+	if s.metrics == nil {
		67	+	return nil
		68	+	}
		69	+	jobs, err := s.ListBackups(ctx, "")
		70	+	if err != nil {
		71	+	return fmt.Errorf("seed backup metrics: %w", err)
		72	+	}
		73	+	for _, job := range jobs {
		74	+	s.metrics.ObserveStoredBackup(domain.BackupRunRecord{
		75	+	Kind: domain.BackupKindDatabase, App: job.App, Service: job.Service, Target: job.DBName,
		76	+	Succeeded: true, SizeBytes: job.SizeBytes, FinishedAt: job.StartedAt,
		77	+	})
		78	+	}
		79	+	return nil
		80	+	}
		81	+
55	// Targets returns every declared database target of one app, ordered by	82		// Targets returns every declared database target of one app, ordered by
56	// service then database.	83		// service then database.
57	func (s *Service) Targets(ctx context.Context, app string) ([]domain.DatabaseTarget, error) {	84		func (s *Service) Targets(ctx context.Context, app string) ([]domain.DatabaseTarget, error) {

@@ -212,7 +239,24 @@ func (s *Service) Status(ctx context.Context) ([]domain.BackupJob, error) {
212		239
213	// runTarget runs one declared database backup through the service	240		// runTarget runs one declared database backup through the service
214	// container that hosts it.	241		// container that hosts it.
		242	+	// runTarget dumps one target and records the outcome in metrics.
215	func (s Service) runTarget(ctx context.Context, target domain.DatabaseTarget, schedule domain.BackupSchedule) (domain.BackupResult, error) {	243		func (s Service) runTarget(ctx context.Context, target domain.DatabaseTarget, schedule domain.BackupSchedule) (domain.BackupResult, error) {
		244	+	started := time.Now()
		245	+	result, err := s.dumpTarget(ctx, target, schedule)
		246	+	if s.metrics != nil {
		247	+	run := domain.BackupRunRecord{
		248	+	Kind: domain.BackupKindDatabase, App: target.App, Service: target.Service, Target: target.Database,
		249	+	Succeeded: err == nil, Duration: time.Since(started), FinishedAt: time.Now().UTC(),
		250	+	}
		251	+	if result != nil {
		252	+	run.SizeBytes = result.Job.SizeBytes
		253	+	}
		254	+	s.metrics.RecordBackupRun(ctx, run)
		255	+	}
		256	+	return result, err
		257	+	}
		258	+
		259	+	func (s Service) dumpTarget(ctx context.Context, target domain.DatabaseTarget, schedule domain.BackupSchedule) (domain.BackupResult, error) {
216	if target.ContainerID == "" {	260		if target.ContainerID == "" {
217	return nil, fmt.Errorf("backup: app %q service %q has no active container", target.App, target.Service)	261		return nil, fmt.Errorf("backup: app %q service %q has no active container", target.App, target.Service)
218	}	262		}

M internal/usecase/backup/volume_service.go +40 -0


@@ -26,6 +26,7 @@ type VolumeService struct {
26	26		config domain.VolumeBackupConfig
27	27		log zerowrap.Logger
28	28		state out.AppStateReader
	29	+	metrics out.BackupMetrics
29	30
30	31		mu sync.Mutex
31	32		recent map[string]domain.VolumeBackupJob

@@ -48,6 +49,32 @@ func (s VolumeService) WithAppState(state out.AppStateReader) VolumeService {
48	49		return s
49	50		}
50	51
	52	+	// WithMetrics wires the backup metrics recorder. Without it no metric is
	53	+	// recorded.
	54	+	func (s VolumeService) WithMetrics(metrics out.BackupMetrics) VolumeService {
	55	+	s.metrics = metrics
	56	+	return s
	57	+	}
	58	+
	59	+	// SeedMetrics reports the newest stored archive of every target as its
	60	+	// last success, so the metric survives a restart. It counts no run.
	61	+	func (s *VolumeService) SeedMetrics(ctx context.Context) error {
	62	+	if s.metrics == nil {
	63	+	return nil
	64	+	}
	65	+	jobs, err := s.storage.ListVolumeArchives(ctx, "")
	66	+	if err != nil {
	67	+	return fmt.Errorf("seed volume backup metrics: %w", err)
	68	+	}
	69	+	for _, job := range jobs {
	70	+	s.metrics.ObserveStoredBackup(domain.BackupRunRecord{
	71	+	Kind: domain.BackupKindVolume, App: job.App, Service: job.Service, Target: job.VolumeName,
	72	+	Succeeded: true, SizeBytes: job.SizeBytes, FinishedAt: job.StartedAt,
	73	+	})
	74	+	}
	75	+	return nil
	76	+	}
	77	+
51	78		// VolumeTargets returns every declared volume target of one app.
52	79		func (s *VolumeService) VolumeTargets(ctx context.Context, app string) ([]domain.VolumeBackupTarget, error) {
53	80		if s.state == nil {

@@ -214,7 +241,20 @@ func (s *VolumeService) runVolumeBackupsConcurrently(ctx context.Context, target
214	241		return jobs
215	242		}
216	243
	244	+	// runVolumeBackup archives one target and records the outcome in metrics.
217	245		func (s *VolumeService) runVolumeBackup(ctx context.Context, target domain.VolumeBackupTarget) domain.VolumeBackupJob {
	246	+	job := s.archiveVolume(ctx, target)
	247	+	if s.metrics != nil {
	248	+	s.metrics.RecordBackupRun(ctx, domain.BackupRunRecord{
	249	+	Kind: domain.BackupKindVolume, App: target.App, Service: target.Service, Target: target.VolumeName,
	250	+	Succeeded: job.Status == domain.BackupStatusCompleted, Duration: job.CompletedAt.Sub(job.StartedAt),
	251	+	SizeBytes: job.SizeBytes, FinishedAt: job.CompletedAt,
	252	+	})
	253	+	}
	254	+	return job
	255	+	}
	256	+
	257	+	func (s *VolumeService) archiveVolume(ctx context.Context, target domain.VolumeBackupTarget) domain.VolumeBackupJob {
218	258		ctx = zerowrap.CtxWithFields(ctx, map[string]any{
219	259		"app": target.App,
220	260		"service": target.Service,


@@ -26,6 +26,7 @@ type VolumeService struct {
26	config domain.VolumeBackupConfig	26		config domain.VolumeBackupConfig
27	log zerowrap.Logger	27		log zerowrap.Logger
28	state out.AppStateReader	28		state out.AppStateReader
		29	+	metrics out.BackupMetrics
29		30
30	mu sync.Mutex	31		mu sync.Mutex
31	recent map[string]domain.VolumeBackupJob	32		recent map[string]domain.VolumeBackupJob

@@ -48,6 +49,32 @@ func (s VolumeService) WithAppState(state out.AppStateReader) VolumeService {
48	return s	49		return s
49	}	50		}
50		51
		52	+	// WithMetrics wires the backup metrics recorder. Without it no metric is
		53	+	// recorded.
		54	+	func (s VolumeService) WithMetrics(metrics out.BackupMetrics) VolumeService {
		55	+	s.metrics = metrics
		56	+	return s
		57	+	}
		58	+
		59	+	// SeedMetrics reports the newest stored archive of every target as its
		60	+	// last success, so the metric survives a restart. It counts no run.
		61	+	func (s *VolumeService) SeedMetrics(ctx context.Context) error {
		62	+	if s.metrics == nil {
		63	+	return nil
		64	+	}
		65	+	jobs, err := s.storage.ListVolumeArchives(ctx, "")
		66	+	if err != nil {
		67	+	return fmt.Errorf("seed volume backup metrics: %w", err)
		68	+	}
		69	+	for _, job := range jobs {
		70	+	s.metrics.ObserveStoredBackup(domain.BackupRunRecord{
		71	+	Kind: domain.BackupKindVolume, App: job.App, Service: job.Service, Target: job.VolumeName,
		72	+	Succeeded: true, SizeBytes: job.SizeBytes, FinishedAt: job.StartedAt,
		73	+	})
		74	+	}
		75	+	return nil
		76	+	}
		77	+
51	// VolumeTargets returns every declared volume target of one app.	78		// VolumeTargets returns every declared volume target of one app.
52	func (s *VolumeService) VolumeTargets(ctx context.Context, app string) ([]domain.VolumeBackupTarget, error) {	79		func (s *VolumeService) VolumeTargets(ctx context.Context, app string) ([]domain.VolumeBackupTarget, error) {
53	if s.state == nil {	80		if s.state == nil {

@@ -214,7 +241,20 @@ func (s *VolumeService) runVolumeBackupsConcurrently(ctx context.Context, target
214	return jobs	241		return jobs
215	}	242		}
216		243
		244	+	// runVolumeBackup archives one target and records the outcome in metrics.
217	func (s *VolumeService) runVolumeBackup(ctx context.Context, target domain.VolumeBackupTarget) domain.VolumeBackupJob {	245		func (s *VolumeService) runVolumeBackup(ctx context.Context, target domain.VolumeBackupTarget) domain.VolumeBackupJob {
		246	+	job := s.archiveVolume(ctx, target)
		247	+	if s.metrics != nil {
		248	+	s.metrics.RecordBackupRun(ctx, domain.BackupRunRecord{
		249	+	Kind: domain.BackupKindVolume, App: target.App, Service: target.Service, Target: target.VolumeName,
		250	+	Succeeded: job.Status == domain.BackupStatusCompleted, Duration: job.CompletedAt.Sub(job.StartedAt),
		251	+	SizeBytes: job.SizeBytes, FinishedAt: job.CompletedAt,
		252	+	})
		253	+	}
		254	+	return job
		255	+	}
		256	+
		257	+	func (s *VolumeService) archiveVolume(ctx context.Context, target domain.VolumeBackupTarget) domain.VolumeBackupJob {
218	ctx = zerowrap.CtxWithFields(ctx, map[string]any{	258		ctx = zerowrap.CtxWithFields(ctx, map[string]any{
219	"app": target.App,	259		"app": target.App,
220	"service": target.Service,	260		"service": target.Service,