Merge pull request 'feat(backup): export OTel metrics for database and volume backups' (#28) from feat/backup-metrics into main

commit: de8f68e191de6327b3b6a4b9c74404f60ad81217
author: bnema · 5d ago · 2026-10-17 07:07 UTC
merge of: 108f11b + 7be96b1 · diff against the first parent
signature: SSH UnverifiedSHA256:VOCA5lYARHcbKDZ7GhTV45VmcjWnJ6ZQIBazj17LivA The key is unknown
refs: tag: v3.4.1

21 files · +1133 -7

M .mockery.yaml +1 -0


@@ -20,6 +20,7 @@ packages:
20	20		SecretFileStore:
21	21		ContainerLogWriter:
22	22		Metrics:
	23	+	BackupMetrics:
23	24		LogExporter:
24	25		ContainerLogStreamer:
25	26		TokenStore:


@@ -20,6 +20,7 @@ packages:
20	SecretFileStore:	20		SecretFileStore:
21	ContainerLogWriter:	21		ContainerLogWriter:
22	Metrics:	22		Metrics:
		23	+	BackupMetrics:
23	LogExporter:	24		LogExporter:
24	ContainerLogStreamer:	25		ContainerLogStreamer:
25	TokenStore:	26		TokenStore:

M docs/config/backups.md +5 -0


@@ -105,6 +105,10 @@ Objects written by earlier versions under a domain prefix are not read or
105	105		migrated: back them up and re-run the app's volume backups if you need them in
106	106		the current layout.
107	107
	108	+	## Monitoring
	109	+
	110	+	With [telemetry](./telemetry.md) metrics enabled, Gordon exports the result, duration, and size of every backup run, the last success of each target, and the next scheduled run. See [Backups metrics](./telemetry.md#backups) for the metric names and example alerts.
	111	+
108	112		## Notes
109	113
110	114		- Volume backups archive the app's declared volumes that its backup declaration references.

@@ -122,3 +126,4 @@ the current layout.
122	126		- [App Manifest](./apps.md)
123	127		- [CLI Backup Command](../cli/backup.md)
124	128		- [Configuration Reference](./reference.md)
	129	+	- [Telemetry](./telemetry.md)


@@ -105,6 +105,10 @@ Objects written by earlier versions under a domain prefix are not read or
105	migrated: back them up and re-run the app's volume backups if you need them in	105		migrated: back them up and re-run the app's volume backups if you need them in
106	the current layout.	106		the current layout.
107		107
		108	+	## Monitoring
		109	+
		110	+	With [telemetry](./telemetry.md) metrics enabled, Gordon exports the result, duration, and size of every backup run, the last success of each target, and the next scheduled run. See [Backups metrics](./telemetry.md#backups) for the metric names and example alerts.
		111	+
108	## Notes	112		## Notes
109		113
110	- Volume backups archive the app's declared volumes that its backup declaration references.	114		- Volume backups archive the app's declared volumes that its backup declaration references.

@@ -122,3 +126,4 @@ the current layout.
122	- [App Manifest](./apps.md)	126		- [App Manifest](./apps.md)
123	- [CLI Backup Command](../cli/backup.md)	127		- [CLI Backup Command](../cli/backup.md)
124	- [Configuration Reference](./reference.md)	128		- [Configuration Reference](./reference.md)
		129	+	- [Telemetry](./telemetry.md)

M docs/config/telemetry.md +28 -1


@@ -25,7 +25,7 @@ trace_sample_rate = 1.0
25	25		\| `endpoint` \| string \| `""` \| OTLP HTTP endpoint URL (e.g. `http://localhost:4318/api/default`) \|
26	26		\| `auth_token` \| string \| `""` \| Base64-encoded `user:password` for Basic auth \|
27	27		\| `traces` \| bool \| `true` \| Export distributed traces \|
28		-	\| `metrics` \| bool \| `true` \| Export metrics (deploy counters, container lifecycle, registry, events) \|
	28	+	\| `metrics` \| bool \| `true` \| Export metrics (deploy counters, container lifecycle, registry, events, backups) \|
29	29		\| `logs` \| bool \| `true` \| Export Gordon, proxy access, and app container logs to OTLP \|
30	30		\| `trace_sample_rate` \| float \| `1.0` \| Fraction of traces to sample (`0.0` = none, `1.0` = all) \|
31	31

@@ -160,6 +160,33 @@ Attributes: `source` (restarts only: `monitor` or `api`). `gordon.container.mana
160	160
161	161		Attributes: `name`, `reference`
162	162
	163	+	### Backups
	164	+
	165	+	\| Metric \| Type \| Unit \| Description \|
	166	+	\|--------\|------\|------\|-------------\|
	167	+	\| `gordon.backup.runs` \| Counter \| - \| Finished backup runs (scheduled and manual) \|
	168	+	\| `gordon.backup.duration_seconds` \| Histogram \| s \| Duration of each backup run \|
	169	+	\| `gordon.backup.last_success_timestamp_seconds` \| Gauge \| s \| Start time (Unix) of the last successful backup per target \|
	170	+	\| `gordon.backup.last_size_bytes` \| Gauge \| By \| Size of the last successful backup per target \|
	171	+	\| `gordon.backup.next_run_timestamp_seconds` \| Gauge \| s \| Unix time of the next scheduled run \|
	172	+
	173	+	Attributes: `kind` (`database` or `volume`), `app`, `service`, `target` (database name or declared volume name), and `result` (`success` or `failure`, runs only). `next_run_timestamp_seconds` only carries `kind`.
	174	+
	175	+	The last success is the start time of the run, the same timestamp the stored dump or archive is named after. At startup Gordon restores it from storage for every declared target and drops targets that are no longer declared, so the gauges survive restarts and removed apps stop being reported. A run missed while Gordon is down never appears as a failure: alert on the age of the last success instead.
	176	+
	177	+	With the OpenTelemetry Collector Prometheus exporter, names become `gordon_backup_runs_total`, `gordon_backup_last_success_timestamp_seconds`, and so on. Example alerts:
	178	+
	179	+	```promql
	180	+	# No successful backup of a target for more than 26 hours
	181	+	time() - gordon_backup_last_success_timestamp_seconds > 26 * 3600
	182	+
	183	+	# A host stopped exporting its volume backup schedule (one rule per expected host and kind).
	184	+	# host_name requires resource_to_telemetry_conversion on the Collector Prometheus exporter.
	185	+	absent_over_time(gordon_backup_next_run_timestamp_seconds{host_name="my-host", kind="volume"}[15m])
	186	+	```
	187	+
	188	+	The age alert only covers targets that succeeded at least once: a target that has never succeeded has no last-success series. Watch `gordon_backup_runs_total{result="failure"}` to catch those.
	189	+
163	190		### Event Bus
164	191
165	192		\| Metric \| Type \| Unit \| Description \|


@@ -25,7 +25,7 @@ trace_sample_rate = 1.0
25		\| `endpoint` \| string \| `""` \| OTLP HTTP endpoint URL (e.g. `http://localhost:4318/api/default`) \|	25		\| `endpoint` \| string \| `""` \| OTLP HTTP endpoint URL (e.g. `http://localhost:4318/api/default`) \|
26		\| `auth_token` \| string \| `""` \| Base64-encoded `user:password` for Basic auth \|	26		\| `auth_token` \| string \| `""` \| Base64-encoded `user:password` for Basic auth \|
27		\| `traces` \| bool \| `true` \| Export distributed traces \|	27		\| `traces` \| bool \| `true` \| Export distributed traces \|
28	-	\| `metrics` \| bool \| `true` \| Export metrics (deploy counters, container lifecycle, registry, events) \|	28	+	\| `metrics` \| bool \| `true` \| Export metrics (deploy counters, container lifecycle, registry, events, backups) \|
29		\| `logs` \| bool \| `true` \| Export Gordon, proxy access, and app container logs to OTLP \|	29		\| `logs` \| bool \| `true` \| Export Gordon, proxy access, and app container logs to OTLP \|
30		\| `trace_sample_rate` \| float \| `1.0` \| Fraction of traces to sample (`0.0` = none, `1.0` = all) \|	30		\| `trace_sample_rate` \| float \| `1.0` \| Fraction of traces to sample (`0.0` = none, `1.0` = all) \|
31			31

@@ -160,6 +160,33 @@ Attributes: `source` (restarts only: `monitor` or `api`). `gordon.container.mana
160			160
161		Attributes: `name`, `reference`	161		Attributes: `name`, `reference`
162			162
			163	+	### Backups
			164	+
			165	+	\| Metric \| Type \| Unit \| Description \|
			166	+	\|--------\|------\|------\|-------------\|
			167	+	\| `gordon.backup.runs` \| Counter \| - \| Finished backup runs (scheduled and manual) \|
			168	+	\| `gordon.backup.duration_seconds` \| Histogram \| s \| Duration of each backup run \|
			169	+	\| `gordon.backup.last_success_timestamp_seconds` \| Gauge \| s \| Start time (Unix) of the last successful backup per target \|
			170	+	\| `gordon.backup.last_size_bytes` \| Gauge \| By \| Size of the last successful backup per target \|
			171	+	\| `gordon.backup.next_run_timestamp_seconds` \| Gauge \| s \| Unix time of the next scheduled run \|
			172	+
			173	+	Attributes: `kind` (`database` or `volume`), `app`, `service`, `target` (database name or declared volume name), and `result` (`success` or `failure`, runs only). `next_run_timestamp_seconds` only carries `kind`.
			174	+
			175	+	The last success is the start time of the run, the same timestamp the stored dump or archive is named after. At startup Gordon restores it from storage for every declared target and drops targets that are no longer declared, so the gauges survive restarts and removed apps stop being reported. A run missed while Gordon is down never appears as a failure: alert on the age of the last success instead.
			176	+
			177	+	With the OpenTelemetry Collector Prometheus exporter, names become `gordon_backup_runs_total`, `gordon_backup_last_success_timestamp_seconds`, and so on. Example alerts:
			178	+
			179	+	```promql
			180	+	# No successful backup of a target for more than 26 hours
			181	+	time() - gordon_backup_last_success_timestamp_seconds > 26 * 3600
			182	+
			183	+	# A host stopped exporting its volume backup schedule (one rule per expected host and kind).
			184	+	# host_name requires resource_to_telemetry_conversion on the Collector Prometheus exporter.
			185	+	absent_over_time(gordon_backup_next_run_timestamp_seconds{host_name="my-host", kind="volume"}[15m])
			186	+	```
			187	+
			188	+	The age alert only covers targets that succeeded at least once: a target that has never succeeded has no last-success series. Watch `gordon_backup_runs_total{result="failure"}` to catch those.
			189	+
163		### Event Bus	190		### Event Bus
164			191
165		\| Metric \| Type \| Unit \| Description \|	192		\| Metric \| Type \| Unit \| Description \|

M internal/adapters/out/s3/volume_backup_storage.go +41 -3


@@ -150,6 +150,42 @@ func (s *VolumeBackupStorage) GetVolumeArchive(ctx context.Context, artifactRef
150	150
151	151		// ListVolumeArchives lists completed volume archive backups for a domain.
152	152		func (s *VolumeBackupStorage) ListVolumeArchives(ctx context.Context, app string) ([]domain.VolumeBackupJob, error) {
	153	+	jobs, err := s.listArchiveObjects(ctx, app)
	154	+	if err != nil {
	155	+	return nil, err
	156	+	}
	157	+	for i := range jobs {
	158	+	s.hydrateVolumeBackupJobMetadata(ctx, &jobs[i], s.keyOf(jobs[i]))
	159	+	}
	160	+	sortVolumeBackupJobs(jobs)
	161	+	return jobs, nil
	162	+	}
	163	+
	164	+	// LatestVolumeArchives returns the newest archive of every app volume from
	165	+	// the object listing alone, without one metadata request per object.
	166	+	func (s *VolumeBackupStorage) LatestVolumeArchives(ctx context.Context) ([]domain.VolumeBackupJob, error) {
	167	+	jobs, err := s.listArchiveObjects(ctx, "")
	168	+	if err != nil {
	169	+	return nil, err
	170	+	}
	171	+	latest := make(map[string]domain.VolumeBackupJob)
	172	+	for _, job := range jobs {
	173	+	key := job.App + "/" + job.VolumeName
	174	+	if prev, ok := latest[key]; !ok \|\| job.StartedAt.After(prev.StartedAt) {
	175	+	latest[key] = job
	176	+	}
	177	+	}
	178	+	out := make([]domain.VolumeBackupJob, 0, len(latest))
	179	+	for _, job := range latest {
	180	+	out = append(out, job)
	181	+	}
	182	+	sortVolumeBackupJobs(out)
	183	+	return out, nil
	184	+	}
	185	+
	186	+	// listArchiveObjects lists the archives of one app, or of every app when
	187	+	// app is empty, from object keys and sizes only.
	188	+	func (s *VolumeBackupStorage) listArchiveObjects(ctx context.Context, app string) ([]domain.VolumeBackupJob, error) {
153	189		listPrefix := s.appPrefix(app)
154	190		jobs := make([]domain.VolumeBackupJob, 0)
155	191		var token *string

@@ -173,7 +209,6 @@ func (s *VolumeBackupStorage) ListVolumeArchives(ctx context.Context, app string
173	209		if obj.Size != nil {
174	210		job.SizeBytes = *obj.Size
175	211		}
176		-	s.hydrateVolumeBackupJobMetadata(ctx, &job, *obj.Key)
177	212		jobs = append(jobs, job)
178	213		}
179	214		if out.IsTruncated == nil \|\| !*out.IsTruncated {

@@ -181,11 +216,14 @@ func (s *VolumeBackupStorage) ListVolumeArchives(ctx context.Context, app string
181	216		}
182	217		token = out.NextContinuationToken
183	218		}
184		-
185		-	sortVolumeBackupJobs(jobs)
186	219		return jobs, nil
187	220		}
188	221
	222	+	// keyOf returns the object key of a listed job.
	223	+	func (s *VolumeBackupStorage) keyOf(job domain.VolumeBackupJob) string {
	224	+	return strings.TrimPrefix(job.ArtifactRef, "s3://"+s.bucket+"/")
	225	+	}
	226	+
189	227		// DeleteVolumeArchive deletes a volume archive by artifact reference.
190	228		func (s VolumeBackupStorage) hydrateVolumeBackupJobMetadata(ctx context.Context, job domain.VolumeBackupJob, key string) {
191	229		out, err := s.client.HeadObject(ctx, &awss3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})


@@ -150,6 +150,42 @@ func (s *VolumeBackupStorage) GetVolumeArchive(ctx context.Context, artifactRef
150			150
151		// ListVolumeArchives lists completed volume archive backups for a domain.	151		// ListVolumeArchives lists completed volume archive backups for a domain.
152		func (s *VolumeBackupStorage) ListVolumeArchives(ctx context.Context, app string) ([]domain.VolumeBackupJob, error) {	152		func (s *VolumeBackupStorage) ListVolumeArchives(ctx context.Context, app string) ([]domain.VolumeBackupJob, error) {
			153	+	jobs, err := s.listArchiveObjects(ctx, app)
			154	+	if err != nil {
			155	+	return nil, err
			156	+	}
			157	+	for i := range jobs {
			158	+	s.hydrateVolumeBackupJobMetadata(ctx, &jobs[i], s.keyOf(jobs[i]))
			159	+	}
			160	+	sortVolumeBackupJobs(jobs)
			161	+	return jobs, nil
			162	+	}
			163	+
			164	+	// LatestVolumeArchives returns the newest archive of every app volume from
			165	+	// the object listing alone, without one metadata request per object.
			166	+	func (s *VolumeBackupStorage) LatestVolumeArchives(ctx context.Context) ([]domain.VolumeBackupJob, error) {
			167	+	jobs, err := s.listArchiveObjects(ctx, "")
			168	+	if err != nil {
			169	+	return nil, err
			170	+	}
			171	+	latest := make(map[string]domain.VolumeBackupJob)
			172	+	for _, job := range jobs {
			173	+	key := job.App + "/" + job.VolumeName
			174	+	if prev, ok := latest[key]; !ok \|\| job.StartedAt.After(prev.StartedAt) {
			175	+	latest[key] = job
			176	+	}
			177	+	}
			178	+	out := make([]domain.VolumeBackupJob, 0, len(latest))
			179	+	for _, job := range latest {
			180	+	out = append(out, job)
			181	+	}
			182	+	sortVolumeBackupJobs(out)
			183	+	return out, nil
			184	+	}
			185	+
			186	+	// listArchiveObjects lists the archives of one app, or of every app when
			187	+	// app is empty, from object keys and sizes only.
			188	+	func (s *VolumeBackupStorage) listArchiveObjects(ctx context.Context, app string) ([]domain.VolumeBackupJob, error) {
153		listPrefix := s.appPrefix(app)	189		listPrefix := s.appPrefix(app)
154		jobs := make([]domain.VolumeBackupJob, 0)	190		jobs := make([]domain.VolumeBackupJob, 0)
155		var token *string	191		var token *string

@@ -173,7 +209,6 @@ func (s *VolumeBackupStorage) ListVolumeArchives(ctx context.Context, app string
173		if obj.Size != nil {	209		if obj.Size != nil {
174		job.SizeBytes = *obj.Size	210		job.SizeBytes = *obj.Size
175		}	211		}
176	-	s.hydrateVolumeBackupJobMetadata(ctx, &job, *obj.Key)
177		jobs = append(jobs, job)	212		jobs = append(jobs, job)
178		}	213		}
179		if out.IsTruncated == nil \|\| !*out.IsTruncated {	214		if out.IsTruncated == nil \|\| !*out.IsTruncated {

@@ -181,11 +216,14 @@ func (s *VolumeBackupStorage) ListVolumeArchives(ctx context.Context, app string
181		}	216		}
182		token = out.NextContinuationToken	217		token = out.NextContinuationToken
183		}	218		}
184	-
185	-	sortVolumeBackupJobs(jobs)
186		return jobs, nil	219		return jobs, nil
187		}	220		}
188			221
			222	+	// keyOf returns the object key of a listed job.
			223	+	func (s *VolumeBackupStorage) keyOf(job domain.VolumeBackupJob) string {
			224	+	return strings.TrimPrefix(job.ArtifactRef, "s3://"+s.bucket+"/")
			225	+	}
			226	+
189		// DeleteVolumeArchive deletes a volume archive by artifact reference.	227		// DeleteVolumeArchive deletes a volume archive by artifact reference.
190		func (s VolumeBackupStorage) hydrateVolumeBackupJobMetadata(ctx context.Context, job domain.VolumeBackupJob, key string) {	228		func (s VolumeBackupStorage) hydrateVolumeBackupJobMetadata(ctx context.Context, job domain.VolumeBackupJob, key string) {
191		out, err := s.client.HeadObject(ctx, &awss3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})	229		out, err := s.client.HeadObject(ctx, &awss3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})

M internal/adapters/out/s3/volume_backup_storage_test.go +34 -0


@@ -22,6 +22,7 @@ type fakeVolumeS3Client struct {
22	22		objects map[string][]byte
23	23		metadata map[string]map[string]string
24	24		deleted []string
	25	+	heads int
25	26		}
26	27
27	28		func newFakeVolumeS3Client() *fakeVolumeS3Client {

@@ -34,6 +35,7 @@ func (f fakeVolumeS3Client) GetObject(_ context.Context, params awss3.GetObjec
34	35		}
35	36
36	37		func (f fakeVolumeS3Client) HeadObject(_ context.Context, params awss3.HeadObjectInput, _ ...func(awss3.Options)) (awss3.HeadObjectOutput, error) {
	38	+	f.heads++
37	39		return &awss3.HeadObjectOutput{Metadata: f.metadata[aws.ToString(params.Key)]}, nil
38	40		}
39	41

@@ -166,6 +168,38 @@ func TestVolumeBackupStorageRetentionMixesLongAndShortIDs(t *testing.T) {
166	168		assert.Equal(t, []string{longID}, client.deleted)
167	169		}
168	170
	171	+	// TestVolumeBackupStorageLatestVolumeArchives proves the newest archive of
	172	+	// each app volume is selected from the listing alone, with no per-object
	173	+	// metadata request.
	174	+	func TestVolumeBackupStorageLatestVolumeArchives(t *testing.T) {
	175	+	client := newFakeVolumeS3Client()
	176	+	storage := NewVolumeBackupStorageWithClients(domain.VolumeBackupConfig{S3Bucket: "bucket", S3Prefix: "gordon"}, client, nil)
	177	+	for key, data := range map[string]string{
	178	+	"gordon/apps/forgejo/volumes/forgejo-data/20261017T013000Z-1a2b3c4d.tar.gz": "old",
	179	+	"gordon/apps/forgejo/volumes/forgejo-data/20261018T013000Z-5e6f7a8b.tar.gz": "newest",
	180	+	"gordon/apps/forgejo/volumes/postgres-data/20261016T013000Z-0a0b0c0d.tar.gz": "pg",
	181	+	"gordon/apps/ermite/volumes/forgejo-data/20261015T013000Z-9a9b9c9d.tar.gz": "other-app",
	182	+	} {
	183	+	client.objects[key] = []byte(data)
	184	+	}
	185	+
	186	+	jobs, err := storage.LatestVolumeArchives(context.Background())
	187	+	require.NoError(t, err)
	188	+	assert.Equal(t, 0, client.heads)
	189	+
	190	+	got := map[string]domain.VolumeBackupJob{}
	191	+	for _, job := range jobs {
	192	+	got[job.App+"/"+job.VolumeName] = job
	193	+	}
	194	+	require.Len(t, got, 3)
	195	+	newest := got["forgejo/forgejo-data"]
	196	+	assert.Equal(t, "5e6f7a8b", newest.ID)
	197	+	assert.Equal(t, int64(len("newest")), newest.SizeBytes)
	198	+	assert.Equal(t, time.Date(2026, 10, 18, 1, 30, 0, 0, time.UTC), newest.StartedAt)
	199	+	assert.Contains(t, got, "forgejo/postgres-data")
	200	+	assert.Contains(t, got, "ermite/forgejo-data")
	201	+	}
	202	+
169	203		func TestVolumeBackupStorageRetentionUsesParsedKeyTimestamp(t *testing.T) {
170	204		client := newFakeVolumeS3Client()
171	205		storage := NewVolumeBackupStorageWithClients(domain.VolumeBackupConfig{S3Bucket: "bucket"}, client, &fakeVolumeUploader{client: client})


@@ -22,6 +22,7 @@ type fakeVolumeS3Client struct {
22	objects map[string][]byte	22		objects map[string][]byte
23	metadata map[string]map[string]string	23		metadata map[string]map[string]string
24	deleted []string	24		deleted []string
		25	+	heads int
25	}	26		}
26		27
27	func newFakeVolumeS3Client() *fakeVolumeS3Client {	28		func newFakeVolumeS3Client() *fakeVolumeS3Client {

@@ -34,6 +35,7 @@ func (f fakeVolumeS3Client) GetObject(_ context.Context, params awss3.GetObjec
34	}	35		}
35		36
36	func (f fakeVolumeS3Client) HeadObject(_ context.Context, params awss3.HeadObjectInput, _ ...func(awss3.Options)) (awss3.HeadObjectOutput, error) {	37		func (f fakeVolumeS3Client) HeadObject(_ context.Context, params awss3.HeadObjectInput, _ ...func(awss3.Options)) (awss3.HeadObjectOutput, error) {
		38	+	f.heads++
37	return &awss3.HeadObjectOutput{Metadata: f.metadata[aws.ToString(params.Key)]}, nil	39		return &awss3.HeadObjectOutput{Metadata: f.metadata[aws.ToString(params.Key)]}, nil
38	}	40		}
39		41

@@ -166,6 +168,38 @@ func TestVolumeBackupStorageRetentionMixesLongAndShortIDs(t *testing.T) {
166	assert.Equal(t, []string{longID}, client.deleted)	168		assert.Equal(t, []string{longID}, client.deleted)
167	}	169		}
168		170
		171	+	// TestVolumeBackupStorageLatestVolumeArchives proves the newest archive of
		172	+	// each app volume is selected from the listing alone, with no per-object
		173	+	// metadata request.
		174	+	func TestVolumeBackupStorageLatestVolumeArchives(t *testing.T) {
		175	+	client := newFakeVolumeS3Client()
		176	+	storage := NewVolumeBackupStorageWithClients(domain.VolumeBackupConfig{S3Bucket: "bucket", S3Prefix: "gordon"}, client, nil)
		177	+	for key, data := range map[string]string{
		178	+	"gordon/apps/forgejo/volumes/forgejo-data/20261017T013000Z-1a2b3c4d.tar.gz": "old",
		179	+	"gordon/apps/forgejo/volumes/forgejo-data/20261018T013000Z-5e6f7a8b.tar.gz": "newest",
		180	+	"gordon/apps/forgejo/volumes/postgres-data/20261016T013000Z-0a0b0c0d.tar.gz": "pg",
		181	+	"gordon/apps/ermite/volumes/forgejo-data/20261015T013000Z-9a9b9c9d.tar.gz": "other-app",
		182	+	} {
		183	+	client.objects[key] = []byte(data)
		184	+	}
		185	+
		186	+	jobs, err := storage.LatestVolumeArchives(context.Background())
		187	+	require.NoError(t, err)
		188	+	assert.Equal(t, 0, client.heads)
		189	+
		190	+	got := map[string]domain.VolumeBackupJob{}
		191	+	for _, job := range jobs {
		192	+	got[job.App+"/"+job.VolumeName] = job
		193	+	}
		194	+	require.Len(t, got, 3)
		195	+	newest := got["forgejo/forgejo-data"]
		196	+	assert.Equal(t, "5e6f7a8b", newest.ID)
		197	+	assert.Equal(t, int64(len("newest")), newest.SizeBytes)
		198	+	assert.Equal(t, time.Date(2026, 10, 18, 1, 30, 0, 0, time.UTC), newest.StartedAt)
		199	+	assert.Contains(t, got, "forgejo/postgres-data")
		200	+	assert.Contains(t, got, "ermite/forgejo-data")
		201	+	}
		202	+
169	func TestVolumeBackupStorageRetentionUsesParsedKeyTimestamp(t *testing.T) {	203		func TestVolumeBackupStorageRetentionUsesParsedKeyTimestamp(t *testing.T) {
170	client := newFakeVolumeS3Client()	204		client := newFakeVolumeS3Client()
171	storage := NewVolumeBackupStorageWithClients(domain.VolumeBackupConfig{S3Bucket: "bucket"}, client, &fakeVolumeUploader{client: client})	205		storage := NewVolumeBackupStorageWithClients(domain.VolumeBackupConfig{S3Bucket: "bucket"}, client, &fakeVolumeUploader{client: client})

A internal/adapters/out/telemetry/backup_metrics.go +158 -0

@@ -0,0 +1,158 @@
	1	+	package telemetry
	2	+
	3	+	import (
	4	+	"context"
	5	+	"sync"
	6	+	"time"
	7	+
	8	+	"go.opentelemetry.io/otel/attribute"
	9	+	"go.opentelemetry.io/otel/metric"
	10	+
	11	+	"github.com/bnema/gordon/internal/boundaries/out"
	12	+	"github.com/bnema/gordon/internal/domain"
	13	+	)
	14	+
	15	+	var _ out.BackupMetrics = (*Metrics)(nil)
	16	+
	17	+	func targetAttrs(ref domain.BackupTargetRef) []attribute.KeyValue {
	18	+	return []attribute.KeyValue{
	19	+	attribute.String("kind", string(ref.Kind)),
	20	+	attribute.String("app", ref.App),
	21	+	attribute.String("service", ref.Service),
	22	+	attribute.String("target", ref.Target),
	23	+	}
	24	+	}
	25	+
	26	+	type backupLastSuccess struct {
	27	+	at time.Time
	28	+	size int64
	29	+	updated time.Time // when this entry was written, for retain
	30	+	}
	31	+
	32	+	// backupState holds the last success per target, exported by observable
	33	+	// gauges at each collection.
	34	+	type backupState struct {
	35	+	now func() time.Time
	36	+	mu sync.Mutex
	37	+	last map[domain.BackupTargetRef]backupLastSuccess
	38	+	nextRun map[domain.BackupKind]func() time.Time
	39	+	}
	40	+
	41	+	func newBackupState() *backupState {
	42	+	return &backupState{
	43	+	now: time.Now,
	44	+	last: map[domain.BackupTargetRef]backupLastSuccess{},
	45	+	nextRun: map[domain.BackupKind]func() time.Time{},
	46	+	}
	47	+	}
	48	+
	49	+	// remember stores a success unless a newer one is already known.
	50	+	func (s *backupState) remember(run domain.BackupRunRecord) {
	51	+	if !run.Succeeded \|\| run.StartedAt.IsZero() {
	52	+	return
	53	+	}
	54	+	key := run.BackupTargetRef
	55	+	s.mu.Lock()
	56	+	defer s.mu.Unlock()
	57	+	if prev, ok := s.last[key]; ok && !run.StartedAt.After(prev.at) {
	58	+	return
	59	+	}
	60	+	s.last[key] = backupLastSuccess{at: run.StartedAt, size: run.SizeBytes, updated: s.now()}
	61	+	}
	62	+
	63	+	// retain drops the last success of every target of kind not in targets,
	64	+	// unless it was updated after asOf.
	65	+	func (s *backupState) retain(kind domain.BackupKind, targets []domain.BackupTargetRef, asOf time.Time) {
	66	+	keep := make(map[domain.BackupTargetRef]struct{}, len(targets))
	67	+	for _, ref := range targets {
	68	+	keep[ref] = struct{}{}
	69	+	}
	70	+	s.mu.Lock()
	71	+	defer s.mu.Unlock()
	72	+	for key, last := range s.last {
	73	+	if _, ok := keep[key]; key.Kind != kind \|\| ok \|\| last.updated.After(asOf) {
	74	+	continue
	75	+	}
	76	+	delete(s.last, key)
	77	+	}
	78	+	}
	79	+
	80	+	// registerBackupInstruments creates the backup counters, histogram, and
	81	+	// observable gauges.
	82	+	func (m *Metrics) registerBackupInstruments() error {
	83	+	var err error
	84	+	if m.BackupRuns, err = m.meter.Int64Counter("gordon.backup.runs",
	85	+	metric.WithDescription("Finished backup runs by result")); err != nil {
	86	+	return err
	87	+	}
	88	+	if m.BackupDuration, err = m.meter.Float64Histogram("gordon.backup.duration_seconds",
	89	+	metric.WithDescription("Backup run duration in seconds"),
	90	+	metric.WithUnit("s"),
	91	+	metric.WithExplicitBucketBoundaries(1, 5, 15, 30, 60, 120, 300, 600, 1800, 3600)); err != nil {
	92	+	return err
	93	+	}
	94	+	lastSuccess, err := m.meter.Float64ObservableGauge("gordon.backup.last_success_timestamp_seconds",
	95	+	metric.WithDescription("Start time (Unix) of the last successful backup per target"),
	96	+	metric.WithUnit("s"))
	97	+	if err != nil {
	98	+	return err
	99	+	}
	100	+	lastSize, err := m.meter.Float64ObservableGauge("gordon.backup.last_size_bytes",
	101	+	metric.WithDescription("Size of the last successful backup per target"),
	102	+	metric.WithUnit("By"))
	103	+	if err != nil {
	104	+	return err
	105	+	}
	106	+	nextRun, err := m.meter.Float64ObservableGauge("gordon.backup.next_run_timestamp_seconds",
	107	+	metric.WithDescription("Unix time of the next scheduled backup run"),
	108	+	metric.WithUnit("s"))
	109	+	if err != nil {
	110	+	return err
	111	+	}
	112	+	_, err = m.meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
	113	+	m.backups.mu.Lock()
	114	+	defer m.backups.mu.Unlock()
	115	+	for key, last := range m.backups.last {
	116	+	attrs := metric.WithAttributes(targetAttrs(key)...)
	117	+	o.ObserveFloat64(lastSuccess, float64(last.at.Unix()), attrs)
	118	+	o.ObserveFloat64(lastSize, float64(last.size), attrs)
	119	+	}
	120	+	for kind, next := range m.backups.nextRun {
	121	+	if at := next(); !at.IsZero() {
	122	+	o.ObserveFloat64(nextRun, float64(at.Unix()), metric.WithAttributes(attribute.String("kind", string(kind))))
	123	+	}
	124	+	}
	125	+	return nil
	126	+	}, lastSuccess, lastSize, nextRun)
	127	+	return err
	128	+	}
	129	+
	130	+	// RecordBackupRun implements out.BackupMetrics.
	131	+	func (m *Metrics) RecordBackupRun(ctx context.Context, run domain.BackupRunRecord) {
	132	+	result := "failure"
	133	+	if run.Succeeded {
	134	+	result = "success"
	135	+	}
	136	+	attrs := targetAttrs(run.BackupTargetRef)
	137	+	m.BackupRuns.Add(ctx, 1, metric.WithAttributes(append(attrs, attribute.String("result", result))...))
	138	+	m.BackupDuration.Record(ctx, run.Duration.Seconds(), metric.WithAttributes(attrs...))
	139	+	m.backups.remember(run)
	140	+	}
	141	+
	142	+	// ObserveStoredBackup implements out.BackupMetrics.
	143	+	func (m *Metrics) ObserveStoredBackup(run domain.BackupRunRecord) {
	144	+	m.backups.remember(run)
	145	+	}
	146	+
	147	+	// RetainBackupTargets implements out.BackupMetrics.
	148	+	func (m *Metrics) RetainBackupTargets(kind domain.BackupKind, targets []domain.BackupTargetRef, asOf time.Time) {
	149	+	m.backups.retain(kind, targets, asOf)
	150	+	}
	151	+
	152	+	// ObserveBackupNextRun exports the next planned run of one backup kind.
	153	+	// next is called at each collection; a zero time is not exported.
	154	+	func (m *Metrics) ObserveBackupNextRun(kind domain.BackupKind, next func() time.Time) {
	155	+	m.backups.mu.Lock()
	156	+	defer m.backups.mu.Unlock()
	157	+	m.backups.nextRun[kind] = next
	158	+	}

@@ -0,0 +1,158 @@
	1	+	package telemetry
	2	+
	3	+	import (
	4	+	"context"
	5	+	"sync"
	6	+	"time"
	7	+
	8	+	"go.opentelemetry.io/otel/attribute"
	9	+	"go.opentelemetry.io/otel/metric"
	10	+
	11	+	"github.com/bnema/gordon/internal/boundaries/out"
	12	+	"github.com/bnema/gordon/internal/domain"
	13	+	)
	14	+
	15	+	var _ out.BackupMetrics = (*Metrics)(nil)
	16	+
	17	+	func targetAttrs(ref domain.BackupTargetRef) []attribute.KeyValue {
	18	+	return []attribute.KeyValue{
	19	+	attribute.String("kind", string(ref.Kind)),
	20	+	attribute.String("app", ref.App),
	21	+	attribute.String("service", ref.Service),
	22	+	attribute.String("target", ref.Target),
	23	+	}
	24	+	}
	25	+
	26	+	type backupLastSuccess struct {
	27	+	at time.Time
	28	+	size int64
	29	+	updated time.Time // when this entry was written, for retain
	30	+	}
	31	+
	32	+	// backupState holds the last success per target, exported by observable
	33	+	// gauges at each collection.
	34	+	type backupState struct {
	35	+	now func() time.Time
	36	+	mu sync.Mutex
	37	+	last map[domain.BackupTargetRef]backupLastSuccess
	38	+	nextRun map[domain.BackupKind]func() time.Time
	39	+	}
	40	+
	41	+	func newBackupState() *backupState {
	42	+	return &backupState{
	43	+	now: time.Now,
	44	+	last: map[domain.BackupTargetRef]backupLastSuccess{},
	45	+	nextRun: map[domain.BackupKind]func() time.Time{},
	46	+	}
	47	+	}
	48	+
	49	+	// remember stores a success unless a newer one is already known.
	50	+	func (s *backupState) remember(run domain.BackupRunRecord) {
	51	+	if !run.Succeeded \|\| run.StartedAt.IsZero() {
	52	+	return
	53	+	}
	54	+	key := run.BackupTargetRef
	55	+	s.mu.Lock()
	56	+	defer s.mu.Unlock()
	57	+	if prev, ok := s.last[key]; ok && !run.StartedAt.After(prev.at) {
	58	+	return
	59	+	}
	60	+	s.last[key] = backupLastSuccess{at: run.StartedAt, size: run.SizeBytes, updated: s.now()}
	61	+	}
	62	+
	63	+	// retain drops the last success of every target of kind not in targets,
	64	+	// unless it was updated after asOf.
	65	+	func (s *backupState) retain(kind domain.BackupKind, targets []domain.BackupTargetRef, asOf time.Time) {
	66	+	keep := make(map[domain.BackupTargetRef]struct{}, len(targets))
	67	+	for _, ref := range targets {
	68	+	keep[ref] = struct{}{}
	69	+	}
	70	+	s.mu.Lock()
	71	+	defer s.mu.Unlock()
	72	+	for key, last := range s.last {
	73	+	if _, ok := keep[key]; key.Kind != kind \|\| ok \|\| last.updated.After(asOf) {
	74	+	continue
	75	+	}
	76	+	delete(s.last, key)
	77	+	}
	78	+	}
	79	+
	80	+	// registerBackupInstruments creates the backup counters, histogram, and
	81	+	// observable gauges.
	82	+	func (m *Metrics) registerBackupInstruments() error {
	83	+	var err error
	84	+	if m.BackupRuns, err = m.meter.Int64Counter("gordon.backup.runs",
	85	+	metric.WithDescription("Finished backup runs by result")); err != nil {
	86	+	return err
	87	+	}
	88	+	if m.BackupDuration, err = m.meter.Float64Histogram("gordon.backup.duration_seconds",
	89	+	metric.WithDescription("Backup run duration in seconds"),
	90	+	metric.WithUnit("s"),
	91	+	metric.WithExplicitBucketBoundaries(1, 5, 15, 30, 60, 120, 300, 600, 1800, 3600)); err != nil {
	92	+	return err
	93	+	}
	94	+	lastSuccess, err := m.meter.Float64ObservableGauge("gordon.backup.last_success_timestamp_seconds",
	95	+	metric.WithDescription("Start time (Unix) of the last successful backup per target"),
	96	+	metric.WithUnit("s"))
	97	+	if err != nil {
	98	+	return err
	99	+	}
	100	+	lastSize, err := m.meter.Float64ObservableGauge("gordon.backup.last_size_bytes",
	101	+	metric.WithDescription("Size of the last successful backup per target"),
	102	+	metric.WithUnit("By"))
	103	+	if err != nil {
	104	+	return err
	105	+	}
	106	+	nextRun, err := m.meter.Float64ObservableGauge("gordon.backup.next_run_timestamp_seconds",
	107	+	metric.WithDescription("Unix time of the next scheduled backup run"),
	108	+	metric.WithUnit("s"))
	109	+	if err != nil {
	110	+	return err
	111	+	}
	112	+	_, err = m.meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
	113	+	m.backups.mu.Lock()
	114	+	defer m.backups.mu.Unlock()
	115	+	for key, last := range m.backups.last {
	116	+	attrs := metric.WithAttributes(targetAttrs(key)...)
	117	+	o.ObserveFloat64(lastSuccess, float64(last.at.Unix()), attrs)
	118	+	o.ObserveFloat64(lastSize, float64(last.size), attrs)
	119	+	}
	120	+	for kind, next := range m.backups.nextRun {
	121	+	if at := next(); !at.IsZero() {
	122	+	o.ObserveFloat64(nextRun, float64(at.Unix()), metric.WithAttributes(attribute.String("kind", string(kind))))
	123	+	}
	124	+	}
	125	+	return nil
	126	+	}, lastSuccess, lastSize, nextRun)
	127	+	return err
	128	+	}
	129	+
	130	+	// RecordBackupRun implements out.BackupMetrics.
	131	+	func (m *Metrics) RecordBackupRun(ctx context.Context, run domain.BackupRunRecord) {
	132	+	result := "failure"
	133	+	if run.Succeeded {
	134	+	result = "success"
	135	+	}
	136	+	attrs := targetAttrs(run.BackupTargetRef)
	137	+	m.BackupRuns.Add(ctx, 1, metric.WithAttributes(append(attrs, attribute.String("result", result))...))
	138	+	m.BackupDuration.Record(ctx, run.Duration.Seconds(), metric.WithAttributes(attrs...))
	139	+	m.backups.remember(run)
	140	+	}
	141	+
	142	+	// ObserveStoredBackup implements out.BackupMetrics.
	143	+	func (m *Metrics) ObserveStoredBackup(run domain.BackupRunRecord) {
	144	+	m.backups.remember(run)
	145	+	}
	146	+
	147	+	// RetainBackupTargets implements out.BackupMetrics.
	148	+	func (m *Metrics) RetainBackupTargets(kind domain.BackupKind, targets []domain.BackupTargetRef, asOf time.Time) {
	149	+	m.backups.retain(kind, targets, asOf)
	150	+	}
	151	+
	152	+	// ObserveBackupNextRun exports the next planned run of one backup kind.
	153	+	// next is called at each collection; a zero time is not exported.
	154	+	func (m *Metrics) ObserveBackupNextRun(kind domain.BackupKind, next func() time.Time) {
	155	+	m.backups.mu.Lock()
	156	+	defer m.backups.mu.Unlock()
	157	+	m.backups.nextRun[kind] = next
	158	+	}

A internal/adapters/out/telemetry/backup_metrics_test.go +168 -0

@@ -0,0 +1,168 @@
	1	+	package telemetry
	2	+
	3	+	import (
	4	+	"context"
	5	+	"testing"
	6	+	"time"
	7	+
	8	+	"github.com/stretchr/testify/assert"
	9	+	"github.com/stretchr/testify/require"
	10	+	"go.opentelemetry.io/otel"
	11	+	"go.opentelemetry.io/otel/attribute"
	12	+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
	13	+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
	14	+
	15	+	"github.com/bnema/gordon/internal/domain"
	16	+	)
	17	+
	18	+	func newTestMetrics(t testing.T) (Metrics, *sdkmetric.ManualReader) {
	19	+	t.Helper()
	20	+	reader := sdkmetric.NewManualReader()
	21	+	prev := otel.GetMeterProvider()
	22	+	otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)))
	23	+	t.Cleanup(func() { otel.SetMeterProvider(prev) })
	24	+	m, err := NewMetrics()
	25	+	require.NoError(t, err)
	26	+	return m, reader
	27	+	}
	28	+
	29	+	func collect(t testing.T, reader sdkmetric.ManualReader) map[string]metricdata.Aggregation {
	30	+	t.Helper()
	31	+	var rm metricdata.ResourceMetrics
	32	+	require.NoError(t, reader.Collect(context.Background(), &rm))
	33	+	out := map[string]metricdata.Aggregation{}
	34	+	for _, sm := range rm.ScopeMetrics {
	35	+	for _, m := range sm.Metrics {
	36	+	out[m.Name] = m.Data
	37	+	}
	38	+	}
	39	+	return out
	40	+	}
	41	+
	42	+	func attrValue(set attribute.Set, key string) string {
	43	+	v, _ := set.Value(attribute.Key(key))
	44	+	return v.AsString()
	45	+	}
	46	+
	47	+	// gaugeByTarget indexes float gauge points by their target attribute.
	48	+	func gaugeByTarget(t *testing.T, data metricdata.Aggregation) map[string]float64 {
	49	+	t.Helper()
	50	+	gauge, ok := data.(metricdata.Gauge[float64])
	51	+	require.True(t, ok, "expected a float64 gauge, got %T", data)
	52	+	out := map[string]float64{}
	53	+	for _, dp := range gauge.DataPoints {
	54	+	out[attrValue(dp.Attributes, "target")] = dp.Value
	55	+	}
	56	+	return out
	57	+	}
	58	+
	59	+	func TestRecordBackupRun_CountsResultAndTracksLastSuccess(t *testing.T) {
	60	+	m, reader := newTestMetrics(t)
	61	+	ctx := context.Background()
	62	+	done := time.Date(2026, 10, 18, 1, 30, 0, 0, time.UTC)
	63	+
	64	+	ref := domain.BackupTargetRef{Kind: domain.BackupKindVolume, App: "forgejo", Service: "forgejo", Target: "forgejo-data"}
	65	+	m.RecordBackupRun(ctx, domain.BackupRunRecord{
	66	+	BackupTargetRef: ref, Succeeded: true, Duration: 40 * time.Second, SizeBytes: 1024, StartedAt: done,
	67	+	})
	68	+	m.RecordBackupRun(ctx, domain.BackupRunRecord{
	69	+	BackupTargetRef: ref, Succeeded: false, Duration: time.Second, StartedAt: done.Add(time.Hour),
	70	+	})
	71	+
	72	+	data := collect(t, reader)
	73	+
	74	+	runs, ok := data["gordon.backup.runs"].(metricdata.Sum[int64])
	75	+	require.True(t, ok)
	76	+	results := map[string]int64{}
	77	+	for _, dp := range runs.DataPoints {
	78	+	assert.Equal(t, "volume", attrValue(dp.Attributes, "kind"))
	79	+	assert.Equal(t, "forgejo", attrValue(dp.Attributes, "app"))
	80	+	assert.Equal(t, "forgejo", attrValue(dp.Attributes, "service"))
	81	+	assert.Equal(t, "forgejo-data", attrValue(dp.Attributes, "target"))
	82	+	results[attrValue(dp.Attributes, "result")] = dp.Value
	83	+	}
	84	+	assert.Equal(t, map[string]int64{"success": 1, "failure": 1}, results)
	85	+
	86	+	hist, ok := data["gordon.backup.duration_seconds"].(metricdata.Histogram[float64])
	87	+	require.True(t, ok)
	88	+	require.Len(t, hist.DataPoints, 1)
	89	+	assert.Equal(t, uint64(2), hist.DataPoints[0].Count)
	90	+	assert.InDelta(t, 41.0, hist.DataPoints[0].Sum, 0.001)
	91	+
	92	+	// A failure never moves the last-success timestamp or size.
	93	+	assert.Equal(t, map[string]float64{"forgejo-data": float64(done.Unix())},
	94	+	gaugeByTarget(t, data["gordon.backup.last_success_timestamp_seconds"]))
	95	+	assert.Equal(t, map[string]float64{"forgejo-data": 1024},
	96	+	gaugeByTarget(t, data["gordon.backup.last_size_bytes"]))
	97	+	}
	98	+
	99	+	func TestObserveStoredBackup_KeepsTheNewestSuccess(t *testing.T) {
	100	+	m, reader := newTestMetrics(t)
	101	+	newer := time.Date(2026, 10, 17, 1, 30, 0, 0, time.UTC)
	102	+	older := newer.Add(-24 * time.Hour)
	103	+	rec := domain.BackupRunRecord{
	104	+	BackupTargetRef: domain.BackupTargetRef{Kind: domain.BackupKindDatabase, App: "shop", Service: "db", Target: "orders"},
	105	+	Succeeded: true,
	106	+	}
	107	+
	108	+	rec.StartedAt, rec.SizeBytes = newer, 200
	109	+	m.ObserveStoredBackup(rec)
	110	+	rec.StartedAt, rec.SizeBytes = older, 100
	111	+	m.ObserveStoredBackup(rec)
	112	+
	113	+	data := collect(t, reader)
	114	+	assert.Equal(t, map[string]float64{"orders": float64(newer.Unix())},
	115	+	gaugeByTarget(t, data["gordon.backup.last_success_timestamp_seconds"]))
	116	+	assert.Equal(t, map[string]float64{"orders": 200},
	117	+	gaugeByTarget(t, data["gordon.backup.last_size_bytes"]))
	118	+
	119	+	// Seeding never counts a run.
	120	+	_, counted := data["gordon.backup.runs"]
	121	+	assert.False(t, counted)
	122	+	}
	123	+
	124	+	func TestRetainBackupTargets_ForgetsUndeclaredTargetsOfThatKindOnly(t *testing.T) {
	125	+	m, reader := newTestMetrics(t)
	126	+	at := time.Date(2026, 10, 17, 1, 30, 0, 0, time.UTC)
	127	+	keep := domain.BackupTargetRef{Kind: domain.BackupKindVolume, App: "shop", Service: "api", Target: "data"}
	128	+	gone := domain.BackupTargetRef{Kind: domain.BackupKindVolume, App: "old", Service: "api", Target: "cache"}
	129	+	db := domain.BackupTargetRef{Kind: domain.BackupKindDatabase, App: "old", Service: "db", Target: "orders"}
	130	+	for _, ref := range []domain.BackupTargetRef{keep, gone, db} {
	131	+	m.ObserveStoredBackup(domain.BackupRunRecord{BackupTargetRef: ref, Succeeded: true, StartedAt: at})
	132	+	}
	133	+
	134	+	m.RetainBackupTargets(domain.BackupKindVolume, []domain.BackupTargetRef{keep}, time.Now())
	135	+
	136	+	assert.Equal(t, map[string]float64{"data": float64(at.Unix()), "orders": float64(at.Unix())},
	137	+	gaugeByTarget(t, collect(t, reader)["gordon.backup.last_success_timestamp_seconds"]))
	138	+	}
	139	+
	140	+	// TestRetainBackupTargets_KeepsSuccessRecordedAfterTheSnapshot covers a
	141	+	// target declared and backed up while seeding read an older declaration.
	142	+	func TestRetainBackupTargets_KeepsSuccessRecordedAfterTheSnapshot(t *testing.T) {
	143	+	m, reader := newTestMetrics(t)
	144	+	clock := time.Date(2026, 10, 17, 8, 0, 0, 0, time.UTC)
	145	+	m.backups.now = func() time.Time { return clock }
	146	+	snapshot := clock
	147	+	clock = clock.Add(time.Second)
	148	+	fresh := domain.BackupTargetRef{Kind: domain.BackupKindVolume, App: "new", Service: "api", Target: "data"}
	149	+	m.RecordBackupRun(context.Background(), domain.BackupRunRecord{BackupTargetRef: fresh, Succeeded: true, StartedAt: clock})
	150	+
	151	+	m.RetainBackupTargets(domain.BackupKindVolume, nil, snapshot)
	152	+
	153	+	assert.Equal(t, map[string]float64{"data": float64(clock.Unix())},
	154	+	gaugeByTarget(t, collect(t, reader)["gordon.backup.last_success_timestamp_seconds"]))
	155	+	}
	156	+
	157	+	func TestObserveBackupNextRun(t *testing.T) {
	158	+	m, reader := newTestMetrics(t)
	159	+	next := time.Date(2026, 10, 18, 1, 30, 0, 0, time.UTC)
	160	+	m.ObserveBackupNextRun(domain.BackupKindVolume, func() time.Time { return next })
	161	+	m.ObserveBackupNextRun(domain.BackupKindDatabase, func() time.Time { return time.Time{} })
	162	+
	163	+	gauge, ok := collect(t, reader)["gordon.backup.next_run_timestamp_seconds"].(metricdata.Gauge[float64])
	164	+	require.True(t, ok)
	165	+	require.Len(t, gauge.DataPoints, 1, "an unknown next run is not exported")
	166	+	assert.Equal(t, "volume", attrValue(gauge.DataPoints[0].Attributes, "kind"))
	167	+	assert.Equal(t, float64(next.Unix()), gauge.DataPoints[0].Value)
	168	+	}

@@ -0,0 +1,168 @@
	1	+	package telemetry
	2	+
	3	+	import (
	4	+	"context"
	5	+	"testing"
	6	+	"time"
	7	+
	8	+	"github.com/stretchr/testify/assert"
	9	+	"github.com/stretchr/testify/require"
	10	+	"go.opentelemetry.io/otel"
	11	+	"go.opentelemetry.io/otel/attribute"
	12	+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
	13	+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
	14	+
	15	+	"github.com/bnema/gordon/internal/domain"
	16	+	)
	17	+
	18	+	func newTestMetrics(t testing.T) (Metrics, *sdkmetric.ManualReader) {
	19	+	t.Helper()
	20	+	reader := sdkmetric.NewManualReader()
	21	+	prev := otel.GetMeterProvider()
	22	+	otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)))
	23	+	t.Cleanup(func() { otel.SetMeterProvider(prev) })
	24	+	m, err := NewMetrics()
	25	+	require.NoError(t, err)
	26	+	return m, reader
	27	+	}
	28	+
	29	+	func collect(t testing.T, reader sdkmetric.ManualReader) map[string]metricdata.Aggregation {
	30	+	t.Helper()
	31	+	var rm metricdata.ResourceMetrics
	32	+	require.NoError(t, reader.Collect(context.Background(), &rm))
	33	+	out := map[string]metricdata.Aggregation{}
	34	+	for _, sm := range rm.ScopeMetrics {
	35	+	for _, m := range sm.Metrics {
	36	+	out[m.Name] = m.Data
	37	+	}
	38	+	}
	39	+	return out
	40	+	}
	41	+
	42	+	func attrValue(set attribute.Set, key string) string {
	43	+	v, _ := set.Value(attribute.Key(key))
	44	+	return v.AsString()
	45	+	}
	46	+
	47	+	// gaugeByTarget indexes float gauge points by their target attribute.
	48	+	func gaugeByTarget(t *testing.T, data metricdata.Aggregation) map[string]float64 {
	49	+	t.Helper()
	50	+	gauge, ok := data.(metricdata.Gauge[float64])
	51	+	require.True(t, ok, "expected a float64 gauge, got %T", data)
	52	+	out := map[string]float64{}
	53	+	for _, dp := range gauge.DataPoints {
	54	+	out[attrValue(dp.Attributes, "target")] = dp.Value
	55	+	}
	56	+	return out
	57	+	}
	58	+
	59	+	func TestRecordBackupRun_CountsResultAndTracksLastSuccess(t *testing.T) {
	60	+	m, reader := newTestMetrics(t)
	61	+	ctx := context.Background()
	62	+	done := time.Date(2026, 10, 18, 1, 30, 0, 0, time.UTC)
	63	+
	64	+	ref := domain.BackupTargetRef{Kind: domain.BackupKindVolume, App: "forgejo", Service: "forgejo", Target: "forgejo-data"}
	65	+	m.RecordBackupRun(ctx, domain.BackupRunRecord{
	66	+	BackupTargetRef: ref, Succeeded: true, Duration: 40 * time.Second, SizeBytes: 1024, StartedAt: done,
	67	+	})
	68	+	m.RecordBackupRun(ctx, domain.BackupRunRecord{
	69	+	BackupTargetRef: ref, Succeeded: false, Duration: time.Second, StartedAt: done.Add(time.Hour),
	70	+	})
	71	+
	72	+	data := collect(t, reader)
	73	+
	74	+	runs, ok := data["gordon.backup.runs"].(metricdata.Sum[int64])
	75	+	require.True(t, ok)
	76	+	results := map[string]int64{}
	77	+	for _, dp := range runs.DataPoints {
	78	+	assert.Equal(t, "volume", attrValue(dp.Attributes, "kind"))
	79	+	assert.Equal(t, "forgejo", attrValue(dp.Attributes, "app"))
	80	+	assert.Equal(t, "forgejo", attrValue(dp.Attributes, "service"))
	81	+	assert.Equal(t, "forgejo-data", attrValue(dp.Attributes, "target"))
	82	+	results[attrValue(dp.Attributes, "result")] = dp.Value
	83	+	}
	84	+	assert.Equal(t, map[string]int64{"success": 1, "failure": 1}, results)
	85	+
	86	+	hist, ok := data["gordon.backup.duration_seconds"].(metricdata.Histogram[float64])
	87	+	require.True(t, ok)
	88	+	require.Len(t, hist.DataPoints, 1)
	89	+	assert.Equal(t, uint64(2), hist.DataPoints[0].Count)
	90	+	assert.InDelta(t, 41.0, hist.DataPoints[0].Sum, 0.001)
	91	+
	92	+	// A failure never moves the last-success timestamp or size.
	93	+	assert.Equal(t, map[string]float64{"forgejo-data": float64(done.Unix())},
	94	+	gaugeByTarget(t, data["gordon.backup.last_success_timestamp_seconds"]))
	95	+	assert.Equal(t, map[string]float64{"forgejo-data": 1024},
	96	+	gaugeByTarget(t, data["gordon.backup.last_size_bytes"]))
	97	+	}
	98	+
	99	+	func TestObserveStoredBackup_KeepsTheNewestSuccess(t *testing.T) {
	100	+	m, reader := newTestMetrics(t)
	101	+	newer := time.Date(2026, 10, 17, 1, 30, 0, 0, time.UTC)
	102	+	older := newer.Add(-24 * time.Hour)
	103	+	rec := domain.BackupRunRecord{
	104	+	BackupTargetRef: domain.BackupTargetRef{Kind: domain.BackupKindDatabase, App: "shop", Service: "db", Target: "orders"},
	105	+	Succeeded: true,
	106	+	}
	107	+
	108	+	rec.StartedAt, rec.SizeBytes = newer, 200
	109	+	m.ObserveStoredBackup(rec)
	110	+	rec.StartedAt, rec.SizeBytes = older, 100
	111	+	m.ObserveStoredBackup(rec)
	112	+
	113	+	data := collect(t, reader)
	114	+	assert.Equal(t, map[string]float64{"orders": float64(newer.Unix())},
	115	+	gaugeByTarget(t, data["gordon.backup.last_success_timestamp_seconds"]))
	116	+	assert.Equal(t, map[string]float64{"orders": 200},
	117	+	gaugeByTarget(t, data["gordon.backup.last_size_bytes"]))
	118	+
	119	+	// Seeding never counts a run.
	120	+	_, counted := data["gordon.backup.runs"]
	121	+	assert.False(t, counted)
	122	+	}
	123	+
	124	+	func TestRetainBackupTargets_ForgetsUndeclaredTargetsOfThatKindOnly(t *testing.T) {
	125	+	m, reader := newTestMetrics(t)
	126	+	at := time.Date(2026, 10, 17, 1, 30, 0, 0, time.UTC)
	127	+	keep := domain.BackupTargetRef{Kind: domain.BackupKindVolume, App: "shop", Service: "api", Target: "data"}
	128	+	gone := domain.BackupTargetRef{Kind: domain.BackupKindVolume, App: "old", Service: "api", Target: "cache"}
	129	+	db := domain.BackupTargetRef{Kind: domain.BackupKindDatabase, App: "old", Service: "db", Target: "orders"}
	130	+	for _, ref := range []domain.BackupTargetRef{keep, gone, db} {
	131	+	m.ObserveStoredBackup(domain.BackupRunRecord{BackupTargetRef: ref, Succeeded: true, StartedAt: at})
	132	+	}
	133	+
	134	+	m.RetainBackupTargets(domain.BackupKindVolume, []domain.BackupTargetRef{keep}, time.Now())
	135	+
	136	+	assert.Equal(t, map[string]float64{"data": float64(at.Unix()), "orders": float64(at.Unix())},
	137	+	gaugeByTarget(t, collect(t, reader)["gordon.backup.last_success_timestamp_seconds"]))
	138	+	}
	139	+
	140	+	// TestRetainBackupTargets_KeepsSuccessRecordedAfterTheSnapshot covers a
	141	+	// target declared and backed up while seeding read an older declaration.
	142	+	func TestRetainBackupTargets_KeepsSuccessRecordedAfterTheSnapshot(t *testing.T) {
	143	+	m, reader := newTestMetrics(t)
	144	+	clock := time.Date(2026, 10, 17, 8, 0, 0, 0, time.UTC)
	145	+	m.backups.now = func() time.Time { return clock }
	146	+	snapshot := clock
	147	+	clock = clock.Add(time.Second)
	148	+	fresh := domain.BackupTargetRef{Kind: domain.BackupKindVolume, App: "new", Service: "api", Target: "data"}
	149	+	m.RecordBackupRun(context.Background(), domain.BackupRunRecord{BackupTargetRef: fresh, Succeeded: true, StartedAt: clock})
	150	+
	151	+	m.RetainBackupTargets(domain.BackupKindVolume, nil, snapshot)
	152	+
	153	+	assert.Equal(t, map[string]float64{"data": float64(clock.Unix())},
	154	+	gaugeByTarget(t, collect(t, reader)["gordon.backup.last_success_timestamp_seconds"]))
	155	+	}
	156	+
	157	+	func TestObserveBackupNextRun(t *testing.T) {
	158	+	m, reader := newTestMetrics(t)
	159	+	next := time.Date(2026, 10, 18, 1, 30, 0, 0, time.UTC)
	160	+	m.ObserveBackupNextRun(domain.BackupKindVolume, func() time.Time { return next })
	161	+	m.ObserveBackupNextRun(domain.BackupKindDatabase, func() time.Time { return time.Time{} })
	162	+
	163	+	gauge, ok := collect(t, reader)["gordon.backup.next_run_timestamp_seconds"].(metricdata.Gauge[float64])
	164	+	require.True(t, ok)
	165	+	require.Len(t, gauge.DataPoints, 1, "an unknown next run is not exported")
	166	+	assert.Equal(t, "volume", attrValue(gauge.DataPoints[0].Attributes, "kind"))
	167	+	assert.Equal(t, float64(next.Unix()), gauge.DataPoints[0].Value)
	168	+	}

M internal/adapters/out/telemetry/metrics.go +9 -1


@@ -33,6 +33,11 @@ type Metrics struct {
33	33		// Events
34	34		EventsProcessed metric.Int64Counter
35	35		EventsDropped metric.Int64Counter
	36	+
	37	+	// Backups
	38	+	BackupRuns metric.Int64Counter
	39	+	BackupDuration metric.Float64Histogram
	40	+	backups *backupState
36	41		}
37	42
38	43		// NewMetrics creates and registers all Gordon metric instruments.

@@ -40,7 +45,7 @@ type Metrics struct {
40	45		// (OTel returns noop instruments when no MeterProvider is set).
41	46		func NewMetrics() (*Metrics, error) {
42	47		meter := otel.Meter("gordon")
43		-	m := &Metrics{meter: meter}
	48	+	m := &Metrics{meter: meter, backups: newBackupState()}
44	49		var err error
45	50
46	51		if m.DeployTotal, err = meter.Int64Counter("gordon.deploy.total",

@@ -82,6 +87,9 @@ func NewMetrics() (*Metrics, error) {
82	87		metric.WithDescription("Total events dropped")); err != nil {
83	88		return nil, err
84	89		}
	90	+	if err := m.registerBackupInstruments(); err != nil {
	91	+	return nil, err
	92	+	}
85	93
86	94		return m, nil
87	95		}

 	// Events
 	EventsProcessed metric.Int64Counter
 	EventsDropped   metric.Int64Counter
+	// Backups
+	BackupRuns     metric.Int64Counter
+	BackupDuration metric.Float64Histogram
+	backups        *backupState
 }
 // NewMetrics creates and registers all Gordon metric instruments.
 // (OTel returns noop instruments when no MeterProvider is set).
 func NewMetrics() (*Metrics, error) {
 	meter := otel.Meter("gordon")
-	m := &Metrics{meter: meter}
+	m := &Metrics{meter: meter, backups: newBackupState()}
 	var err error
 	if m.DeployTotal, err = meter.Int64Counter("gordon.deploy.total",
 		metric.WithDescription("Total events dropped")); err != nil {
 		return nil, err
 	}
+	if err := m.registerBackupInstruments(); err != nil {
+		return nil, err
+	}
 	return m, nil
 }

A internal/app/backup_metrics_wiring.go +71 -0

@@ -0,0 +1,71 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+	"time"
	6	+
	7	+	"github.com/bnema/zerowrap"
	8	+
	9	+	"github.com/bnema/gordon/internal/adapters/out/telemetry"
	10	+	"github.com/bnema/gordon/internal/domain"
	11	+	cronSvc "github.com/bnema/gordon/internal/usecase/cron"
	12	+	)
	13	+
	14	+	// nextRunObserver exports the next planned run of one backup kind.
	15	+	type nextRunObserver interface {
	16	+	ObserveBackupNextRun(kind domain.BackupKind, next func() time.Time)
	17	+	}
	18	+
	19	+	// nextRunObserverPort converts a possibly nil metrics adapter into an
	20	+	// interface that is nil when metrics are disabled. A typed nil must not
	21	+	// leak into the interface.
	22	+	func nextRunObserverPort(metrics *telemetry.Metrics) nextRunObserver {
	23	+	if metrics == nil {
	24	+	return nil
	25	+	}
	26	+	return metrics
	27	+	}
	28	+
	29	+	// wireBackupMetrics injects backup metrics into the backup services. It
	30	+	// must run before any backup can start, so no run reads the field while
	31	+	// it is written.
	32	+	func wireBackupMetrics(svc *services) {
	33	+	if svc == nil \|\| svc.metrics == nil {
	34	+	return
	35	+	}
	36	+	if svc.backupSvc != nil {
	37	+	svc.backupSvc.WithMetrics(svc.metrics)
	38	+	}
	39	+	if svc.volumeBackupSvc != nil {
	40	+	svc.volumeBackupSvc.WithMetrics(svc.metrics)
	41	+	}
	42	+	}
	43	+
	44	+	// seedBackupMetrics reports the last success of every target from
	45	+	// storage, so the metric is correct right after a restart. It only feeds
	46	+	// the metrics state (mutex-guarded), so it may run alongside backups.
	47	+	// Failures are logged, never fatal.
	48	+	func seedBackupMetrics(ctx context.Context, svc *services, log zerowrap.Logger) {
	49	+	if svc == nil \|\| svc.metrics == nil {
	50	+	return
	51	+	}
	52	+	if svc.backupSvc != nil {
	53	+	if err := svc.backupSvc.SeedMetrics(ctx); err != nil {
	54	+	log.Warn().Err(err).Msg("failed to seed database backup metrics")
	55	+	}
	56	+	}
	57	+	if svc.volumeBackupSvc != nil {
	58	+	if err := svc.volumeBackupSvc.SeedMetrics(ctx); err != nil {
	59	+	log.Warn().Err(err).Msg("failed to seed volume backup metrics")
	60	+	}
	61	+	}
	62	+	}
	63	+
	64	+	// observeSchedulerNextRun exports the scheduler entry's next run, read at
	65	+	// each metrics collection.
	66	+	func observeSchedulerNextRun(obs nextRunObserver, kind domain.BackupKind, scheduler *cronSvc.Scheduler, id string) {
	67	+	if obs == nil \|\| scheduler == nil {
	68	+	return
	69	+	}
	70	+	obs.ObserveBackupNextRun(kind, func() time.Time { return nextRunOf(scheduler, id) })
	71	+	}

@@ -0,0 +1,71 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+	"time"
	6	+
	7	+	"github.com/bnema/zerowrap"
	8	+
	9	+	"github.com/bnema/gordon/internal/adapters/out/telemetry"
	10	+	"github.com/bnema/gordon/internal/domain"
	11	+	cronSvc "github.com/bnema/gordon/internal/usecase/cron"
	12	+	)
	13	+
	14	+	// nextRunObserver exports the next planned run of one backup kind.
	15	+	type nextRunObserver interface {
	16	+	ObserveBackupNextRun(kind domain.BackupKind, next func() time.Time)
	17	+	}
	18	+
	19	+	// nextRunObserverPort converts a possibly nil metrics adapter into an
	20	+	// interface that is nil when metrics are disabled. A typed nil must not
	21	+	// leak into the interface.
	22	+	func nextRunObserverPort(metrics *telemetry.Metrics) nextRunObserver {
	23	+	if metrics == nil {
	24	+	return nil
	25	+	}
	26	+	return metrics
	27	+	}
	28	+
	29	+	// wireBackupMetrics injects backup metrics into the backup services. It
	30	+	// must run before any backup can start, so no run reads the field while
	31	+	// it is written.
	32	+	func wireBackupMetrics(svc *services) {
	33	+	if svc == nil \|\| svc.metrics == nil {
	34	+	return
	35	+	}
	36	+	if svc.backupSvc != nil {
	37	+	svc.backupSvc.WithMetrics(svc.metrics)
	38	+	}
	39	+	if svc.volumeBackupSvc != nil {
	40	+	svc.volumeBackupSvc.WithMetrics(svc.metrics)
	41	+	}
	42	+	}
	43	+
	44	+	// seedBackupMetrics reports the last success of every target from
	45	+	// storage, so the metric is correct right after a restart. It only feeds
	46	+	// the metrics state (mutex-guarded), so it may run alongside backups.
	47	+	// Failures are logged, never fatal.
	48	+	func seedBackupMetrics(ctx context.Context, svc *services, log zerowrap.Logger) {
	49	+	if svc == nil \|\| svc.metrics == nil {
	50	+	return
	51	+	}
	52	+	if svc.backupSvc != nil {
	53	+	if err := svc.backupSvc.SeedMetrics(ctx); err != nil {
	54	+	log.Warn().Err(err).Msg("failed to seed database backup metrics")
	55	+	}
	56	+	}
	57	+	if svc.volumeBackupSvc != nil {
	58	+	if err := svc.volumeBackupSvc.SeedMetrics(ctx); err != nil {
	59	+	log.Warn().Err(err).Msg("failed to seed volume backup metrics")
	60	+	}
	61	+	}
	62	+	}
	63	+
	64	+	// observeSchedulerNextRun exports the scheduler entry's next run, read at
	65	+	// each metrics collection.
	66	+	func observeSchedulerNextRun(obs nextRunObserver, kind domain.BackupKind, scheduler *cronSvc.Scheduler, id string) {
	67	+	if obs == nil \|\| scheduler == nil {
	68	+	return
	69	+	}
	70	+	obs.ObserveBackupNextRun(kind, func() time.Time { return nextRunOf(scheduler, id) })
	71	+	}

A internal/app/backup_metrics_wiring_test.go +49 -0

@@ -0,0 +1,49 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+	"testing"
	6	+	"time"
	7	+
	8	+	"github.com/bnema/zerowrap"
	9	+	"github.com/stretchr/testify/assert"
	10	+	"github.com/stretchr/testify/require"
	11	+
	12	+	"github.com/bnema/gordon/internal/domain"
	13	+	cronSvc "github.com/bnema/gordon/internal/usecase/cron"
	14	+	)
	15	+
	16	+	type fakeNextRunObserver struct {
	17	+	got map[domain.BackupKind]func() time.Time
	18	+	}
	19	+
	20	+	func (f *fakeNextRunObserver) ObserveBackupNextRun(kind domain.BackupKind, next func() time.Time) {
	21	+	if f.got == nil {
	22	+	f.got = map[domain.BackupKind]func() time.Time{}
	23	+	}
	24	+	f.got[kind] = next
	25	+	}
	26	+
	27	+	func TestObserveSchedulerNextRun_ReadsTheLiveEntry(t *testing.T) {
	28	+	scheduler := cronSvc.NewScheduler(zerowrap.Default())
	29	+	require.NoError(t, scheduler.Add("volume-backup-scheduler", "Volume Backups",
	30	+	domain.CronSchedule{Preset: domain.ScheduleDaily}, func(context.Context) error { return nil }))
	31	+
	32	+	obs := &fakeNextRunObserver{}
	33	+	observeSchedulerNextRun(obs, domain.BackupKindVolume, scheduler, "volume-backup-scheduler")
	34	+
	35	+	require.Contains(t, obs.got, domain.BackupKindVolume)
	36	+	next := obs.got[domain.BackupKindVolume]()
	37	+	assert.Equal(t, nextRunOf(scheduler, "volume-backup-scheduler"), next)
	38	+	assert.True(t, next.After(time.Now()))
	39	+	}
	40	+
	41	+	func TestObserveSchedulerNextRun_NilSafe(t *testing.T) {
	42	+	scheduler := cronSvc.NewScheduler(zerowrap.Default())
	43	+	assert.NotPanics(t, func() {
	44	+	observeSchedulerNextRun(nil, domain.BackupKindVolume, scheduler, "x")
	45	+	})
	46	+	obs := &fakeNextRunObserver{}
	47	+	observeSchedulerNextRun(obs, domain.BackupKindVolume, nil, "x")
	48	+	assert.Empty(t, obs.got)
	49	+	}

@@ -0,0 +1,49 @@
	1	+	package app
	2	+
	3	+	import (
	4	+	"context"
	5	+	"testing"
	6	+	"time"
	7	+
	8	+	"github.com/bnema/zerowrap"
	9	+	"github.com/stretchr/testify/assert"
	10	+	"github.com/stretchr/testify/require"
	11	+
	12	+	"github.com/bnema/gordon/internal/domain"
	13	+	cronSvc "github.com/bnema/gordon/internal/usecase/cron"
	14	+	)
	15	+
	16	+	type fakeNextRunObserver struct {
	17	+	got map[domain.BackupKind]func() time.Time
	18	+	}
	19	+
	20	+	func (f *fakeNextRunObserver) ObserveBackupNextRun(kind domain.BackupKind, next func() time.Time) {
	21	+	if f.got == nil {
	22	+	f.got = map[domain.BackupKind]func() time.Time{}
	23	+	}
	24	+	f.got[kind] = next
	25	+	}
	26	+
	27	+	func TestObserveSchedulerNextRun_ReadsTheLiveEntry(t *testing.T) {
	28	+	scheduler := cronSvc.NewScheduler(zerowrap.Default())
	29	+	require.NoError(t, scheduler.Add("volume-backup-scheduler", "Volume Backups",
	30	+	domain.CronSchedule{Preset: domain.ScheduleDaily}, func(context.Context) error { return nil }))
	31	+
	32	+	obs := &fakeNextRunObserver{}
	33	+	observeSchedulerNextRun(obs, domain.BackupKindVolume, scheduler, "volume-backup-scheduler")
	34	+
	35	+	require.Contains(t, obs.got, domain.BackupKindVolume)
	36	+	next := obs.got[domain.BackupKindVolume]()
	37	+	assert.Equal(t, nextRunOf(scheduler, "volume-backup-scheduler"), next)
	38	+	assert.True(t, next.After(time.Now()))
	39	+	}
	40	+
	41	+	func TestObserveSchedulerNextRun_NilSafe(t *testing.T) {
	42	+	scheduler := cronSvc.NewScheduler(zerowrap.Default())
	43	+	assert.NotPanics(t, func() {
	44	+	observeSchedulerNextRun(nil, domain.BackupKindVolume, scheduler, "x")
	45	+	})
	46	+	obs := &fakeNextRunObserver{}
	47	+	observeSchedulerNextRun(obs, domain.BackupKindVolume, nil, "x")
	48	+	assert.Empty(t, obs.got)
	49	+	}

M internal/app/run.go +3 -0


@@ -296,6 +296,9 @@ func runServers(ctx context.Context, v viper.Viper, cfg Config, svc services,
296	296
297	297		startPublicTLSRuntimeWithWarning(ctx, svc.publicTLSRuntime, log)
298	298
	299	+	// Seeding lists stored backups (S3 for volumes); never delay startup.
	300	+	go seedBackupMetrics(ctx, svc, log)
	301	+
299	302		schedulerCleanup, err := startOptionalSchedulers(ctx, cfg, svc, log, v)
300	303		if err != nil {
301	304		cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)


@@ -296,6 +296,9 @@ func runServers(ctx context.Context, v viper.Viper, cfg Config, svc services,
296		296
297	startPublicTLSRuntimeWithWarning(ctx, svc.publicTLSRuntime, log)	297		startPublicTLSRuntimeWithWarning(ctx, svc.publicTLSRuntime, log)
298		298
		299	+	// Seeding lists stored backups (S3 for volumes); never delay startup.
		300	+	go seedBackupMetrics(ctx, svc, log)
		301	+
299	schedulerCleanup, err := startOptionalSchedulers(ctx, cfg, svc, log, v)	302		schedulerCleanup, err := startOptionalSchedulers(ctx, cfg, svc, log, v)
300	if err != nil {	303		if err != nil {
301	cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)	304		cleanupAfterLocalAdmin(registrySrv, internalRegistrySrv, proxySrv, tlsSrv)

M internal/app/schedulers.go +2 -0


@@ -27,6 +27,7 @@ func startOptionalSchedulers(ctx context.Context, cfg Config, svc *services, log
27	27		}
28	28		if backupScheduler != nil {
29	29		schedulers = append(schedulers, backupScheduler)
	30	+	observeSchedulerNextRun(nextRunObserverPort(svc.metrics), domain.BackupKindDatabase, backupScheduler, "backup-scheduler")
30	31		}
31	32
32	33		volumeBackupScheduler, err := startVolumeBackupScheduler(ctx, cfg, svc, log)

@@ -36,6 +37,7 @@ func startOptionalSchedulers(ctx context.Context, cfg Config, svc *services, log
36	37		}
37	38		if volumeBackupScheduler != nil {
38	39		schedulers = append(schedulers, volumeBackupScheduler)
	40	+	observeSchedulerNextRun(nextRunObserverPort(svc.metrics), domain.BackupKindVolume, volumeBackupScheduler, "volume-backup-scheduler")
39	41		}
40	42
41	43		imageScheduler, err := startImagePruneScheduler(ctx, cfg, svc, log, func() int {


@@ -27,6 +27,7 @@ func startOptionalSchedulers(ctx context.Context, cfg Config, svc *services, log
27	}	27		}
28	if backupScheduler != nil {	28		if backupScheduler != nil {
29	schedulers = append(schedulers, backupScheduler)	29		schedulers = append(schedulers, backupScheduler)
		30	+	observeSchedulerNextRun(nextRunObserverPort(svc.metrics), domain.BackupKindDatabase, backupScheduler, "backup-scheduler")
30	}	31		}
31		32
32	volumeBackupScheduler, err := startVolumeBackupScheduler(ctx, cfg, svc, log)	33		volumeBackupScheduler, err := startVolumeBackupScheduler(ctx, cfg, svc, log)

@@ -36,6 +37,7 @@ func startOptionalSchedulers(ctx context.Context, cfg Config, svc *services, log
36	}	37		}
37	if volumeBackupScheduler != nil {	38		if volumeBackupScheduler != nil {
38	schedulers = append(schedulers, volumeBackupScheduler)	39		schedulers = append(schedulers, volumeBackupScheduler)
		40	+	observeSchedulerNextRun(nextRunObserverPort(svc.metrics), domain.BackupKindVolume, volumeBackupScheduler, "volume-backup-scheduler")
39	}	41		}
40		42
41	imageScheduler, err := startImagePruneScheduler(ctx, cfg, svc, log, func() int {	43		imageScheduler, err := startImagePruneScheduler(ctx, cfg, svc, log, func() int {

M internal/app/services.go +1 -0


@@ -483,6 +483,7 @@ func injectTelemetryMetrics(cfg Config, svc *services, log zerowrap.Logger) {
483	483		svc.registrySvc.SetMetrics(gordonMetrics)
484	484		svc.eventBus.SetMetrics(gordonMetrics)
485	485		svc.metrics = gordonMetrics
	486	+	wireBackupMetrics(svc)
486	487		}
487	488
488	489		// observeManagedContainers exports the managed-container gauge from app


@@ -483,6 +483,7 @@ func injectTelemetryMetrics(cfg Config, svc *services, log zerowrap.Logger) {
483	svc.registrySvc.SetMetrics(gordonMetrics)	483		svc.registrySvc.SetMetrics(gordonMetrics)
484	svc.eventBus.SetMetrics(gordonMetrics)	484		svc.eventBus.SetMetrics(gordonMetrics)
485	svc.metrics = gordonMetrics	485		svc.metrics = gordonMetrics
		486	+	wireBackupMetrics(svc)
486	}	487		}
487		488
488	// observeManagedContainers exports the managed-container gauge from app	489		// observeManagedContainers exports the managed-container gauge from app

A internal/boundaries/out/backup_metrics.go +25 -0

@@ -0,0 +1,25 @@
	1	+	package out
	2	+
	3	+	import (
	4	+	"context"
	5	+	"time"
	6	+
	7	+	"github.com/bnema/gordon/internal/domain"
	8	+	)
	9	+
	10	+	// BackupMetrics records backup outcomes.
	11	+	// Implementations must be safe for concurrent use.
	12	+	type BackupMetrics interface {
	13	+	// RecordBackupRun counts one finished backup run. A successful run
	14	+	// also becomes the target's last success.
	15	+	RecordBackupRun(ctx context.Context, run domain.BackupRunRecord)
	16	+	// ObserveStoredBackup reports an existing stored backup as a success
	17	+	// without counting a run, so the last success survives a restart.
	18	+	// Older records never replace a newer one.
	19	+	ObserveStoredBackup(run domain.BackupRunRecord)
	20	+	// RetainBackupTargets forgets the last success of every target of kind
	21	+	// that is not in targets, so a removed app or backup declaration stops
	22	+	// being reported. targets is the declaration read at asOf: a target
	23	+	// updated after asOf is kept, since it may have been declared since.
	24	+	RetainBackupTargets(kind domain.BackupKind, targets []domain.BackupTargetRef, asOf time.Time)
	25	+	}

@@ -0,0 +1,25 @@
	1	+	package out
	2	+
	3	+	import (
	4	+	"context"
	5	+	"time"
	6	+
	7	+	"github.com/bnema/gordon/internal/domain"
	8	+	)
	9	+
	10	+	// BackupMetrics records backup outcomes.
	11	+	// Implementations must be safe for concurrent use.
	12	+	type BackupMetrics interface {
	13	+	// RecordBackupRun counts one finished backup run. A successful run
	14	+	// also becomes the target's last success.
	15	+	RecordBackupRun(ctx context.Context, run domain.BackupRunRecord)
	16	+	// ObserveStoredBackup reports an existing stored backup as a success
	17	+	// without counting a run, so the last success survives a restart.
	18	+	// Older records never replace a newer one.
	19	+	ObserveStoredBackup(run domain.BackupRunRecord)
	20	+	// RetainBackupTargets forgets the last success of every target of kind
	21	+	// that is not in targets, so a removed app or backup declaration stops
	22	+	// being reported. targets is the declaration read at asOf: a target
	23	+	// updated after asOf is kept, since it may have been declared since.
	24	+	RetainBackupTargets(kind domain.BackupKind, targets []domain.BackupTargetRef, asOf time.Time)
	25	+	}

M internal/boundaries/out/backupstore.go +4 -0


@@ -30,6 +30,10 @@ type VolumeBackupStorage interface {
30	30		StoreVolumeArchive(ctx context.Context, job domain.VolumeBackupJob, data io.Reader) (string, error)
31	31		GetVolumeArchive(ctx context.Context, artifactRef string) (io.ReadCloser, error)
32	32		ListVolumeArchives(ctx context.Context, domainName string) ([]domain.VolumeBackupJob, error)
	33	+	// LatestVolumeArchives returns the newest archive of every app volume
	34	+	// from the object listing alone: Service and other per-object metadata
	35	+	// are left empty.
	36	+	LatestVolumeArchives(ctx context.Context) ([]domain.VolumeBackupJob, error)
33	37		DeleteVolumeArchive(ctx context.Context, artifactRef string) error
34	38		ApplyVolumeRetention(ctx context.Context, domainName string, policy domain.VolumeBackupRetentionPolicy) (int, error)
35	39		}


@@ -30,6 +30,10 @@ type VolumeBackupStorage interface {
30	StoreVolumeArchive(ctx context.Context, job domain.VolumeBackupJob, data io.Reader) (string, error)	30		StoreVolumeArchive(ctx context.Context, job domain.VolumeBackupJob, data io.Reader) (string, error)
31	GetVolumeArchive(ctx context.Context, artifactRef string) (io.ReadCloser, error)	31		GetVolumeArchive(ctx context.Context, artifactRef string) (io.ReadCloser, error)
32	ListVolumeArchives(ctx context.Context, domainName string) ([]domain.VolumeBackupJob, error)	32		ListVolumeArchives(ctx context.Context, domainName string) ([]domain.VolumeBackupJob, error)
		33	+	// LatestVolumeArchives returns the newest archive of every app volume
		34	+	// from the object listing alone: Service and other per-object metadata
		35	+	// are left empty.
		36	+	LatestVolumeArchives(ctx context.Context) ([]domain.VolumeBackupJob, error)
33	DeleteVolumeArchive(ctx context.Context, artifactRef string) error	37		DeleteVolumeArchive(ctx context.Context, artifactRef string) error
34	ApplyVolumeRetention(ctx context.Context, domainName string, policy domain.VolumeBackupRetentionPolicy) (int, error)	38		ApplyVolumeRetention(ctx context.Context, domainName string, policy domain.VolumeBackupRetentionPolicy) (int, error)
35	}	39		}

A internal/boundaries/out/mocks/mock_backup_metrics.go +187 -0

@@ -0,0 +1,187 @@
	1	+	// Code generated by mockery; DO NOT EDIT.
	2	+	// github.com/vektra/mockery
	3	+	// template: testify
	4	+
	5	+	package mocks
	6	+
	7	+	import (
	8	+	"context"
	9	+	"time"
	10	+
	11	+	"github.com/bnema/gordon/internal/domain"
	12	+	mock "github.com/stretchr/testify/mock"
	13	+	)
	14	+
	15	+	// NewMockBackupMetrics creates a new instance of MockBackupMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
	16	+	// The first argument is typically a *testing.T value.
	17	+	func NewMockBackupMetrics(t interface {
	18	+	mock.TestingT
	19	+	Cleanup(func())
	20	+	}) *MockBackupMetrics {
	21	+	if helper, ok := t.(interface{ Helper() }); ok {
	22	+	helper.Helper()
	23	+	}
	24	+
	25	+	mock := &MockBackupMetrics{}
	26	+	mock.Mock.Test(t)
	27	+
	28	+	t.Cleanup(func() {
	29	+	if helper, ok := t.(interface{ Helper() }); ok {
	30	+	helper.Helper()
	31	+	}
	32	+	mock.AssertExpectations(t)
	33	+	})
	34	+
	35	+	return mock
	36	+	}
	37	+
	38	+	// MockBackupMetrics is an autogenerated mock type for the BackupMetrics type
	39	+	type MockBackupMetrics struct {
	40	+	mock.Mock
	41	+	}
	42	+
	43	+	type MockBackupMetrics_Expecter struct {
	44	+	mock *mock.Mock
	45	+	}
	46	+
	47	+	func (_m MockBackupMetrics) EXPECT() MockBackupMetrics_Expecter {
	48	+	return &MockBackupMetrics_Expecter{mock: &_m.Mock}
	49	+	}
	50	+
	51	+	// ObserveStoredBackup provides a mock function for the type MockBackupMetrics
	52	+	func (_mock *MockBackupMetrics) ObserveStoredBackup(run domain.BackupRunRecord) {
	53	+	_mock.Called(run)
	54	+	return
	55	+	}
	56	+
	57	+	// MockBackupMetrics_ObserveStoredBackup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ObserveStoredBackup'
	58	+	type MockBackupMetrics_ObserveStoredBackup_Call struct {
	59	+	*mock.Call
	60	+	}
	61	+
	62	+	// ObserveStoredBackup is a helper method to define mock.On call
	63	+	// - run domain.BackupRunRecord
	64	+	func (_e MockBackupMetrics_Expecter) ObserveStoredBackup(run any) MockBackupMetrics_ObserveStoredBackup_Call {
	65	+	return &MockBackupMetrics_ObserveStoredBackup_Call{Call: _e.mock.On("ObserveStoredBackup", run)}
	66	+	}
	67	+
	68	+	func (_c MockBackupMetrics_ObserveStoredBackup_Call) Run(run func(run domain.BackupRunRecord)) MockBackupMetrics_ObserveStoredBackup_Call {
	69	+	_c.Call.Run(func(args mock.Arguments) {
	70	+	var arg0 domain.BackupRunRecord
	71	+	if args[0] != nil {
	72	+	arg0 = args[0].(domain.BackupRunRecord)
	73	+	}
	74	+	run(
	75	+	arg0,
	76	+	)
	77	+	})
	78	+	return _c
	79	+	}
	80	+
	81	+	func (_c MockBackupMetrics_ObserveStoredBackup_Call) Return() MockBackupMetrics_ObserveStoredBackup_Call {
	82	+	_c.Call.Return()
	83	+	return _c
	84	+	}
	85	+
	86	+	func (_c MockBackupMetrics_ObserveStoredBackup_Call) RunAndReturn(run func(run domain.BackupRunRecord)) MockBackupMetrics_ObserveStoredBackup_Call {
	87	+	_c.Run(run)
	88	+	return _c
	89	+	}
	90	+
	91	+	// RecordBackupRun provides a mock function for the type MockBackupMetrics
	92	+	func (_mock *MockBackupMetrics) RecordBackupRun(ctx context.Context, run domain.BackupRunRecord) {
	93	+	_mock.Called(ctx, run)
	94	+	return
	95	+	}
	96	+
	97	+	// MockBackupMetrics_RecordBackupRun_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RecordBackupRun'
	98	+	type MockBackupMetrics_RecordBackupRun_Call struct {
	99	+	*mock.Call
	100	+	}
	101	+
	102	+	// RecordBackupRun is a helper method to define mock.On call
	103	+	// - ctx context.Context
	104	+	// - run domain.BackupRunRecord
	105	+	func (_e MockBackupMetrics_Expecter) RecordBackupRun(ctx any, run any) MockBackupMetrics_RecordBackupRun_Call {
	106	+	return &MockBackupMetrics_RecordBackupRun_Call{Call: _e.mock.On("RecordBackupRun", ctx, run)}
	107	+	}
	108	+
	109	+	func (_c MockBackupMetrics_RecordBackupRun_Call) Run(run func(ctx context.Context, run domain.BackupRunRecord)) MockBackupMetrics_RecordBackupRun_Call {
	110	+	_c.Call.Run(func(args mock.Arguments) {
	111	+	var arg0 context.Context
	112	+	if args[0] != nil {
	113	+	arg0 = args[0].(context.Context)
	114	+	}
	115	+	var arg1 domain.BackupRunRecord
	116	+	if args[1] != nil {
	117	+	arg1 = args[1].(domain.BackupRunRecord)
	118	+	}
	119	+	run(
	120	+	arg0,
	121	+	arg1,
	122	+	)
	123	+	})
	124	+	return _c
	125	+	}
	126	+
	127	+	func (_c MockBackupMetrics_RecordBackupRun_Call) Return() MockBackupMetrics_RecordBackupRun_Call {
	128	+	_c.Call.Return()
	129	+	return _c
	130	+	}
	131	+
	132	+	func (_c MockBackupMetrics_RecordBackupRun_Call) RunAndReturn(run func(ctx context.Context, run domain.BackupRunRecord)) MockBackupMetrics_RecordBackupRun_Call {
	133	+	_c.Run(run)
	134	+	return _c
	135	+	}
	136	+
	137	+	// RetainBackupTargets provides a mock function for the type MockBackupMetrics
	138	+	func (_mock *MockBackupMetrics) RetainBackupTargets(kind domain.BackupKind, targets []domain.BackupTargetRef, asOf time.Time) {
	139	+	_mock.Called(kind, targets, asOf)
	140	+	return
	141	+	}
	142	+
	143	+	// MockBackupMetrics_RetainBackupTargets_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RetainBackupTargets'
	144	+	type MockBackupMetrics_RetainBackupTargets_Call struct {
	145	+	*mock.Call
	146	+	}
	147	+
	148	+	// RetainBackupTargets is a helper method to define mock.On call
	149	+	// - kind domain.BackupKind
	150	+	// - targets []domain.BackupTargetRef
	151	+	// - asOf time.Time
	152	+	func (_e MockBackupMetrics_Expecter) RetainBackupTargets(kind any, targets any, asOf any) MockBackupMetrics_RetainBackupTargets_Call {
	153	+	return &MockBackupMetrics_RetainBackupTargets_Call{Call: _e.mock.On("RetainBackupTargets", kind, targets, asOf)}
	154	+	}
	155	+
	156	+	func (_c MockBackupMetrics_RetainBackupTargets_Call) Run(run func(kind domain.BackupKind, targets []domain.BackupTargetRef, asOf time.Time)) MockBackupMetrics_RetainBackupTargets_Call {
	157	+	_c.Call.Run(func(args mock.Arguments) {
	158	+	var arg0 domain.BackupKind
	159	+	if args[0] != nil {
	160	+	arg0 = args[0].(domain.BackupKind)
	161	+	}
	162	+	var arg1 []domain.BackupTargetRef
	163	+	if args[1] != nil {
	164	+	arg1 = args[1].([]domain.BackupTargetRef)
	165	+	}
	166	+	var arg2 time.Time
	167	+	if args[2] != nil {
	168	+	arg2 = args[2].(time.Time)
	169	+	}
	170	+	run(
	171	+	arg0,
	172	+	arg1,
	173	+	arg2,
	174	+	)
	175	+	})
	176	+	return _c
	177	+	}
	178	+
	179	+	func (_c MockBackupMetrics_RetainBackupTargets_Call) Return() MockBackupMetrics_RetainBackupTargets_Call {
	180	+	_c.Call.Return()
	181	+	return _c
	182	+	}
	183	+
	184	+	func (_c MockBackupMetrics_RetainBackupTargets_Call) RunAndReturn(run func(kind domain.BackupKind, targets []domain.BackupTargetRef, asOf time.Time)) MockBackupMetrics_RetainBackupTargets_Call {
	185	+	_c.Run(run)
	186	+	return _c
	187	+	}

@@ -0,0 +1,187 @@
	1	+	// Code generated by mockery; DO NOT EDIT.
	2	+	// github.com/vektra/mockery
	3	+	// template: testify
	4	+
	5	+	package mocks
	6	+
	7	+	import (
	8	+	"context"
	9	+	"time"
	10	+
	11	+	"github.com/bnema/gordon/internal/domain"
	12	+	mock "github.com/stretchr/testify/mock"
	13	+	)
	14	+
	15	+	// NewMockBackupMetrics creates a new instance of MockBackupMetrics. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
	16	+	// The first argument is typically a *testing.T value.
	17	+	func NewMockBackupMetrics(t interface {
	18	+	mock.TestingT
	19	+	Cleanup(func())
	20	+	}) *MockBackupMetrics {
	21	+	if helper, ok := t.(interface{ Helper() }); ok {
	22	+	helper.Helper()
	23	+	}
	24	+
	25	+	mock := &MockBackupMetrics{}
	26	+	mock.Mock.Test(t)
	27	+
	28	+	t.Cleanup(func() {
	29	+	if helper, ok := t.(interface{ Helper() }); ok {
	30	+	helper.Helper()
	31	+	}
	32	+	mock.AssertExpectations(t)
	33	+	})
	34	+
	35	+	return mock
	36	+	}
	37	+
	38	+	// MockBackupMetrics is an autogenerated mock type for the BackupMetrics type
	39	+	type MockBackupMetrics struct {
	40	+	mock.Mock
	41	+	}
	42	+
	43	+	type MockBackupMetrics_Expecter struct {
	44	+	mock *mock.Mock
	45	+	}
	46	+
	47	+	func (_m MockBackupMetrics) EXPECT() MockBackupMetrics_Expecter {
	48	+	return &MockBackupMetrics_Expecter{mock: &_m.Mock}
	49	+	}
	50	+
	51	+	// ObserveStoredBackup provides a mock function for the type MockBackupMetrics
	52	+	func (_mock *MockBackupMetrics) ObserveStoredBackup(run domain.BackupRunRecord) {
	53	+	_mock.Called(run)
	54	+	return
	55	+	}
	56	+
	57	+	// MockBackupMetrics_ObserveStoredBackup_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ObserveStoredBackup'
	58	+	type MockBackupMetrics_ObserveStoredBackup_Call struct {
	59	+	*mock.Call
	60	+	}
	61	+
	62	+	// ObserveStoredBackup is a helper method to define mock.On call
	63	+	// - run domain.BackupRunRecord
	64	+	func (_e MockBackupMetrics_Expecter) ObserveStoredBackup(run any) MockBackupMetrics_ObserveStoredBackup_Call {
	65	+	return &MockBackupMetrics_ObserveStoredBackup_Call{Call: _e.mock.On("ObserveStoredBackup", run)}
	66	+	}
	67	+
	68	+	func (_c MockBackupMetrics_ObserveStoredBackup_Call) Run(run func(run domain.BackupRunRecord)) MockBackupMetrics_ObserveStoredBackup_Call {
	69	+	_c.Call.Run(func(args mock.Arguments) {
	70	+	var arg0 domain.BackupRunRecord
	71	+	if args[0] != nil {
	72	+	arg0 = args[0].(domain.BackupRunRecord)
	73	+	}
	74	+	run(
	75	+	arg0,
	76	+	)
	77	+	})
	78	+	return _c
	79	+	}
	80	+
	81	+	func (_c MockBackupMetrics_ObserveStoredBackup_Call) Return() MockBackupMetrics_ObserveStoredBackup_Call {
	82	+	_c.Call.Return()
	83	+	return _c
	84	+	}
	85	+
	86	+	func (_c MockBackupMetrics_ObserveStoredBackup_Call) RunAndReturn(run func(run domain.BackupRunRecord)) MockBackupMetrics_ObserveStoredBackup_Call {
	87	+	_c.Run(run)
	88	+	return _c
	89	+	}
	90	+
	91	+	// RecordBackupRun provides a mock function for the type MockBackupMetrics
	92	+	func (_mock *MockBackupMetrics) RecordBackupRun(ctx context.Context, run domain.BackupRunRecord) {
	93	+	_mock.Called(ctx, run)
	94	+	return
	95	+	}
	96	+
	97	+	// MockBackupMetrics_RecordBackupRun_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RecordBackupRun'
	98	+	type MockBackupMetrics_RecordBackupRun_Call struct {
	99	+	*mock.Call
	100	+	}
	101	+
	102	+	// RecordBackupRun is a helper method to define mock.On call
	103	+	// - ctx context.Context
	104	+	// - run domain.BackupRunRecord
	105	+	func (_e MockBackupMetrics_Expecter) RecordBackupRun(ctx any, run any) MockBackupMetrics_RecordBackupRun_Call {
	106	+	return &MockBackupMetrics_RecordBackupRun_Call{Call: _e.mock.On("RecordBackupRun", ctx, run)}
	107	+	}
	108	+
	109	+	func (_c MockBackupMetrics_RecordBackupRun_Call) Run(run func(ctx context.Context, run domain.BackupRunRecord)) MockBackupMetrics_RecordBackupRun_Call {
	110	+	_c.Call.Run(func(args mock.Arguments) {
	111	+	var arg0 context.Context
	112	+	if args[0] != nil {
	113	+	arg0 = args[0].(context.Context)
	114	+	}
	115	+	var arg1 domain.BackupRunRecord
	116	+	if args[1] != nil {
	117	+	arg1 = args[1].(domain.BackupRunRecord)
	118	+	}
	119	+	run(
	120	+	arg0,
	121	+	arg1,
	122	+	)
	123	+	})
	124	+	return _c
	125	+	}
	126	+
	127	+	func (_c MockBackupMetrics_RecordBackupRun_Call) Return() MockBackupMetrics_RecordBackupRun_Call {
	128	+	_c.Call.Return()
	129	+	return _c
	130	+	}
	131	+
	132	+	func (_c MockBackupMetrics_RecordBackupRun_Call) RunAndReturn(run func(ctx context.Context, run domain.BackupRunRecord)) MockBackupMetrics_RecordBackupRun_Call {
	133	+	_c.Run(run)
	134	+	return _c
	135	+	}
	136	+
	137	+	// RetainBackupTargets provides a mock function for the type MockBackupMetrics
	138	+	func (_mock *MockBackupMetrics) RetainBackupTargets(kind domain.BackupKind, targets []domain.BackupTargetRef, asOf time.Time) {
	139	+	_mock.Called(kind, targets, asOf)
	140	+	return
	141	+	}
	142	+
	143	+	// MockBackupMetrics_RetainBackupTargets_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RetainBackupTargets'
	144	+	type MockBackupMetrics_RetainBackupTargets_Call struct {
	145	+	*mock.Call
	146	+	}
	147	+
	148	+	// RetainBackupTargets is a helper method to define mock.On call
	149	+	// - kind domain.BackupKind
	150	+	// - targets []domain.BackupTargetRef
	151	+	// - asOf time.Time
	152	+	func (_e MockBackupMetrics_Expecter) RetainBackupTargets(kind any, targets any, asOf any) MockBackupMetrics_RetainBackupTargets_Call {
	153	+	return &MockBackupMetrics_RetainBackupTargets_Call{Call: _e.mock.On("RetainBackupTargets", kind, targets, asOf)}
	154	+	}
	155	+
	156	+	func (_c MockBackupMetrics_RetainBackupTargets_Call) Run(run func(kind domain.BackupKind, targets []domain.BackupTargetRef, asOf time.Time)) MockBackupMetrics_RetainBackupTargets_Call {
	157	+	_c.Call.Run(func(args mock.Arguments) {
	158	+	var arg0 domain.BackupKind
	159	+	if args[0] != nil {
	160	+	arg0 = args[0].(domain.BackupKind)
	161	+	}
	162	+	var arg1 []domain.BackupTargetRef
	163	+	if args[1] != nil {
	164	+	arg1 = args[1].([]domain.BackupTargetRef)
	165	+	}
	166	+	var arg2 time.Time
	167	+	if args[2] != nil {
	168	+	arg2 = args[2].(time.Time)
	169	+	}
	170	+	run(
	171	+	arg0,
	172	+	arg1,
	173	+	arg2,
	174	+	)
	175	+	})
	176	+	return _c
	177	+	}
	178	+
	179	+	func (_c MockBackupMetrics_RetainBackupTargets_Call) Return() MockBackupMetrics_RetainBackupTargets_Call {
	180	+	_c.Call.Return()
	181	+	return _c
	182	+	}
	183	+
	184	+	func (_c MockBackupMetrics_RetainBackupTargets_Call) RunAndReturn(run func(kind domain.BackupKind, targets []domain.BackupTargetRef, asOf time.Time)) MockBackupMetrics_RetainBackupTargets_Call {
	185	+	_c.Run(run)
	186	+	return _c
	187	+	}

M internal/domain/backup.go +29 -0


@@ -85,6 +85,35 @@ type BackupResult struct {
85	85		Duration time.Duration
86	86		}
87	87
	88	+	// BackupKind tells database dumps and volume archives apart in metrics.
	89	+	type BackupKind string
	90	+
	91	+	const (
	92	+	BackupKindDatabase BackupKind = "database"
	93	+	BackupKindVolume BackupKind = "volume"
	94	+	)
	95	+
	96	+	// BackupTargetRef identifies one backed-up resource in metrics. Target is
	97	+	// the database name for a dump and the declared volume name for an archive.
	98	+	type BackupTargetRef struct {
	99	+	Kind BackupKind
	100	+	App string
	101	+	Service string
	102	+	Target string
	103	+	}
	104	+
	105	+	// BackupRunRecord is the outcome of one backup, as reported to metrics.
	106	+	// StartedAt is the start of the run: stored backups are named after it, so
	107	+	// a live run and the same backup read back from storage after a restart
	108	+	// report the same time.
	109	+	type BackupRunRecord struct {
	110	+	BackupTargetRef
	111	+	Succeeded bool
	112	+	Duration time.Duration
	113	+	SizeBytes int64
	114	+	StartedAt time.Time
	115	+	}
	116	+
88	117		// DatabaseBackupJob represents a logical database backup operation.
89	118		type DatabaseBackupJob = BackupJob
90	119


@@ -85,6 +85,35 @@ type BackupResult struct {
85	Duration time.Duration	85		Duration time.Duration
86	}	86		}
87		87
		88	+	// BackupKind tells database dumps and volume archives apart in metrics.
		89	+	type BackupKind string
		90	+
		91	+	const (
		92	+	BackupKindDatabase BackupKind = "database"
		93	+	BackupKindVolume BackupKind = "volume"
		94	+	)
		95	+
		96	+	// BackupTargetRef identifies one backed-up resource in metrics. Target is
		97	+	// the database name for a dump and the declared volume name for an archive.
		98	+	type BackupTargetRef struct {
		99	+	Kind BackupKind
		100	+	App string
		101	+	Service string
		102	+	Target string
		103	+	}
		104	+
		105	+	// BackupRunRecord is the outcome of one backup, as reported to metrics.
		106	+	// StartedAt is the start of the run: stored backups are named after it, so
		107	+	// a live run and the same backup read back from storage after a restart
		108	+	// report the same time.
		109	+	type BackupRunRecord struct {
		110	+	BackupTargetRef
		111	+	Succeeded bool
		112	+	Duration time.Duration
		113	+	SizeBytes int64
		114	+	StartedAt time.Time
		115	+	}
		116	+
88	// DatabaseBackupJob represents a logical database backup operation.	117		// DatabaseBackupJob represents a logical database backup operation.
89	type DatabaseBackupJob = BackupJob	118		type DatabaseBackupJob = BackupJob
90		119

A internal/usecase/backup/metrics_test.go +175 -0

@@ -0,0 +1,175 @@
	1	+	package backup
	2	+
	3	+	import (
	4	+	"context"
	5	+	"errors"
	6	+	"io"
	7	+	"strings"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/stretchr/testify/assert"
	12	+	"github.com/stretchr/testify/mock"
	13	+	"github.com/stretchr/testify/require"
	14	+
	15	+	"github.com/bnema/gordon/internal/boundaries/out"
	16	+	outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
	17	+	"github.com/bnema/gordon/internal/domain"
	18	+	)
	19	+
	20	+	var (
	21	+	shopData = domain.BackupTargetRef{Kind: domain.BackupKindVolume, App: "shop", Service: "api", Target: "data"}
	22	+	shopOrders = domain.BackupTargetRef{Kind: domain.BackupKindDatabase, App: "shop", Service: "api", Target: "orders"}
	23	+	)
	24	+
	25	+	// captureRuns records every RecordBackupRun call on the mock.
	26	+	func captureRuns(m outmocks.MockBackupMetrics) []domain.BackupRunRecord {
	27	+	runs := &[]domain.BackupRunRecord{}
	28	+	m.EXPECT().RecordBackupRun(mock.Anything, mock.Anything).Run(func(_ context.Context, run domain.BackupRunRecord) {
	29	+	runs = append(runs, run)
	30	+	})
	31	+	return runs
	32	+	}
	33	+
	34	+	func expectShopVolume(state *outmocks.MockAppStateReader) {
	35	+	state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once()
	36	+	state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{
	37	+	"api": volumeSpec("api", "data", "/data"),
	38	+	}, "ctr-api"), true, nil).Once()
	39	+	}
	40	+
	41	+	// TestVolumeService_RecordsRunMetrics covers the success and failure of a
	42	+	// volume run: the record carries the declared target and the run's start,
	43	+	// which is also the time stored archives are named after.
	44	+	func TestVolumeService_RecordsRunMetrics(t *testing.T) {
	45	+	for _, tc := range []struct {
	46	+	name string
	47	+	export error
	48	+	success bool
	49	+	}{
	50	+	{"success", nil, true},
	51	+	{"export failure", errors.New("helper failed"), false},
	52	+	} {
	53	+	t.Run(tc.name, func(t *testing.T) {
	54	+	state := outmocks.NewMockAppStateReader(t)
	55	+	expectShopVolume(state)
	56	+	metrics := outmocks.NewMockBackupMetrics(t)
	57	+	runs := captureRuns(metrics)
	58	+	storage := &fakeVolumeBackupStorage{}
	59	+	svc := volumeTestService(t, state, &fakeVolumeArchiveExporter{err: tc.export}, storage, true).WithMetrics(metrics)
	60	+
	61	+	jobs, _ := svc.RunVolumeBackups(context.Background(), "shop", "api", "data")
	62	+	require.Len(t, jobs, 1)
	63	+	require.Len(t, *runs, 1)
	64	+	run := (*runs)[0]
	65	+	assert.Equal(t, shopData, run.BackupTargetRef)
	66	+	assert.Equal(t, tc.success, run.Succeeded)
	67	+	assert.Equal(t, jobs[0].StartedAt, run.StartedAt)
	68	+	assert.GreaterOrEqual(t, run.Duration, time.Duration(0))
	69	+	if tc.success {
	70	+	assert.Equal(t, int64(len("archive")), run.SizeBytes)
	71	+	assert.Equal(t, jobs[0].StartedAt, storage.stored[0].StartedAt, "the archive is named after the recorded start")
	72	+	}
	73	+	})
	74	+	}
	75	+	}
	76	+
	77	+	// TestVolumeService_SeedMetrics proves seeding forgets undeclared targets,
	78	+	// restores the declared target's newest archive with its declared service,
	79	+	// and ignores archives of targets that are no longer declared.
	80	+	func TestVolumeService_SeedMetrics(t *testing.T) {
	81	+	state := outmocks.NewMockAppStateReader(t)
	82	+	state.EXPECT().ListApps(mock.Anything).Return([]string{"shop"}, nil).Once()
	83	+	expectShopVolume(state)
	84	+	started := time.Date(2026, 10, 17, 1, 30, 0, 0, time.UTC)
	85	+	storage := &fakeVolumeBackupStorage{jobs: []domain.VolumeBackupJob{
	86	+	{App: "shop", VolumeName: "data", StartedAt: started, SizeBytes: 42},
	87	+	{App: "gone", VolumeName: "old", StartedAt: started, SizeBytes: 1},
	88	+	}}
	89	+	metrics := outmocks.NewMockBackupMetrics(t)
	90	+	metrics.EXPECT().RetainBackupTargets(domain.BackupKindVolume, []domain.BackupTargetRef{shopData}, mock.Anything).Once()
	91	+	metrics.EXPECT().ObserveStoredBackup(domain.BackupRunRecord{
	92	+	BackupTargetRef: shopData, Succeeded: true, SizeBytes: 42, StartedAt: started,
	93	+	}).Once()
	94	+	svc := volumeTestService(t, state, &fakeVolumeArchiveExporter{}, storage, true).WithMetrics(metrics)
	95	+
	96	+	require.NoError(t, svc.SeedMetrics(context.Background()))
	97	+	}
	98	+
	99	+	func expectShopDatabase(state *outmocks.MockAppStateReader) {
	100	+	state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once()
	101	+	state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{
	102	+	"api": appSpecWithDatabase("api", "orders", "daily"),
	103	+	}, "ctr-api"), true, nil).Once()
	104	+	}
	105	+
	106	+	// TestService_RecordsRunMetrics covers a failed and a successful dump. The
	107	+	// recorded start is the timestamp the dump is stored under.
	108	+	func TestService_RecordsRunMetrics(t *testing.T) {
	109	+	t.Run("pg_dump failure", func(t *testing.T) {
	110	+	state := outmocks.NewMockAppStateReader(t)
	111	+	expectShopDatabase(state)
	112	+	runtime, _, svc := backupTestService(t, state)
	113	+	metrics := outmocks.NewMockBackupMetrics(t)
	114	+	runs := captureRuns(metrics)
	115	+	svc.WithMetrics(metrics)
	116	+	runtime.EXPECT().ExecInContainer(mock.Anything, "ctr-api", mock.Anything).
	117	+	Return(&out.ExecResult{ExitCode: 1, Stderr: []byte("boom")}, nil).Twice()
	118	+
	119	+	_, err := svc.RunBackup(context.Background(), "shop", "api", "orders")
	120	+	require.Error(t, err)
	121	+	require.Len(t, *runs, 1)
	122	+	assert.Equal(t, shopOrders, (*runs)[0].BackupTargetRef)
	123	+	assert.False(t, (*runs)[0].Succeeded)
	124	+	assert.False(t, (*runs)[0].StartedAt.IsZero())
	125	+	})
	126	+
	127	+	t.Run("success", func(t *testing.T) {
	128	+	state := outmocks.NewMockAppStateReader(t)
	129	+	expectShopDatabase(state)
	130	+	runtime, storage, svc := backupTestService(t, state)
	131	+	metrics := outmocks.NewMockBackupMetrics(t)
	132	+	runs := captureRuns(metrics)
	133	+	svc.WithMetrics(metrics)
	134	+	runtime.EXPECT().ExecInContainer(mock.Anything, "ctr-api", mock.Anything).Return(&out.ExecResult{ExitCode: 0}, nil).Twice()
	135	+	runtime.EXPECT().CopyFromContainer(mock.Anything, "ctr-api", mock.Anything).
	136	+	Return(io.NopCloser(strings.NewReader("dump")), nil).Once()
	137	+	var storedAt time.Time
	138	+	storage.EXPECT().Store(mock.Anything, "shop", "api", "orders", domain.BackupSchedule(""), mock.Anything, mock.Anything).
	139	+	RunAndReturn(func(_ context.Context, _, _, _ string, _ domain.BackupSchedule, at time.Time, data io.Reader) (string, error) {
	140	+	storedAt = at
	141	+	_, err := io.Copy(io.Discard, data)
	142	+	return "/b/file.bak", err
	143	+	}).Once()
	144	+
	145	+	_, err := svc.RunBackup(context.Background(), "shop", "api", "orders")
	146	+	require.NoError(t, err)
	147	+	require.Len(t, *runs, 1)
	148	+	assert.True(t, (*runs)[0].Succeeded)
	149	+	assert.Equal(t, int64(len("dump")), (*runs)[0].SizeBytes)
	150	+	assert.Equal(t, storedAt, (*runs)[0].StartedAt, "the dump is stored under the recorded start")
	151	+	})
	152	+	}
	153	+
	154	+	// TestService_SeedMetrics proves database seeding forgets undeclared
	155	+	// targets and only restores dumps of declared ones.
	156	+	func TestService_SeedMetrics(t *testing.T) {
	157	+	state := outmocks.NewMockAppStateReader(t)
	158	+	state.EXPECT().ListApps(mock.Anything).Return([]string{"shop"}, nil).Twice()
	159	+	expectShopDatabase(state)
	160	+	started := time.Date(2026, 10, 17, 2, 0, 0, 0, time.UTC)
	161	+
	162	+	_, storage, svc := backupTestService(t, state)
	163	+	metrics := outmocks.NewMockBackupMetrics(t)
	164	+	svc.WithMetrics(metrics)
	165	+	storage.EXPECT().List(mock.Anything, "shop", mock.Anything).Return([]domain.BackupJob{
	166	+	{App: "shop", Service: "api", DBName: "orders", StartedAt: started, SizeBytes: 7},
	167	+	{App: "shop", Service: "api", DBName: "dropped", StartedAt: started, SizeBytes: 1},
	168	+	}, nil).Once()
	169	+	metrics.EXPECT().RetainBackupTargets(domain.BackupKindDatabase, []domain.BackupTargetRef{shopOrders}, mock.Anything).Once()
	170	+	metrics.EXPECT().ObserveStoredBackup(domain.BackupRunRecord{
	171	+	BackupTargetRef: shopOrders, Succeeded: true, SizeBytes: 7, StartedAt: started,
	172	+	}).Once()
	173	+
	174	+	require.NoError(t, svc.SeedMetrics(context.Background()))
	175	+	}

@@ -0,0 +1,175 @@
	1	+	package backup
	2	+
	3	+	import (
	4	+	"context"
	5	+	"errors"
	6	+	"io"
	7	+	"strings"
	8	+	"testing"
	9	+	"time"
	10	+
	11	+	"github.com/stretchr/testify/assert"
	12	+	"github.com/stretchr/testify/mock"
	13	+	"github.com/stretchr/testify/require"
	14	+
	15	+	"github.com/bnema/gordon/internal/boundaries/out"
	16	+	outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
	17	+	"github.com/bnema/gordon/internal/domain"
	18	+	)
	19	+
	20	+	var (
	21	+	shopData = domain.BackupTargetRef{Kind: domain.BackupKindVolume, App: "shop", Service: "api", Target: "data"}
	22	+	shopOrders = domain.BackupTargetRef{Kind: domain.BackupKindDatabase, App: "shop", Service: "api", Target: "orders"}
	23	+	)
	24	+
	25	+	// captureRuns records every RecordBackupRun call on the mock.
	26	+	func captureRuns(m outmocks.MockBackupMetrics) []domain.BackupRunRecord {
	27	+	runs := &[]domain.BackupRunRecord{}
	28	+	m.EXPECT().RecordBackupRun(mock.Anything, mock.Anything).Run(func(_ context.Context, run domain.BackupRunRecord) {
	29	+	runs = append(runs, run)
	30	+	})
	31	+	return runs
	32	+	}
	33	+
	34	+	func expectShopVolume(state *outmocks.MockAppStateReader) {
	35	+	state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once()
	36	+	state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{
	37	+	"api": volumeSpec("api", "data", "/data"),
	38	+	}, "ctr-api"), true, nil).Once()
	39	+	}
	40	+
	41	+	// TestVolumeService_RecordsRunMetrics covers the success and failure of a
	42	+	// volume run: the record carries the declared target and the run's start,
	43	+	// which is also the time stored archives are named after.
	44	+	func TestVolumeService_RecordsRunMetrics(t *testing.T) {
	45	+	for _, tc := range []struct {
	46	+	name string
	47	+	export error
	48	+	success bool
	49	+	}{
	50	+	{"success", nil, true},
	51	+	{"export failure", errors.New("helper failed"), false},
	52	+	} {
	53	+	t.Run(tc.name, func(t *testing.T) {
	54	+	state := outmocks.NewMockAppStateReader(t)
	55	+	expectShopVolume(state)
	56	+	metrics := outmocks.NewMockBackupMetrics(t)
	57	+	runs := captureRuns(metrics)
	58	+	storage := &fakeVolumeBackupStorage{}
	59	+	svc := volumeTestService(t, state, &fakeVolumeArchiveExporter{err: tc.export}, storage, true).WithMetrics(metrics)
	60	+
	61	+	jobs, _ := svc.RunVolumeBackups(context.Background(), "shop", "api", "data")
	62	+	require.Len(t, jobs, 1)
	63	+	require.Len(t, *runs, 1)
	64	+	run := (*runs)[0]
	65	+	assert.Equal(t, shopData, run.BackupTargetRef)
	66	+	assert.Equal(t, tc.success, run.Succeeded)
	67	+	assert.Equal(t, jobs[0].StartedAt, run.StartedAt)
	68	+	assert.GreaterOrEqual(t, run.Duration, time.Duration(0))
	69	+	if tc.success {
	70	+	assert.Equal(t, int64(len("archive")), run.SizeBytes)
	71	+	assert.Equal(t, jobs[0].StartedAt, storage.stored[0].StartedAt, "the archive is named after the recorded start")
	72	+	}
	73	+	})
	74	+	}
	75	+	}
	76	+
	77	+	// TestVolumeService_SeedMetrics proves seeding forgets undeclared targets,
	78	+	// restores the declared target's newest archive with its declared service,
	79	+	// and ignores archives of targets that are no longer declared.
	80	+	func TestVolumeService_SeedMetrics(t *testing.T) {
	81	+	state := outmocks.NewMockAppStateReader(t)
	82	+	state.EXPECT().ListApps(mock.Anything).Return([]string{"shop"}, nil).Once()
	83	+	expectShopVolume(state)
	84	+	started := time.Date(2026, 10, 17, 1, 30, 0, 0, time.UTC)
	85	+	storage := &fakeVolumeBackupStorage{jobs: []domain.VolumeBackupJob{
	86	+	{App: "shop", VolumeName: "data", StartedAt: started, SizeBytes: 42},
	87	+	{App: "gone", VolumeName: "old", StartedAt: started, SizeBytes: 1},
	88	+	}}
	89	+	metrics := outmocks.NewMockBackupMetrics(t)
	90	+	metrics.EXPECT().RetainBackupTargets(domain.BackupKindVolume, []domain.BackupTargetRef{shopData}, mock.Anything).Once()
	91	+	metrics.EXPECT().ObserveStoredBackup(domain.BackupRunRecord{
	92	+	BackupTargetRef: shopData, Succeeded: true, SizeBytes: 42, StartedAt: started,
	93	+	}).Once()
	94	+	svc := volumeTestService(t, state, &fakeVolumeArchiveExporter{}, storage, true).WithMetrics(metrics)
	95	+
	96	+	require.NoError(t, svc.SeedMetrics(context.Background()))
	97	+	}
	98	+
	99	+	func expectShopDatabase(state *outmocks.MockAppStateReader) {
	100	+	state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once()
	101	+	state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{
	102	+	"api": appSpecWithDatabase("api", "orders", "daily"),
	103	+	}, "ctr-api"), true, nil).Once()
	104	+	}
	105	+
	106	+	// TestService_RecordsRunMetrics covers a failed and a successful dump. The
	107	+	// recorded start is the timestamp the dump is stored under.
	108	+	func TestService_RecordsRunMetrics(t *testing.T) {
	109	+	t.Run("pg_dump failure", func(t *testing.T) {
	110	+	state := outmocks.NewMockAppStateReader(t)
	111	+	expectShopDatabase(state)
	112	+	runtime, _, svc := backupTestService(t, state)
	113	+	metrics := outmocks.NewMockBackupMetrics(t)
	114	+	runs := captureRuns(metrics)
	115	+	svc.WithMetrics(metrics)
	116	+	runtime.EXPECT().ExecInContainer(mock.Anything, "ctr-api", mock.Anything).
	117	+	Return(&out.ExecResult{ExitCode: 1, Stderr: []byte("boom")}, nil).Twice()
	118	+
	119	+	_, err := svc.RunBackup(context.Background(), "shop", "api", "orders")
	120	+	require.Error(t, err)
	121	+	require.Len(t, *runs, 1)
	122	+	assert.Equal(t, shopOrders, (*runs)[0].BackupTargetRef)
	123	+	assert.False(t, (*runs)[0].Succeeded)
	124	+	assert.False(t, (*runs)[0].StartedAt.IsZero())
	125	+	})
	126	+
	127	+	t.Run("success", func(t *testing.T) {
	128	+	state := outmocks.NewMockAppStateReader(t)
	129	+	expectShopDatabase(state)
	130	+	runtime, storage, svc := backupTestService(t, state)
	131	+	metrics := outmocks.NewMockBackupMetrics(t)
	132	+	runs := captureRuns(metrics)
	133	+	svc.WithMetrics(metrics)
	134	+	runtime.EXPECT().ExecInContainer(mock.Anything, "ctr-api", mock.Anything).Return(&out.ExecResult{ExitCode: 0}, nil).Twice()
	135	+	runtime.EXPECT().CopyFromContainer(mock.Anything, "ctr-api", mock.Anything).
	136	+	Return(io.NopCloser(strings.NewReader("dump")), nil).Once()
	137	+	var storedAt time.Time
	138	+	storage.EXPECT().Store(mock.Anything, "shop", "api", "orders", domain.BackupSchedule(""), mock.Anything, mock.Anything).
	139	+	RunAndReturn(func(_ context.Context, _, _, _ string, _ domain.BackupSchedule, at time.Time, data io.Reader) (string, error) {
	140	+	storedAt = at
	141	+	_, err := io.Copy(io.Discard, data)
	142	+	return "/b/file.bak", err
	143	+	}).Once()
	144	+
	145	+	_, err := svc.RunBackup(context.Background(), "shop", "api", "orders")
	146	+	require.NoError(t, err)
	147	+	require.Len(t, *runs, 1)
	148	+	assert.True(t, (*runs)[0].Succeeded)
	149	+	assert.Equal(t, int64(len("dump")), (*runs)[0].SizeBytes)
	150	+	assert.Equal(t, storedAt, (*runs)[0].StartedAt, "the dump is stored under the recorded start")
	151	+	})
	152	+	}
	153	+
	154	+	// TestService_SeedMetrics proves database seeding forgets undeclared
	155	+	// targets and only restores dumps of declared ones.
	156	+	func TestService_SeedMetrics(t *testing.T) {
	157	+	state := outmocks.NewMockAppStateReader(t)
	158	+	state.EXPECT().ListApps(mock.Anything).Return([]string{"shop"}, nil).Twice()
	159	+	expectShopDatabase(state)
	160	+	started := time.Date(2026, 10, 17, 2, 0, 0, 0, time.UTC)
	161	+
	162	+	_, storage, svc := backupTestService(t, state)
	163	+	metrics := outmocks.NewMockBackupMetrics(t)
	164	+	svc.WithMetrics(metrics)
	165	+	storage.EXPECT().List(mock.Anything, "shop", mock.Anything).Return([]domain.BackupJob{
	166	+	{App: "shop", Service: "api", DBName: "orders", StartedAt: started, SizeBytes: 7},
	167	+	{App: "shop", Service: "api", DBName: "dropped", StartedAt: started, SizeBytes: 1},
	168	+	}, nil).Once()
	169	+	metrics.EXPECT().RetainBackupTargets(domain.BackupKindDatabase, []domain.BackupTargetRef{shopOrders}, mock.Anything).Once()
	170	+	metrics.EXPECT().ObserveStoredBackup(domain.BackupRunRecord{
	171	+	BackupTargetRef: shopOrders, Succeeded: true, SizeBytes: 7, StartedAt: started,
	172	+	}).Once()
	173	+
	174	+	require.NoError(t, svc.SeedMetrics(context.Background()))
	175	+	}

M internal/usecase/backup/service.go +69 -2


@@ -28,6 +28,7 @@ type Service struct {
28	28		config domain.BackupConfig
29	29		log zerowrap.Logger
30	30		state out.AppStateReader
	31	+	metrics out.BackupMetrics
31	32		}
32	33
33	34		// NewService creates a backup service.

@@ -52,6 +53,55 @@ func (s Service) WithAppState(state out.AppStateReader) Service {
52	53		return s
53	54		}
54	55
	56	+	// WithMetrics wires the backup metrics recorder. Without it no metric is
	57	+	// recorded.
	58	+	func (s Service) WithMetrics(metrics out.BackupMetrics) Service {
	59	+	s.metrics = metrics
	60	+	return s
	61	+	}
	62	+
	63	+	// SeedMetrics reconciles backup metrics with the declared database
	64	+	// targets: it forgets targets that are no longer declared and reports the
	65	+	// newest stored dump of each declared one as its last success, so the
	66	+	// metric survives a restart. It counts no run.
	67	+	func (s *Service) SeedMetrics(ctx context.Context) error {
	68	+	if s.metrics == nil {
	69	+	return nil
	70	+	}
	71	+	asOf := time.Now()
	72	+	targets, err := s.allTargets(ctx)
	73	+	if err != nil {
	74	+	return fmt.Errorf("seed backup metrics: %w", err)
	75	+	}
	76	+	refs := make([]domain.BackupTargetRef, 0, len(targets))
	77	+	declared := make(map[domain.BackupTargetRef]struct{}, len(targets))
	78	+	for _, target := range targets {
	79	+	ref := databaseTargetRef(target)
	80	+	refs = append(refs, ref)
	81	+	declared[ref] = struct{}{}
	82	+	}
	83	+	s.metrics.RetainBackupTargets(domain.BackupKindDatabase, refs, asOf)
	84	+
	85	+	jobs, err := s.ListBackups(ctx, "")
	86	+	if err != nil {
	87	+	return fmt.Errorf("seed backup metrics: %w", err)
	88	+	}
	89	+	for _, job := range jobs {
	90	+	ref := domain.BackupTargetRef{Kind: domain.BackupKindDatabase, App: job.App, Service: job.Service, Target: job.DBName}
	91	+	if _, ok := declared[ref]; !ok {
	92	+	continue
	93	+	}
	94	+	s.metrics.ObserveStoredBackup(domain.BackupRunRecord{
	95	+	BackupTargetRef: ref, Succeeded: true, SizeBytes: job.SizeBytes, StartedAt: job.StartedAt,
	96	+	})
	97	+	}
	98	+	return nil
	99	+	}
	100	+
	101	+	func databaseTargetRef(target domain.DatabaseTarget) domain.BackupTargetRef {
	102	+	return domain.BackupTargetRef{Kind: domain.BackupKindDatabase, App: target.App, Service: target.Service, Target: target.Database}
	103	+	}
	104	+
55	105		// Targets returns every declared database target of one app, ordered by
56	106		// service then database.
57	107		func (s *Service) Targets(ctx context.Context, app string) ([]domain.DatabaseTarget, error) {

@@ -211,12 +261,29 @@ func (s *Service) Status(ctx context.Context) ([]domain.BackupJob, error) {
211	261		}
212	262
213	263		// runTarget runs one declared database backup through the service
214		-	// container that hosts it.
	264	+	// container that hosts it and records the outcome in metrics.
215	265		func (s Service) runTarget(ctx context.Context, target domain.DatabaseTarget, schedule domain.BackupSchedule) (domain.BackupResult, error) {
	266	+	started := time.Now().UTC()
	267	+	result, err := s.dumpTarget(ctx, target, schedule, started)
	268	+	if s.metrics != nil {
	269	+	run := domain.BackupRunRecord{
	270	+	BackupTargetRef: databaseTargetRef(target),
	271	+	Succeeded: err == nil,
	272	+	Duration: time.Since(started),
	273	+	StartedAt: started,
	274	+	}
	275	+	if result != nil {
	276	+	run.SizeBytes = result.Job.SizeBytes
	277	+	}
	278	+	s.metrics.RecordBackupRun(ctx, run)
	279	+	}
	280	+	return result, err
	281	+	}
	282	+
	283	+	func (s Service) dumpTarget(ctx context.Context, target domain.DatabaseTarget, schedule domain.BackupSchedule, started time.Time) (domain.BackupResult, error) {
216	284		if target.ContainerID == "" {
217	285		return nil, fmt.Errorf("backup: app %q service %q has no active container", target.App, target.Service)
218	286		}
219		-	started := time.Now().UTC()
220	287
221	288		execCtx, cancelExec := context.WithTimeout(ctx, backupExecTimeout)
222	289		defer cancelExec()


@@ -28,6 +28,7 @@ type Service struct {
28		config domain.BackupConfig	28		config domain.BackupConfig
29		log zerowrap.Logger	29		log zerowrap.Logger
30		state out.AppStateReader	30		state out.AppStateReader
			31	+	metrics out.BackupMetrics
31		}	32		}
32			33
33		// NewService creates a backup service.	34		// NewService creates a backup service.

@@ -52,6 +53,55 @@ func (s Service) WithAppState(state out.AppStateReader) Service {
52		return s	53		return s
53		}	54		}
54			55
			56	+	// WithMetrics wires the backup metrics recorder. Without it no metric is
			57	+	// recorded.
			58	+	func (s Service) WithMetrics(metrics out.BackupMetrics) Service {
			59	+	s.metrics = metrics
			60	+	return s
			61	+	}
			62	+
			63	+	// SeedMetrics reconciles backup metrics with the declared database
			64	+	// targets: it forgets targets that are no longer declared and reports the
			65	+	// newest stored dump of each declared one as its last success, so the
			66	+	// metric survives a restart. It counts no run.
			67	+	func (s *Service) SeedMetrics(ctx context.Context) error {
			68	+	if s.metrics == nil {
			69	+	return nil
			70	+	}
			71	+	asOf := time.Now()
			72	+	targets, err := s.allTargets(ctx)
			73	+	if err != nil {
			74	+	return fmt.Errorf("seed backup metrics: %w", err)
			75	+	}
			76	+	refs := make([]domain.BackupTargetRef, 0, len(targets))
			77	+	declared := make(map[domain.BackupTargetRef]struct{}, len(targets))
			78	+	for _, target := range targets {
			79	+	ref := databaseTargetRef(target)
			80	+	refs = append(refs, ref)
			81	+	declared[ref] = struct{}{}
			82	+	}
			83	+	s.metrics.RetainBackupTargets(domain.BackupKindDatabase, refs, asOf)
			84	+
			85	+	jobs, err := s.ListBackups(ctx, "")
			86	+	if err != nil {
			87	+	return fmt.Errorf("seed backup metrics: %w", err)
			88	+	}
			89	+	for _, job := range jobs {
			90	+	ref := domain.BackupTargetRef{Kind: domain.BackupKindDatabase, App: job.App, Service: job.Service, Target: job.DBName}
			91	+	if _, ok := declared[ref]; !ok {
			92	+	continue
			93	+	}
			94	+	s.metrics.ObserveStoredBackup(domain.BackupRunRecord{
			95	+	BackupTargetRef: ref, Succeeded: true, SizeBytes: job.SizeBytes, StartedAt: job.StartedAt,
			96	+	})
			97	+	}
			98	+	return nil
			99	+	}
			100	+
			101	+	func databaseTargetRef(target domain.DatabaseTarget) domain.BackupTargetRef {
			102	+	return domain.BackupTargetRef{Kind: domain.BackupKindDatabase, App: target.App, Service: target.Service, Target: target.Database}
			103	+	}
			104	+
55		// Targets returns every declared database target of one app, ordered by	105		// Targets returns every declared database target of one app, ordered by
56		// service then database.	106		// service then database.
57		func (s *Service) Targets(ctx context.Context, app string) ([]domain.DatabaseTarget, error) {	107		func (s *Service) Targets(ctx context.Context, app string) ([]domain.DatabaseTarget, error) {

@@ -211,12 +261,29 @@ func (s *Service) Status(ctx context.Context) ([]domain.BackupJob, error) {
211		}	261		}
212			262
213		// runTarget runs one declared database backup through the service	263		// runTarget runs one declared database backup through the service
214	-	// container that hosts it.	264	+	// container that hosts it and records the outcome in metrics.
215		func (s Service) runTarget(ctx context.Context, target domain.DatabaseTarget, schedule domain.BackupSchedule) (domain.BackupResult, error) {	265		func (s Service) runTarget(ctx context.Context, target domain.DatabaseTarget, schedule domain.BackupSchedule) (domain.BackupResult, error) {
			266	+	started := time.Now().UTC()
			267	+	result, err := s.dumpTarget(ctx, target, schedule, started)
			268	+	if s.metrics != nil {
			269	+	run := domain.BackupRunRecord{
			270	+	BackupTargetRef: databaseTargetRef(target),
			271	+	Succeeded: err == nil,
			272	+	Duration: time.Since(started),
			273	+	StartedAt: started,
			274	+	}
			275	+	if result != nil {
			276	+	run.SizeBytes = result.Job.SizeBytes
			277	+	}
			278	+	s.metrics.RecordBackupRun(ctx, run)
			279	+	}
			280	+	return result, err
			281	+	}
			282	+
			283	+	func (s Service) dumpTarget(ctx context.Context, target domain.DatabaseTarget, schedule domain.BackupSchedule, started time.Time) (domain.BackupResult, error) {
216		if target.ContainerID == "" {	284		if target.ContainerID == "" {
217		return nil, fmt.Errorf("backup: app %q service %q has no active container", target.App, target.Service)	285		return nil, fmt.Errorf("backup: app %q service %q has no active container", target.App, target.Service)
218		}	286		}
219	-	started := time.Now().UTC()
220			287
221		execCtx, cancelExec := context.WithTimeout(ctx, backupExecTimeout)	288		execCtx, cancelExec := context.WithTimeout(ctx, backupExecTimeout)
222		defer cancelExec()	289		defer cancelExec()

M internal/usecase/backup/volume_service.go +65 -0


@@ -26,6 +26,7 @@ type VolumeService struct {
26	26		config domain.VolumeBackupConfig
27	27		log zerowrap.Logger
28	28		state out.AppStateReader
	29	+	metrics out.BackupMetrics
29	30
30	31		mu sync.Mutex
31	32		recent map[string]domain.VolumeBackupJob

@@ -48,6 +49,55 @@ func (s VolumeService) WithAppState(state out.AppStateReader) VolumeService {
48	49		return s
49	50		}
50	51
	52	+	// WithMetrics wires the backup metrics recorder. Without it no metric is
	53	+	// recorded.
	54	+	func (s VolumeService) WithMetrics(metrics out.BackupMetrics) VolumeService {
	55	+	s.metrics = metrics
	56	+	return s
	57	+	}
	58	+
	59	+	// SeedMetrics reconciles backup metrics with the declared volume targets:
	60	+	// it forgets targets that are no longer declared and reports the newest
	61	+	// stored archive of each declared one as its last success, so the metric
	62	+	// survives a restart. It counts no run.
	63	+	func (s *VolumeService) SeedMetrics(ctx context.Context) error {
	64	+	if s.metrics == nil {
	65	+	return nil
	66	+	}
	67	+	asOf := time.Now()
	68	+	targets, err := s.allVolumeTargets(ctx)
	69	+	if err != nil {
	70	+	return fmt.Errorf("seed volume backup metrics: %w", err)
	71	+	}
	72	+	refs := make([]domain.BackupTargetRef, 0, len(targets))
	73	+	byVolume := make(map[string]domain.BackupTargetRef, len(targets))
	74	+	for _, target := range targets {
	75	+	ref := volumeTargetRef(target)
	76	+	refs = append(refs, ref)
	77	+	byVolume[target.App+"/"+target.VolumeName] = ref
	78	+	}
	79	+	s.metrics.RetainBackupTargets(domain.BackupKindVolume, refs, asOf)
	80	+
	81	+	archives, err := s.storage.LatestVolumeArchives(ctx)
	82	+	if err != nil {
	83	+	return fmt.Errorf("seed volume backup metrics: %w", err)
	84	+	}
	85	+	for _, archive := range archives {
	86	+	ref, declared := byVolume[archive.App+"/"+archive.VolumeName]
	87	+	if !declared {
	88	+	continue
	89	+	}
	90	+	s.metrics.ObserveStoredBackup(domain.BackupRunRecord{
	91	+	BackupTargetRef: ref, Succeeded: true, SizeBytes: archive.SizeBytes, StartedAt: archive.StartedAt,
	92	+	})
	93	+	}
	94	+	return nil
	95	+	}
	96	+
	97	+	func volumeTargetRef(target domain.VolumeBackupTarget) domain.BackupTargetRef {
	98	+	return domain.BackupTargetRef{Kind: domain.BackupKindVolume, App: target.App, Service: target.Service, Target: target.VolumeName}
	99	+	}
	100	+
51	101		// VolumeTargets returns every declared volume target of one app.
52	102		func (s *VolumeService) VolumeTargets(ctx context.Context, app string) ([]domain.VolumeBackupTarget, error) {
53	103		if s.state == nil {

@@ -214,7 +264,22 @@ func (s *VolumeService) runVolumeBackupsConcurrently(ctx context.Context, target
214	264		return jobs
215	265		}
216	266
	267	+	// runVolumeBackup archives one target and records the outcome in metrics.
217	268		func (s *VolumeService) runVolumeBackup(ctx context.Context, target domain.VolumeBackupTarget) domain.VolumeBackupJob {
	269	+	job := s.archiveVolume(ctx, target)
	270	+	if s.metrics != nil {
	271	+	s.metrics.RecordBackupRun(ctx, domain.BackupRunRecord{
	272	+	BackupTargetRef: volumeTargetRef(target),
	273	+	Succeeded: job.Status == domain.BackupStatusCompleted,
	274	+	Duration: job.CompletedAt.Sub(job.StartedAt),
	275	+	SizeBytes: job.SizeBytes,
	276	+	StartedAt: job.StartedAt,
	277	+	})
	278	+	}
	279	+	return job
	280	+	}
	281	+
	282	+	func (s *VolumeService) archiveVolume(ctx context.Context, target domain.VolumeBackupTarget) domain.VolumeBackupJob {
218	283		ctx = zerowrap.CtxWithFields(ctx, map[string]any{
219	284		"app": target.App,
220	285		"service": target.Service,


@@ -26,6 +26,7 @@ type VolumeService struct {
26	config domain.VolumeBackupConfig	26		config domain.VolumeBackupConfig
27	log zerowrap.Logger	27		log zerowrap.Logger
28	state out.AppStateReader	28		state out.AppStateReader
		29	+	metrics out.BackupMetrics
29		30
30	mu sync.Mutex	31		mu sync.Mutex
31	recent map[string]domain.VolumeBackupJob	32		recent map[string]domain.VolumeBackupJob

@@ -48,6 +49,55 @@ func (s VolumeService) WithAppState(state out.AppStateReader) VolumeService {
48	return s	49		return s
49	}	50		}
50		51
		52	+	// WithMetrics wires the backup metrics recorder. Without it no metric is
		53	+	// recorded.
		54	+	func (s VolumeService) WithMetrics(metrics out.BackupMetrics) VolumeService {
		55	+	s.metrics = metrics
		56	+	return s
		57	+	}
		58	+
		59	+	// SeedMetrics reconciles backup metrics with the declared volume targets:
		60	+	// it forgets targets that are no longer declared and reports the newest
		61	+	// stored archive of each declared one as its last success, so the metric
		62	+	// survives a restart. It counts no run.
		63	+	func (s *VolumeService) SeedMetrics(ctx context.Context) error {
		64	+	if s.metrics == nil {
		65	+	return nil
		66	+	}
		67	+	asOf := time.Now()
		68	+	targets, err := s.allVolumeTargets(ctx)
		69	+	if err != nil {
		70	+	return fmt.Errorf("seed volume backup metrics: %w", err)
		71	+	}
		72	+	refs := make([]domain.BackupTargetRef, 0, len(targets))
		73	+	byVolume := make(map[string]domain.BackupTargetRef, len(targets))
		74	+	for _, target := range targets {
		75	+	ref := volumeTargetRef(target)
		76	+	refs = append(refs, ref)
		77	+	byVolume[target.App+"/"+target.VolumeName] = ref
		78	+	}
		79	+	s.metrics.RetainBackupTargets(domain.BackupKindVolume, refs, asOf)
		80	+
		81	+	archives, err := s.storage.LatestVolumeArchives(ctx)
		82	+	if err != nil {
		83	+	return fmt.Errorf("seed volume backup metrics: %w", err)
		84	+	}
		85	+	for _, archive := range archives {
		86	+	ref, declared := byVolume[archive.App+"/"+archive.VolumeName]
		87	+	if !declared {
		88	+	continue
		89	+	}
		90	+	s.metrics.ObserveStoredBackup(domain.BackupRunRecord{
		91	+	BackupTargetRef: ref, Succeeded: true, SizeBytes: archive.SizeBytes, StartedAt: archive.StartedAt,
		92	+	})
		93	+	}
		94	+	return nil
		95	+	}
		96	+
		97	+	func volumeTargetRef(target domain.VolumeBackupTarget) domain.BackupTargetRef {
		98	+	return domain.BackupTargetRef{Kind: domain.BackupKindVolume, App: target.App, Service: target.Service, Target: target.VolumeName}
		99	+	}
		100	+
51	// VolumeTargets returns every declared volume target of one app.	101		// VolumeTargets returns every declared volume target of one app.
52	func (s *VolumeService) VolumeTargets(ctx context.Context, app string) ([]domain.VolumeBackupTarget, error) {	102		func (s *VolumeService) VolumeTargets(ctx context.Context, app string) ([]domain.VolumeBackupTarget, error) {
53	if s.state == nil {	103		if s.state == nil {

@@ -214,7 +264,22 @@ func (s *VolumeService) runVolumeBackupsConcurrently(ctx context.Context, target
214	return jobs	264		return jobs
215	}	265		}
216		266
		267	+	// runVolumeBackup archives one target and records the outcome in metrics.
217	func (s *VolumeService) runVolumeBackup(ctx context.Context, target domain.VolumeBackupTarget) domain.VolumeBackupJob {	268		func (s *VolumeService) runVolumeBackup(ctx context.Context, target domain.VolumeBackupTarget) domain.VolumeBackupJob {
		269	+	job := s.archiveVolume(ctx, target)
		270	+	if s.metrics != nil {
		271	+	s.metrics.RecordBackupRun(ctx, domain.BackupRunRecord{
		272	+	BackupTargetRef: volumeTargetRef(target),
		273	+	Succeeded: job.Status == domain.BackupStatusCompleted,
		274	+	Duration: job.CompletedAt.Sub(job.StartedAt),
		275	+	SizeBytes: job.SizeBytes,
		276	+	StartedAt: job.StartedAt,
		277	+	})
		278	+	}
		279	+	return job
		280	+	}
		281	+
		282	+	func (s *VolumeService) archiveVolume(ctx context.Context, target domain.VolumeBackupTarget) domain.VolumeBackupJob {
218	ctx = zerowrap.CtxWithFields(ctx, map[string]any{	283		ctx = zerowrap.CtxWithFields(ctx, map[string]any{
219	"app": target.App,	284		"app": target.App,
220	"service": target.Service,	285		"service": target.Service,

M internal/usecase/backup/volume_service_test.go +9 -0


@@ -90,6 +90,15 @@ func (f *fakeVolumeBackupStorage) ListVolumeArchives(context.Context, string) ([
90	90		return f.jobs, nil
91	91		}
92	92
	93	+	// LatestVolumeArchives returns the configured jobs as-is; the S3 adapter
	94	+	// owns the newest-per-volume selection.
	95	+	func (f *fakeVolumeBackupStorage) LatestVolumeArchives(context.Context) ([]domain.VolumeBackupJob, error) {
	96	+	if f.listErr != nil {
	97	+	return nil, f.listErr
	98	+	}
	99	+	return f.jobs, nil
	100	+	}
	101	+
93	102		func (f *fakeVolumeBackupStorage) DeleteVolumeArchive(context.Context, string) error { return nil }
94	103
95	104		func (f *fakeVolumeBackupStorage) ApplyVolumeRetention(_ context.Context, app string, _ domain.VolumeBackupRetentionPolicy) (int, error) {


@@ -90,6 +90,15 @@ func (f *fakeVolumeBackupStorage) ListVolumeArchives(context.Context, string) ([
90	return f.jobs, nil	90		return f.jobs, nil
91	}	91		}
92		92
		93	+	// LatestVolumeArchives returns the configured jobs as-is; the S3 adapter
		94	+	// owns the newest-per-volume selection.
		95	+	func (f *fakeVolumeBackupStorage) LatestVolumeArchives(context.Context) ([]domain.VolumeBackupJob, error) {
		96	+	if f.listErr != nil {
		97	+	return nil, f.listErr
		98	+	}
		99	+	return f.jobs, nil
		100	+	}
		101	+
93	func (f *fakeVolumeBackupStorage) DeleteVolumeArchive(context.Context, string) error { return nil }	102		func (f *fakeVolumeBackupStorage) DeleteVolumeArchive(context.Context, string) error { return nil }
94		103
95	func (f *fakeVolumeBackupStorage) ApplyVolumeRetention(_ context.Context, app string, _ domain.VolumeBackupRetentionPolicy) (int, error) {	104		func (f *fakeVolumeBackupStorage) ApplyVolumeRetention(_ context.Context, app string, _ domain.VolumeBackupRetentionPolicy) (int, error) {