Merge pull request 'fix(backup): local-time schedules, parallel volume runs, and safe status routes' (#26) from fix/backup-scheduling-and-safety into main
25 files · +718 -105
M docs/cli/backup.md +6 -4
| @@ -99,10 +99,12 @@ gordon backups volume status | |||
| 99 | 99 | ||
| 100 | 100 | ## Scheduling | |
| 101 | 101 | ||
| 102 | - | The installation backup schedule runs every declared database whose own | |
| 103 | - | `schedule` matches the firing tier, and applies retention under the app name. | |
| 104 | - | Volume declarations carry no schedule of their own: the installation volume | |
| 105 | - | backup interval runs the declared volume targets. | |
| 102 | + | Backups run on the installation schedule (`backups.databases.schedule` and | |
| 103 | + | `backups.volumes.schedule`, each with an optional `time` in server local time; | |
| 104 | + | see [Backups](../config/backups.md#schedule)). The database schedule runs every | |
| 105 | + | declared database whose own `schedule` matches the firing tier. Volume | |
| 106 | + | declarations carry no schedule of their own: every declared volume target runs, | |
| 107 | + | up to `max_concurrency` at once. Retention is applied per app after the run. | |
| 106 | 108 | ||
| 107 | 109 | ## JSON Output | |
| 108 | 110 | ||
| @@ -99,10 +99,12 @@ gordon backups volume status | |||||
| 99 | 99 | ||||
| 100 | ## Scheduling | 100 | ## Scheduling | ||
| 101 | 101 | ||||
| 102 | - | The installation backup schedule runs every declared database whose own | 102 | + | Backups run on the installation schedule (`backups.databases.schedule` and |
| 103 | - | `schedule` matches the firing tier, and applies retention under the app name. | 103 | + | `backups.volumes.schedule`, each with an optional `time` in server local time; |
| 104 | - | Volume declarations carry no schedule of their own: the installation volume | 104 | + | see [Backups](../config/backups.md#schedule)). The database schedule runs every |
| 105 | - | backup interval runs the declared volume targets. | 105 | + | declared database whose own `schedule` matches the firing tier. Volume |
| 106 | + | declarations carry no schedule of their own: every declared volume target runs, | |||
| 107 | + | up to `max_concurrency` at once. Retention is applied per app after the run. | |||
| 106 | 108 | ||||
| 107 | ## JSON Output | 109 | ## JSON Output | ||
| 108 | 110 | ||||
M docs/config/backups.md +24 -4
| @@ -9,12 +9,29 @@ Backups are identified by app, service, and the declared database or volume | |||
| 9 | 9 | (see [Apps](../config/apps.md)). Domains are routing addresses and are never a | |
| 10 | 10 | backup identity. | |
| 11 | 11 | ||
| 12 | + | ## Schedule | |
| 13 | + | ||
| 14 | + | Each backup flow runs on one installation-wide schedule. Apps only declare | |
| 15 | + | *what* to back up; the installation decides *when*. | |
| 16 | + | ||
| 17 | + | | `schedule` | Runs | Default `time` | | |
| 18 | + | |---|---|---| | |
| 19 | + | | `hourly` | every hour at the `time` minutes | `00:00` (on the hour) | | |
| 20 | + | | `daily` | every day | `02:00` | | |
| 21 | + | | `weekly` | every Sunday | `03:00` | | |
| 22 | + | | `monthly` | the first day of each month | `04:00` | | |
| 23 | + | ||
| 24 | + | - `time` is `HH:MM` (24-hour) in the **server's local timezone** (`TZ`, or `/etc/localtime`). The run keeps the same wall-clock time across daylight-saving changes. A time skipped by the spring-forward change runs when the gap ends. A time repeated by the fall-back change runs once. | |
| 25 | + | - The next run is computed from the current time when Gordon starts. A run that falls while Gordon is down is skipped, not caught up: no workload runs then, so there is nothing new to back up. | |
| 26 | + | - Gordon logs the schedule and `next_run` at startup (`backup scheduler enabled`, `volume backup scheduler enabled`). | |
| 27 | + | ||
| 12 | 28 | ## Database backups | |
| 13 | 29 | ||
| 14 | 30 | ```toml | |
| 15 | 31 | [backups.databases] | |
| 16 | 32 | enabled = true | |
| 17 | 33 | schedule = "daily" | |
| 34 | + | time = "03:00" | |
| 18 | 35 | storage_dir = "~/.gordon/backups" | |
| 19 | 36 | ||
| 20 | 37 | [backups.databases.retention] | |
| @@ -27,7 +44,8 @@ monthly = 12 | |||
| 27 | 44 | | Key | Default | Description | | |
| 28 | 45 | |---|---:|---| | |
| 29 | 46 | | `backups.databases.enabled` | `false` | Enables PostgreSQL backup service wiring | | |
| 30 | - | | `backups.databases.schedule` | `"daily"` | Scheduler preset: `hourly`, `daily`, `weekly`, `monthly` | | |
| 47 | + | | `backups.databases.schedule` | `"daily"` | Run frequency: `hourly`, `daily`, `weekly`, `monthly`; databases run when their declared `schedule` matches | | |
| 48 | + | | `backups.databases.time` | `""` | `HH:MM` server local time; empty uses the preset default | | |
| 31 | 49 | | `backups.databases.storage_dir` | `""` | Root backup directory; defaults to `{server.data_dir}/backups` | | |
| 32 | 50 | | `backups.databases.retention.*` | `0` | Number of backups to keep per DB and schedule tier | | |
| 33 | 51 | ||
| @@ -36,7 +54,8 @@ monthly = 12 | |||
| 36 | 54 | ```toml | |
| 37 | 55 | [backups.volumes] | |
| 38 | 56 | enabled = true | |
| 39 | - | interval = "24h" | |
| 57 | + | schedule = "daily" | |
| 58 | + | time = "03:30" | |
| 40 | 59 | compression = "gzip" | |
| 41 | 60 | timeout = "2h" | |
| 42 | 61 | max_concurrency = 2 | |
| @@ -59,10 +78,11 @@ keep = 14 | |||
| 59 | 78 | | Key | Default | Description | | |
| 60 | 79 | |---|---:|---| | |
| 61 | 80 | | `backups.volumes.enabled` | `false` | Enables automatic volume archives | | |
| 62 | - | | `backups.volumes.interval` | `"24h"` | Interval between scheduled runs | | |
| 81 | + | | `backups.volumes.schedule` | `"daily"` | Run frequency: `hourly`, `daily`, `weekly`, `monthly` | | |
| 82 | + | | `backups.volumes.time` | `""` | `HH:MM` server local time; empty uses the preset default | | |
| 63 | 83 | | `backups.volumes.compression` | `"gzip"` | `gzip` or `zstd`; helper image must provide the tool | | |
| 64 | 84 | | `backups.volumes.timeout` | `"2h"` | Per-volume archive/upload timeout | | |
| 65 | - | | `backups.volumes.max_concurrency` | `2` | Maximum concurrent volume backups | | |
| 85 | + | | `backups.volumes.max_concurrency` | `2` | Volumes archived in parallel during a scheduled run | | |
| 66 | 86 | | `backups.volumes.helper_image` | `"alpine:3.20"` | Helper container image used for tar/compression | | |
| 67 | 87 | | `backups.volumes.s3.bucket` | `""` | Required when volume backups are enabled | | |
| 68 | 88 | | `backups.volumes.s3.region` | `""` | Required when volume backups are enabled | | |
| @@ -9,12 +9,29 @@ Backups are identified by app, service, and the declared database or volume | |||||
| 9 | (see [Apps](../config/apps.md)). Domains are routing addresses and are never a | 9 | (see [Apps](../config/apps.md)). Domains are routing addresses and are never a | ||
| 10 | backup identity. | 10 | backup identity. | ||
| 11 | 11 | ||||
| 12 | + | ## Schedule | |||
| 13 | + | ||||
| 14 | + | Each backup flow runs on one installation-wide schedule. Apps only declare | |||
| 15 | + | *what* to back up; the installation decides *when*. | |||
| 16 | + | ||||
| 17 | + | | `schedule` | Runs | Default `time` | | |||
| 18 | + | |---|---|---| | |||
| 19 | + | | `hourly` | every hour at the `time` minutes | `00:00` (on the hour) | | |||
| 20 | + | | `daily` | every day | `02:00` | | |||
| 21 | + | | `weekly` | every Sunday | `03:00` | | |||
| 22 | + | | `monthly` | the first day of each month | `04:00` | | |||
| 23 | + | ||||
| 24 | + | - `time` is `HH:MM` (24-hour) in the **server's local timezone** (`TZ`, or `/etc/localtime`). The run keeps the same wall-clock time across daylight-saving changes. A time skipped by the spring-forward change runs when the gap ends. A time repeated by the fall-back change runs once. | |||
| 25 | + | - The next run is computed from the current time when Gordon starts. A run that falls while Gordon is down is skipped, not caught up: no workload runs then, so there is nothing new to back up. | |||
| 26 | + | - Gordon logs the schedule and `next_run` at startup (`backup scheduler enabled`, `volume backup scheduler enabled`). | |||
| 27 | + | ||||
| 12 | ## Database backups | 28 | ## Database backups | ||
| 13 | 29 | ||||
| 14 | ```toml | 30 | ```toml | ||
| 15 | [backups.databases] | 31 | [backups.databases] | ||
| 16 | enabled = true | 32 | enabled = true | ||
| 17 | schedule = "daily" | 33 | schedule = "daily" | ||
| 34 | + | time = "03:00" | |||
| 18 | storage_dir = "~/.gordon/backups" | 35 | storage_dir = "~/.gordon/backups" | ||
| 19 | 36 | ||||
| 20 | [backups.databases.retention] | 37 | [backups.databases.retention] | ||
| @@ -27,7 +44,8 @@ monthly = 12 | |||||
| 27 | | Key | Default | Description | | 44 | | Key | Default | Description | | ||
| 28 | |---|---:|---| | 45 | |---|---:|---| | ||
| 29 | | `backups.databases.enabled` | `false` | Enables PostgreSQL backup service wiring | | 46 | | `backups.databases.enabled` | `false` | Enables PostgreSQL backup service wiring | | ||
| 30 | - | | `backups.databases.schedule` | `"daily"` | Scheduler preset: `hourly`, `daily`, `weekly`, `monthly` | | 47 | + | | `backups.databases.schedule` | `"daily"` | Run frequency: `hourly`, `daily`, `weekly`, `monthly`; databases run when their declared `schedule` matches | |
| 48 | + | | `backups.databases.time` | `""` | `HH:MM` server local time; empty uses the preset default | | |||
| 31 | | `backups.databases.storage_dir` | `""` | Root backup directory; defaults to `{server.data_dir}/backups` | | 49 | | `backups.databases.storage_dir` | `""` | Root backup directory; defaults to `{server.data_dir}/backups` | | ||
| 32 | | `backups.databases.retention.*` | `0` | Number of backups to keep per DB and schedule tier | | 50 | | `backups.databases.retention.*` | `0` | Number of backups to keep per DB and schedule tier | | ||
| 33 | 51 | ||||
| @@ -36,7 +54,8 @@ monthly = 12 | |||||
| 36 | ```toml | 54 | ```toml | ||
| 37 | [backups.volumes] | 55 | [backups.volumes] | ||
| 38 | enabled = true | 56 | enabled = true | ||
| 39 | - | interval = "24h" | 57 | + | schedule = "daily" |
| 58 | + | time = "03:30" | |||
| 40 | compression = "gzip" | 59 | compression = "gzip" | ||
| 41 | timeout = "2h" | 60 | timeout = "2h" | ||
| 42 | max_concurrency = 2 | 61 | max_concurrency = 2 | ||
| @@ -59,10 +78,11 @@ keep = 14 | |||||
| 59 | | Key | Default | Description | | 78 | | Key | Default | Description | | ||
| 60 | |---|---:|---| | 79 | |---|---:|---| | ||
| 61 | | `backups.volumes.enabled` | `false` | Enables automatic volume archives | | 80 | | `backups.volumes.enabled` | `false` | Enables automatic volume archives | | ||
| 62 | - | | `backups.volumes.interval` | `"24h"` | Interval between scheduled runs | | 81 | + | | `backups.volumes.schedule` | `"daily"` | Run frequency: `hourly`, `daily`, `weekly`, `monthly` | |
| 82 | + | | `backups.volumes.time` | `""` | `HH:MM` server local time; empty uses the preset default | | |||
| 63 | | `backups.volumes.compression` | `"gzip"` | `gzip` or `zstd`; helper image must provide the tool | | 83 | | `backups.volumes.compression` | `"gzip"` | `gzip` or `zstd`; helper image must provide the tool | | ||
| 64 | | `backups.volumes.timeout` | `"2h"` | Per-volume archive/upload timeout | | 84 | | `backups.volumes.timeout` | `"2h"` | Per-volume archive/upload timeout | | ||
| 65 | - | | `backups.volumes.max_concurrency` | `2` | Maximum concurrent volume backups | | 85 | + | | `backups.volumes.max_concurrency` | `2` | Volumes archived in parallel during a scheduled run | |
| 66 | | `backups.volumes.helper_image` | `"alpine:3.20"` | Helper container image used for tar/compression | | 86 | | `backups.volumes.helper_image` | `"alpine:3.20"` | Helper container image used for tar/compression | | ||
| 67 | | `backups.volumes.s3.bucket` | `""` | Required when volume backups are enabled | | 87 | | `backups.volumes.s3.bucket` | `""` | Required when volume backups are enabled | | ||
| 68 | | `backups.volumes.s3.region` | `""` | Required when volume backups are enabled | | 88 | | `backups.volumes.s3.region` | `""` | Required when volume backups are enabled | | ||
M docs/config/index.md +6 -2
| @@ -183,8 +183,12 @@ Docker Hub (`docker.io` and `registry-1.docker.io`), `ghcr.io`, `quay.io`, and G | |||
| 183 | 183 | | `volumes.preserve` | `true` | | |
| 184 | 184 | | `network_isolation.enabled` | `true` | | |
| 185 | 185 | | `network_isolation.internal` | `false` | | |
| 186 | - | | `backups.enabled` | `false` | | |
| 187 | - | | `backups.schedule` | `"daily"` (`"hourly"`, `"daily"`, `"weekly"`, `"monthly"`) | | |
| 186 | + | | `backups.databases.enabled` | `false` | | |
| 187 | + | | `backups.databases.schedule` | `"daily"` (`"hourly"`, `"daily"`, `"weekly"`, `"monthly"`) | | |
| 188 | + | | `backups.databases.time` | `""` (preset default, server local time) | | |
| 189 | + | | `backups.volumes.enabled` | `false` | | |
| 190 | + | | `backups.volumes.schedule` | `"daily"` | | |
| 191 | + | | `backups.volumes.time` | `""` (preset default, server local time) | | |
| 188 | 192 | | `images.allowed_registries` | `[]` | | |
| 189 | 193 | | `images.require_digest` | `false` | | |
| 190 | 194 | | `images.prune.enabled` | `false` | | |
| @@ -183,8 +183,12 @@ Docker Hub (`docker.io` and `registry-1.docker.io`), `ghcr.io`, `quay.io`, and G | |||||
| 183 | | `volumes.preserve` | `true` | | 183 | | `volumes.preserve` | `true` | | ||
| 184 | | `network_isolation.enabled` | `true` | | 184 | | `network_isolation.enabled` | `true` | | ||
| 185 | | `network_isolation.internal` | `false` | | 185 | | `network_isolation.internal` | `false` | | ||
| 186 | - | | `backups.enabled` | `false` | | 186 | + | | `backups.databases.enabled` | `false` | |
| 187 | - | | `backups.schedule` | `"daily"` (`"hourly"`, `"daily"`, `"weekly"`, `"monthly"`) | | 187 | + | | `backups.databases.schedule` | `"daily"` (`"hourly"`, `"daily"`, `"weekly"`, `"monthly"`) | |
| 188 | + | | `backups.databases.time` | `""` (preset default, server local time) | | |||
| 189 | + | | `backups.volumes.enabled` | `false` | | |||
| 190 | + | | `backups.volumes.schedule` | `"daily"` | | |||
| 191 | + | | `backups.volumes.time` | `""` (preset default, server local time) | | |||
| 188 | | `images.allowed_registries` | `[]` | | 192 | | `images.allowed_registries` | `[]` | | ||
| 189 | | `images.require_digest` | `false` | | 193 | | `images.require_digest` | `false` | | ||
| 190 | | `images.prune.enabled` | `false` | | 194 | | `images.prune.enabled` | `false` | | ||
M docs/config/reference.md +25 -12
| @@ -208,17 +208,24 @@ preserve = true # Keep volumes when containers are | |||
| 208 | 208 | # ============================================================================= | |
| 209 | 209 | # BACKUPS | |
| 210 | 210 | # ============================================================================= | |
| 211 | - | [backups] | |
| 212 | - | enabled = false # Enable backup service | |
| 211 | + | [backups.databases] | |
| 212 | + | enabled = false # Enable PostgreSQL backups | |
| 213 | 213 | schedule = "daily" # "hourly", "daily", "weekly", "monthly" | |
| 214 | + | time = "" # "HH:MM" server local time (empty: preset default) | |
| 214 | 215 | storage_dir = "" # Backup root (default: {data_dir}/backups) | |
| 215 | 216 | ||
| 216 | - | [backups.retention] | |
| 217 | + | [backups.databases.retention] | |
| 217 | 218 | hourly = 0 # Keep N hourly backups per DB | |
| 218 | 219 | daily = 0 # Keep N daily backups per DB | |
| 219 | 220 | weekly = 0 # Keep N weekly backups per DB | |
| 220 | 221 | monthly = 0 # Keep N monthly backups per DB | |
| 221 | 222 | ||
| 223 | + | [backups.volumes] | |
| 224 | + | enabled = false # Enable S3 volume archives (see backups.md) | |
| 225 | + | schedule = "daily" # "hourly", "daily", "weekly", "monthly" | |
| 226 | + | time = "" # "HH:MM" server local time (empty: preset default) | |
| 227 | + | max_concurrency = 2 # Volumes archived in parallel | |
| 228 | + | ||
| 222 | 229 | # ============================================================================= | |
| 223 | 230 | # IMAGES | |
| 224 | 231 | # ============================================================================= | |
| @@ -338,22 +345,28 @@ keep_last = 3 # Keep N newest tags per repository | |||
| 338 | 345 | | `services[].readiness.timeout` | default wait | Positive readiness timeout when set | | |
| 339 | 346 | | `services[].cleanup.preserve_volumes` | `true` | Preserve managed image-discovered volumes on cleanup | | |
| 340 | 347 | | `services[].cleanup.remove_container` | `true` | Remove old, disabled, or removed service containers | | |
| 341 | - | | `backups.enabled` | `false` | Backup service disabled | | |
| 342 | - | | `backups.schedule` | `"daily"` | Backup scheduler preset | | |
| 343 | - | | `backups.storage_dir` | `""` | Uses `{server.data_dir}/backups` when empty | | |
| 344 | - | | `backups.retention.hourly` | `0` | Keep no hourly backups by default | | |
| 345 | - | | `backups.retention.daily` | `0` | Keep no daily backups by default (recommend `7`) | | |
| 346 | - | | `backups.retention.weekly` | `0` | Keep no weekly backups by default | | |
| 347 | - | | `backups.retention.monthly` | `0` | Keep no monthly backups by default | | |
| 348 | + | | `backups.databases.enabled` | `false` | PostgreSQL backups disabled | | |
| 349 | + | | `backups.databases.schedule` | `"daily"` | `hourly`, `daily`, `weekly`, or `monthly` | | |
| 350 | + | | `backups.databases.time` | `""` | `HH:MM` server local time; empty uses the preset default | | |
| 351 | + | | `backups.databases.storage_dir` | `""` | Uses `{server.data_dir}/backups` when empty | | |
| 352 | + | | `backups.databases.retention.hourly` | `0` | Keep no hourly backups by default | | |
| 353 | + | | `backups.databases.retention.daily` | `0` | Keep no daily backups by default (recommend `7`) | | |
| 354 | + | | `backups.databases.retention.weekly` | `0` | Keep no weekly backups by default | | |
| 355 | + | | `backups.databases.retention.monthly` | `0` | Keep no monthly backups by default | | |
| 356 | + | | `backups.volumes.enabled` | `false` | S3 volume archives disabled | | |
| 357 | + | | `backups.volumes.schedule` | `"daily"` | `hourly`, `daily`, `weekly`, or `monthly` | | |
| 358 | + | | `backups.volumes.time` | `""` | `HH:MM` server local time; empty uses the preset default | | |
| 359 | + | | `backups.volumes.max_concurrency` | `2` | Volumes archived in parallel | | |
| 360 | + | | `backups.volumes.retention.keep` | `14` | Archives kept per app + volume | | |
| 348 | 361 | | `images.allowed_registries` | `[]` | Additional exact hostname+port entries. Defaults allow Docker Hub (`docker.io` and `registry-1.docker.io`), `ghcr.io`, `quay.io`, and Gordon's registry. This allowlist does not configure registry credentials and is not DNS/IP or runtime egress enforcement. | | |
| 349 | 362 | | `images.require_digest` | `false` | Require valid SHA-256 digest-pinned references for every registry, including Gordon | | |
| 350 | 363 | | `images.prune.enabled` | `false` | Scheduled image cleanup disabled | | |
| 351 | 364 | | `images.prune.schedule` | `"daily"` | Cleanup schedule preset | | |
| 352 | 365 | | `images.prune.keep_last` | `3` | Number of recent tags kept per repository | | |
| 353 | 366 | ||
| 354 | - | Note: for all `backups.retention.*` keys, `0` means keep no backups for that retention tier. | |
| 367 | + | Note: for all `backups.databases.retention.*` keys, `0` means keep no backups for that retention tier. The top-level `backups.enabled`, `backups.schedule`, `backups.storage_dir`, and `backups.retention.*` keys are aliases of the `backups.databases.*` keys. | |
| 355 | 368 | ||
| 356 | - | Database (`backups.databases.*`) and S3 volume backup (`backups.volumes.*`) keys, including `backups.volumes.s3.storage_class`, are listed in [Backups](./backups.md). | |
| 369 | + | Every volume backup key, including compression, the helper image, and the `backups.volumes.s3.*` settings, is listed in [Backups](./backups.md). | |
| 357 | 370 | ||
| 358 | 371 | ## App Manifest HTTP Interfaces | |
| 359 | 372 | ||
| @@ -208,17 +208,24 @@ preserve = true # Keep volumes when containers are | |||||
| 208 | # ============================================================================= | 208 | # ============================================================================= | ||
| 209 | # BACKUPS | 209 | # BACKUPS | ||
| 210 | # ============================================================================= | 210 | # ============================================================================= | ||
| 211 | - | [backups] | 211 | + | [backups.databases] |
| 212 | - | enabled = false # Enable backup service | 212 | + | enabled = false # Enable PostgreSQL backups |
| 213 | schedule = "daily" # "hourly", "daily", "weekly", "monthly" | 213 | schedule = "daily" # "hourly", "daily", "weekly", "monthly" | ||
| 214 | + | time = "" # "HH:MM" server local time (empty: preset default) | |||
| 214 | storage_dir = "" # Backup root (default: {data_dir}/backups) | 215 | storage_dir = "" # Backup root (default: {data_dir}/backups) | ||
| 215 | 216 | ||||
| 216 | - | [backups.retention] | 217 | + | [backups.databases.retention] |
| 217 | hourly = 0 # Keep N hourly backups per DB | 218 | hourly = 0 # Keep N hourly backups per DB | ||
| 218 | daily = 0 # Keep N daily backups per DB | 219 | daily = 0 # Keep N daily backups per DB | ||
| 219 | weekly = 0 # Keep N weekly backups per DB | 220 | weekly = 0 # Keep N weekly backups per DB | ||
| 220 | monthly = 0 # Keep N monthly backups per DB | 221 | monthly = 0 # Keep N monthly backups per DB | ||
| 221 | 222 | ||||
| 223 | + | [backups.volumes] | |||
| 224 | + | enabled = false # Enable S3 volume archives (see backups.md) | |||
| 225 | + | schedule = "daily" # "hourly", "daily", "weekly", "monthly" | |||
| 226 | + | time = "" # "HH:MM" server local time (empty: preset default) | |||
| 227 | + | max_concurrency = 2 # Volumes archived in parallel | |||
| 228 | + | ||||
| 222 | # ============================================================================= | 229 | # ============================================================================= | ||
| 223 | # IMAGES | 230 | # IMAGES | ||
| 224 | # ============================================================================= | 231 | # ============================================================================= | ||
| @@ -338,22 +345,28 @@ keep_last = 3 # Keep N newest tags per repository | |||||
| 338 | | `services[].readiness.timeout` | default wait | Positive readiness timeout when set | | 345 | | `services[].readiness.timeout` | default wait | Positive readiness timeout when set | | ||
| 339 | | `services[].cleanup.preserve_volumes` | `true` | Preserve managed image-discovered volumes on cleanup | | 346 | | `services[].cleanup.preserve_volumes` | `true` | Preserve managed image-discovered volumes on cleanup | | ||
| 340 | | `services[].cleanup.remove_container` | `true` | Remove old, disabled, or removed service containers | | 347 | | `services[].cleanup.remove_container` | `true` | Remove old, disabled, or removed service containers | | ||
| 341 | - | | `backups.enabled` | `false` | Backup service disabled | | 348 | + | | `backups.databases.enabled` | `false` | PostgreSQL backups disabled | |
| 342 | - | | `backups.schedule` | `"daily"` | Backup scheduler preset | | 349 | + | | `backups.databases.schedule` | `"daily"` | `hourly`, `daily`, `weekly`, or `monthly` | |
| 343 | - | | `backups.storage_dir` | `""` | Uses `{server.data_dir}/backups` when empty | | 350 | + | | `backups.databases.time` | `""` | `HH:MM` server local time; empty uses the preset default | |
| 344 | - | | `backups.retention.hourly` | `0` | Keep no hourly backups by default | | 351 | + | | `backups.databases.storage_dir` | `""` | Uses `{server.data_dir}/backups` when empty | |
| 345 | - | | `backups.retention.daily` | `0` | Keep no daily backups by default (recommend `7`) | | 352 | + | | `backups.databases.retention.hourly` | `0` | Keep no hourly backups by default | |
| 346 | - | | `backups.retention.weekly` | `0` | Keep no weekly backups by default | | 353 | + | | `backups.databases.retention.daily` | `0` | Keep no daily backups by default (recommend `7`) | |
| 347 | - | | `backups.retention.monthly` | `0` | Keep no monthly backups by default | | 354 | + | | `backups.databases.retention.weekly` | `0` | Keep no weekly backups by default | |
| 355 | + | | `backups.databases.retention.monthly` | `0` | Keep no monthly backups by default | | |||
| 356 | + | | `backups.volumes.enabled` | `false` | S3 volume archives disabled | | |||
| 357 | + | | `backups.volumes.schedule` | `"daily"` | `hourly`, `daily`, `weekly`, or `monthly` | | |||
| 358 | + | | `backups.volumes.time` | `""` | `HH:MM` server local time; empty uses the preset default | | |||
| 359 | + | | `backups.volumes.max_concurrency` | `2` | Volumes archived in parallel | | |||
| 360 | + | | `backups.volumes.retention.keep` | `14` | Archives kept per app + volume | | |||
| 348 | | `images.allowed_registries` | `[]` | Additional exact hostname+port entries. Defaults allow Docker Hub (`docker.io` and `registry-1.docker.io`), `ghcr.io`, `quay.io`, and Gordon's registry. This allowlist does not configure registry credentials and is not DNS/IP or runtime egress enforcement. | | 361 | | `images.allowed_registries` | `[]` | Additional exact hostname+port entries. Defaults allow Docker Hub (`docker.io` and `registry-1.docker.io`), `ghcr.io`, `quay.io`, and Gordon's registry. This allowlist does not configure registry credentials and is not DNS/IP or runtime egress enforcement. | | ||
| 349 | | `images.require_digest` | `false` | Require valid SHA-256 digest-pinned references for every registry, including Gordon | | 362 | | `images.require_digest` | `false` | Require valid SHA-256 digest-pinned references for every registry, including Gordon | | ||
| 350 | | `images.prune.enabled` | `false` | Scheduled image cleanup disabled | | 363 | | `images.prune.enabled` | `false` | Scheduled image cleanup disabled | | ||
| 351 | | `images.prune.schedule` | `"daily"` | Cleanup schedule preset | | 364 | | `images.prune.schedule` | `"daily"` | Cleanup schedule preset | | ||
| 352 | | `images.prune.keep_last` | `3` | Number of recent tags kept per repository | | 365 | | `images.prune.keep_last` | `3` | Number of recent tags kept per repository | | ||
| 353 | 366 | ||||
| 354 | - | Note: for all `backups.retention.*` keys, `0` means keep no backups for that retention tier. | 367 | + | Note: for all `backups.databases.retention.*` keys, `0` means keep no backups for that retention tier. The top-level `backups.enabled`, `backups.schedule`, `backups.storage_dir`, and `backups.retention.*` keys are aliases of the `backups.databases.*` keys. |
| 355 | 368 | ||||
| 356 | - | Database (`backups.databases.*`) and S3 volume backup (`backups.volumes.*`) keys, including `backups.volumes.s3.storage_class`, are listed in [Backups](./backups.md). | 369 | + | Every volume backup key, including compression, the helper image, and the `backups.volumes.s3.*` settings, is listed in [Backups](./backups.md). |
| 357 | 370 | ||||
| 358 | ## App Manifest HTTP Interfaces | 371 | ## App Manifest HTTP Interfaces | ||
| 359 | 372 | ||||
M gordon.toml.example +8 -1
| @@ -151,9 +151,15 @@ metrics = true | |||
| 151 | 151 | logs = true | |
| 152 | 152 | trace_sample_rate = 1.0 | |
| 153 | 153 | ||
| 154 | + | # Backup schedules: hourly | daily | weekly (Sunday) | monthly (day 1). | |
| 155 | + | # time is HH:MM in the server's local timezone (TZ or /etc/localtime); | |
| 156 | + | # hourly uses only the minutes. Empty time keeps the preset default | |
| 157 | + | # (daily 02:00, weekly 03:00, monthly 04:00). A run missed while Gordon | |
| 158 | + | # is down is skipped, not caught up. | |
| 154 | 159 | [backups.databases] | |
| 155 | 160 | enabled = false | |
| 156 | 161 | schedule = "daily" | |
| 162 | + | # time = "03:00" | |
| 157 | 163 | storage_dir = "" | |
| 158 | 164 | ||
| 159 | 165 | [backups.databases.retention] | |
| @@ -164,7 +170,8 @@ monthly = 0 | |||
| 164 | 170 | ||
| 165 | 171 | [backups.volumes] | |
| 166 | 172 | enabled = false | |
| 167 | - | interval = "24h" | |
| 173 | + | schedule = "daily" | |
| 174 | + | # time = "03:30" | |
| 168 | 175 | compression = "gzip" | |
| 169 | 176 | timeout = "2h" | |
| 170 | 177 | max_concurrency = 2 | |
| @@ -151,9 +151,15 @@ metrics = true | |||||
| 151 | logs = true | 151 | logs = true | ||
| 152 | trace_sample_rate = 1.0 | 152 | trace_sample_rate = 1.0 | ||
| 153 | 153 | ||||
| 154 | + | # Backup schedules: hourly | daily | weekly (Sunday) | monthly (day 1). | |||
| 155 | + | # time is HH:MM in the server's local timezone (TZ or /etc/localtime); | |||
| 156 | + | # hourly uses only the minutes. Empty time keeps the preset default | |||
| 157 | + | # (daily 02:00, weekly 03:00, monthly 04:00). A run missed while Gordon | |||
| 158 | + | # is down is skipped, not caught up. | |||
| 154 | [backups.databases] | 159 | [backups.databases] | ||
| 155 | enabled = false | 160 | enabled = false | ||
| 156 | schedule = "daily" | 161 | schedule = "daily" | ||
| 162 | + | # time = "03:00" | |||
| 157 | storage_dir = "" | 163 | storage_dir = "" | ||
| 158 | 164 | ||||
| 159 | [backups.databases.retention] | 165 | [backups.databases.retention] | ||
| @@ -164,7 +170,8 @@ monthly = 0 | |||||
| 164 | 170 | ||||
| 165 | [backups.volumes] | 171 | [backups.volumes] | ||
| 166 | enabled = false | 172 | enabled = false | ||
| 167 | - | interval = "24h" | 173 | + | schedule = "daily" |
| 174 | + | # time = "03:30" | |||
| 168 | compression = "gzip" | 175 | compression = "gzip" | ||
| 169 | timeout = "2h" | 176 | timeout = "2h" | ||
| 170 | max_concurrency = 2 | 177 | max_concurrency = 2 | ||
M internal/adapters/out/s3/volume_backup_storage_test.go +25 -0
| @@ -141,6 +141,31 @@ func TestVolumeBackupStorageSendsConfiguredStorageClass(t *testing.T) { | |||
| 141 | 141 | assert.Equal(t, "ONEZONE_IA", string(uploader.last.StorageClass)) | |
| 142 | 142 | } | |
| 143 | 143 | ||
| 144 | + | // TestVolumeBackupStorageRetentionMixesLongAndShortIDs proves archives | |
| 145 | + | // written with the former timestamped job IDs stay listed and age out | |
| 146 | + | // under retention next to archives with short IDs. | |
| 147 | + | func TestVolumeBackupStorageRetentionMixesLongAndShortIDs(t *testing.T) { | |
| 148 | + | client := newFakeVolumeS3Client() | |
| 149 | + | storage := NewVolumeBackupStorageWithClients(domain.VolumeBackupConfig{S3Bucket: "bucket", S3Prefix: "gordon"}, client, &fakeVolumeUploader{client: client}) | |
| 150 | + | longID := "gordon/apps/forgejo/volumes/forgejo-data/20261017T052537Z-2026-10-17T05_25_37.725487298Z-7cddcd8f.tar.gz" | |
| 151 | + | short1 := "gordon/apps/forgejo/volumes/forgejo-data/20261018T013000Z-1a2b3c4d.tar.gz" | |
| 152 | + | short2 := "gordon/apps/forgejo/volumes/forgejo-data/20261019T013000Z-5e6f7a8b.tar.gz" | |
| 153 | + | for _, key := range []string{longID, short1, short2} { | |
| 154 | + | client.objects[key] = []byte("x") | |
| 155 | + | } | |
| 156 | + | ||
| 157 | + | jobs, err := storage.ListVolumeArchives(context.Background(), "forgejo") | |
| 158 | + | require.NoError(t, err) | |
| 159 | + | require.Len(t, jobs, 3) | |
| 160 | + | assert.Equal(t, "5e6f7a8b", jobs[0].ID) | |
| 161 | + | assert.Equal(t, "2026-10-17T05_25_37.725487298Z-7cddcd8f", jobs[2].ID) | |
| 162 | + | ||
| 163 | + | deleted, err := storage.ApplyVolumeRetention(context.Background(), "forgejo", domain.VolumeBackupRetentionPolicy{Keep: 2}) | |
| 164 | + | require.NoError(t, err) | |
| 165 | + | assert.Equal(t, 1, deleted) | |
| 166 | + | assert.Equal(t, []string{longID}, client.deleted) | |
| 167 | + | } | |
| 168 | + | ||
| 144 | 169 | func TestVolumeBackupStorageRetentionUsesParsedKeyTimestamp(t *testing.T) { | |
| 145 | 170 | client := newFakeVolumeS3Client() | |
| 146 | 171 | storage := NewVolumeBackupStorageWithClients(domain.VolumeBackupConfig{S3Bucket: "bucket"}, client, &fakeVolumeUploader{client: client}) | |
| @@ -141,6 +141,31 @@ func TestVolumeBackupStorageSendsConfiguredStorageClass(t *testing.T) { | |||||
| 141 | assert.Equal(t, "ONEZONE_IA", string(uploader.last.StorageClass)) | 141 | assert.Equal(t, "ONEZONE_IA", string(uploader.last.StorageClass)) | ||
| 142 | } | 142 | } | ||
| 143 | 143 | ||||
| 144 | + | // TestVolumeBackupStorageRetentionMixesLongAndShortIDs proves archives | |||
| 145 | + | // written with the former timestamped job IDs stay listed and age out | |||
| 146 | + | // under retention next to archives with short IDs. | |||
| 147 | + | func TestVolumeBackupStorageRetentionMixesLongAndShortIDs(t *testing.T) { | |||
| 148 | + | client := newFakeVolumeS3Client() | |||
| 149 | + | storage := NewVolumeBackupStorageWithClients(domain.VolumeBackupConfig{S3Bucket: "bucket", S3Prefix: "gordon"}, client, &fakeVolumeUploader{client: client}) | |||
| 150 | + | longID := "gordon/apps/forgejo/volumes/forgejo-data/20261017T052537Z-2026-10-17T05_25_37.725487298Z-7cddcd8f.tar.gz" | |||
| 151 | + | short1 := "gordon/apps/forgejo/volumes/forgejo-data/20261018T013000Z-1a2b3c4d.tar.gz" | |||
| 152 | + | short2 := "gordon/apps/forgejo/volumes/forgejo-data/20261019T013000Z-5e6f7a8b.tar.gz" | |||
| 153 | + | for _, key := range []string{longID, short1, short2} { | |||
| 154 | + | client.objects[key] = []byte("x") | |||
| 155 | + | } | |||
| 156 | + | ||||
| 157 | + | jobs, err := storage.ListVolumeArchives(context.Background(), "forgejo") | |||
| 158 | + | require.NoError(t, err) | |||
| 159 | + | require.Len(t, jobs, 3) | |||
| 160 | + | assert.Equal(t, "5e6f7a8b", jobs[0].ID) | |||
| 161 | + | assert.Equal(t, "2026-10-17T05_25_37.725487298Z-7cddcd8f", jobs[2].ID) | |||
| 162 | + | ||||
| 163 | + | deleted, err := storage.ApplyVolumeRetention(context.Background(), "forgejo", domain.VolumeBackupRetentionPolicy{Keep: 2}) | |||
| 164 | + | require.NoError(t, err) | |||
| 165 | + | assert.Equal(t, 1, deleted) | |||
| 166 | + | assert.Equal(t, []string{longID}, client.deleted) | |||
| 167 | + | } | |||
| 168 | + | ||||
| 144 | func TestVolumeBackupStorageRetentionUsesParsedKeyTimestamp(t *testing.T) { | 169 | func TestVolumeBackupStorageRetentionUsesParsedKeyTimestamp(t *testing.T) { | ||
| 145 | client := newFakeVolumeS3Client() | 170 | client := newFakeVolumeS3Client() | ||
| 146 | storage := NewVolumeBackupStorageWithClients(domain.VolumeBackupConfig{S3Bucket: "bucket"}, client, &fakeVolumeUploader{client: client}) | 171 | storage := NewVolumeBackupStorageWithClients(domain.VolumeBackupConfig{S3Bucket: "bucket"}, client, &fakeVolumeUploader{client: client}) | ||
M internal/app/backup_wiring.go +4 -2
| @@ -19,6 +19,7 @@ import ( | |||
| 19 | 19 | type databaseBackupSettingsConfig struct { | |
| 20 | 20 | Enabled bool | |
| 21 | 21 | Schedule string | |
| 22 | + | Time string | |
| 22 | 23 | StorageDir string | |
| 23 | 24 | Retention struct { | |
| 24 | 25 | Hourly int | |
| @@ -32,6 +33,7 @@ func databaseBackupSettings(cfg Config) databaseBackupSettingsConfig { | |||
| 32 | 33 | out := databaseBackupSettingsConfig{ | |
| 33 | 34 | Enabled: cfg.Backups.Databases.Enabled, | |
| 34 | 35 | Schedule: cfg.Backups.Databases.Schedule, | |
| 36 | + | Time: cfg.Backups.Databases.Time, | |
| 35 | 37 | StorageDir: cfg.Backups.Databases.StorageDir, | |
| 36 | 38 | } | |
| 37 | 39 | out.Retention.Hourly = cfg.Backups.Databases.Retention.Hourly | |
| @@ -157,7 +159,7 @@ func validateBackupRetention(cfg Config) (domain.RetentionPolicy, error) { | |||
| 157 | 159 | ||
| 158 | 160 | func validateVolumeBackupConfig(cfg Config) (domain.VolumeBackupConfig, error) { | |
| 159 | 161 | volumeCfg := cfg.Backups.Volumes | |
| 160 | - | interval, err := parsePositiveDurationDefault(volumeCfg.Interval, "24h", "backups.volumes.interval") | |
| 162 | + | schedule, err := resolveCronSchedule(volumeCfg.Schedule, volumeCfg.Time, "backups.volumes") | |
| 161 | 163 | if err != nil { | |
| 162 | 164 | return domain.VolumeBackupConfig{}, err | |
| 163 | 165 | } | |
| @@ -193,7 +195,7 @@ func validateVolumeBackupConfig(cfg Config) (domain.VolumeBackupConfig, error) { | |||
| 193 | 195 | } | |
| 194 | 196 | return domain.VolumeBackupConfig{ | |
| 195 | 197 | Enabled: volumeCfg.Enabled, | |
| 196 | - | Interval: interval, | |
| 198 | + | Schedule: schedule, | |
| 197 | 199 | Compression: compression, | |
| 198 | 200 | Retention: domain.VolumeBackupRetentionPolicy{Keep: volumeCfg.Retention.Keep}, | |
| 199 | 201 | Timeout: timeout, | |
| @@ -19,6 +19,7 @@ import ( | |||||
| 19 | type databaseBackupSettingsConfig struct { | 19 | type databaseBackupSettingsConfig struct { | ||
| 20 | Enabled bool | 20 | Enabled bool | ||
| 21 | Schedule string | 21 | Schedule string | ||
| 22 | + | Time string | |||
| 22 | StorageDir string | 23 | StorageDir string | ||
| 23 | Retention struct { | 24 | Retention struct { | ||
| 24 | Hourly int | 25 | Hourly int | ||
| @@ -32,6 +33,7 @@ func databaseBackupSettings(cfg Config) databaseBackupSettingsConfig { | |||||
| 32 | out := databaseBackupSettingsConfig{ | 33 | out := databaseBackupSettingsConfig{ | ||
| 33 | Enabled: cfg.Backups.Databases.Enabled, | 34 | Enabled: cfg.Backups.Databases.Enabled, | ||
| 34 | Schedule: cfg.Backups.Databases.Schedule, | 35 | Schedule: cfg.Backups.Databases.Schedule, | ||
| 36 | + | Time: cfg.Backups.Databases.Time, | |||
| 35 | StorageDir: cfg.Backups.Databases.StorageDir, | 37 | StorageDir: cfg.Backups.Databases.StorageDir, | ||
| 36 | } | 38 | } | ||
| 37 | out.Retention.Hourly = cfg.Backups.Databases.Retention.Hourly | 39 | out.Retention.Hourly = cfg.Backups.Databases.Retention.Hourly | ||
| @@ -157,7 +159,7 @@ func validateBackupRetention(cfg Config) (domain.RetentionPolicy, error) { | |||||
| 157 | 159 | ||||
| 158 | func validateVolumeBackupConfig(cfg Config) (domain.VolumeBackupConfig, error) { | 160 | func validateVolumeBackupConfig(cfg Config) (domain.VolumeBackupConfig, error) { | ||
| 159 | volumeCfg := cfg.Backups.Volumes | 161 | volumeCfg := cfg.Backups.Volumes | ||
| 160 | - | interval, err := parsePositiveDurationDefault(volumeCfg.Interval, "24h", "backups.volumes.interval") | 162 | + | schedule, err := resolveCronSchedule(volumeCfg.Schedule, volumeCfg.Time, "backups.volumes") |
| 161 | if err != nil { | 163 | if err != nil { | ||
| 162 | return domain.VolumeBackupConfig{}, err | 164 | return domain.VolumeBackupConfig{}, err | ||
| 163 | } | 165 | } | ||
| @@ -193,7 +195,7 @@ func validateVolumeBackupConfig(cfg Config) (domain.VolumeBackupConfig, error) { | |||||
| 193 | } | 195 | } | ||
| 194 | return domain.VolumeBackupConfig{ | 196 | return domain.VolumeBackupConfig{ | ||
| 195 | Enabled: volumeCfg.Enabled, | 197 | Enabled: volumeCfg.Enabled, | ||
| 196 | - | Interval: interval, | 198 | + | Schedule: schedule, |
| 197 | Compression: compression, | 199 | Compression: compression, | ||
| 198 | Retention: domain.VolumeBackupRetentionPolicy{Keep: volumeCfg.Retention.Keep}, | 200 | Retention: domain.VolumeBackupRetentionPolicy{Keep: volumeCfg.Retention.Keep}, | ||
| 199 | Timeout: timeout, | 201 | Timeout: timeout, | ||
M internal/app/config.go +7 -2
| @@ -138,6 +138,7 @@ type Config struct { | |||
| 138 | 138 | Databases struct { | |
| 139 | 139 | Enabled bool `mapstructure:"enabled"` | |
| 140 | 140 | Schedule string `mapstructure:"schedule"` | |
| 141 | + | Time string `mapstructure:"time"` | |
| 141 | 142 | StorageDir string `mapstructure:"storage_dir"` | |
| 142 | 143 | Retention struct { | |
| 143 | 144 | Hourly int `mapstructure:"hourly"` | |
| @@ -148,7 +149,8 @@ type Config struct { | |||
| 148 | 149 | } `mapstructure:"databases"` | |
| 149 | 150 | Volumes struct { | |
| 150 | 151 | Enabled bool `mapstructure:"enabled"` | |
| 151 | - | Interval string `mapstructure:"interval"` | |
| 152 | + | Schedule string `mapstructure:"schedule"` | |
| 153 | + | Time string `mapstructure:"time"` | |
| 152 | 154 | Compression string `mapstructure:"compression"` | |
| 153 | 155 | Timeout string `mapstructure:"timeout"` | |
| 154 | 156 | MaxConcurrency int `mapstructure:"max_concurrency"` | |
| @@ -267,6 +269,7 @@ var retiredAppConfigKeys = []struct { | |||
| 267 | 269 | {"auto_route_allowed_domains", "feature removed; declare explicit interfaces"}, | |
| 268 | 270 | {"previews", "staging is an ordinary app file"}, | |
| 269 | 271 | {"env", "remove the [env] section; declare [env] and [services.<name>.secrets] in app files"}, | |
| 272 | + | {"backups.volumes.interval", "use backups.volumes.schedule and backups.volumes.time (server local time)"}, | |
| 270 | 273 | } | |
| 271 | 274 | ||
| 272 | 275 | // validateRetiredAppConfig rejects obsolete application configuration | |
| @@ -414,13 +417,15 @@ func loadConfig(v *viper.Viper, configPath string) error { | |||
| 414 | 417 | v.SetDefault("volumes.preserve", true) | |
| 415 | 418 | v.SetDefault("backups.databases.enabled", false) | |
| 416 | 419 | v.SetDefault("backups.databases.schedule", string(domain.ScheduleDaily)) | |
| 420 | + | v.SetDefault("backups.databases.time", "") | |
| 417 | 421 | v.SetDefault("backups.databases.storage_dir", "") | |
| 418 | 422 | v.SetDefault("backups.databases.retention.hourly", 0) | |
| 419 | 423 | v.SetDefault("backups.databases.retention.daily", 0) | |
| 420 | 424 | v.SetDefault("backups.databases.retention.weekly", 0) | |
| 421 | 425 | v.SetDefault("backups.databases.retention.monthly", 0) | |
| 422 | 426 | v.SetDefault("backups.volumes.enabled", false) | |
| 423 | - | v.SetDefault("backups.volumes.interval", "24h") | |
| 427 | + | v.SetDefault("backups.volumes.schedule", string(domain.ScheduleDaily)) | |
| 428 | + | v.SetDefault("backups.volumes.time", "") | |
| 424 | 429 | v.SetDefault("backups.volumes.compression", string(domain.VolumeBackupCompressionGzip)) | |
| 425 | 430 | v.SetDefault("backups.volumes.timeout", "2h") | |
| 426 | 431 | v.SetDefault("backups.volumes.max_concurrency", 2) | |
| @@ -138,6 +138,7 @@ type Config struct { | |||||
| 138 | Databases struct { | 138 | Databases struct { | ||
| 139 | Enabled bool `mapstructure:"enabled"` | 139 | Enabled bool `mapstructure:"enabled"` | ||
| 140 | Schedule string `mapstructure:"schedule"` | 140 | Schedule string `mapstructure:"schedule"` | ||
| 141 | + | Time string `mapstructure:"time"` | |||
| 141 | StorageDir string `mapstructure:"storage_dir"` | 142 | StorageDir string `mapstructure:"storage_dir"` | ||
| 142 | Retention struct { | 143 | Retention struct { | ||
| 143 | Hourly int `mapstructure:"hourly"` | 144 | Hourly int `mapstructure:"hourly"` | ||
| @@ -148,7 +149,8 @@ type Config struct { | |||||
| 148 | } `mapstructure:"databases"` | 149 | } `mapstructure:"databases"` | ||
| 149 | Volumes struct { | 150 | Volumes struct { | ||
| 150 | Enabled bool `mapstructure:"enabled"` | 151 | Enabled bool `mapstructure:"enabled"` | ||
| 151 | - | Interval string `mapstructure:"interval"` | 152 | + | Schedule string `mapstructure:"schedule"` |
| 153 | + | Time string `mapstructure:"time"` | |||
| 152 | Compression string `mapstructure:"compression"` | 154 | Compression string `mapstructure:"compression"` | ||
| 153 | Timeout string `mapstructure:"timeout"` | 155 | Timeout string `mapstructure:"timeout"` | ||
| 154 | MaxConcurrency int `mapstructure:"max_concurrency"` | 156 | MaxConcurrency int `mapstructure:"max_concurrency"` | ||
| @@ -267,6 +269,7 @@ var retiredAppConfigKeys = []struct { | |||||
| 267 | {"auto_route_allowed_domains", "feature removed; declare explicit interfaces"}, | 269 | {"auto_route_allowed_domains", "feature removed; declare explicit interfaces"}, | ||
| 268 | {"previews", "staging is an ordinary app file"}, | 270 | {"previews", "staging is an ordinary app file"}, | ||
| 269 | {"env", "remove the [env] section; declare [env] and [services.<name>.secrets] in app files"}, | 271 | {"env", "remove the [env] section; declare [env] and [services.<name>.secrets] in app files"}, | ||
| 272 | + | {"backups.volumes.interval", "use backups.volumes.schedule and backups.volumes.time (server local time)"}, | |||
| 270 | } | 273 | } | ||
| 271 | 274 | ||||
| 272 | // validateRetiredAppConfig rejects obsolete application configuration | 275 | // validateRetiredAppConfig rejects obsolete application configuration | ||
| @@ -414,13 +417,15 @@ func loadConfig(v *viper.Viper, configPath string) error { | |||||
| 414 | v.SetDefault("volumes.preserve", true) | 417 | v.SetDefault("volumes.preserve", true) | ||
| 415 | v.SetDefault("backups.databases.enabled", false) | 418 | v.SetDefault("backups.databases.enabled", false) | ||
| 416 | v.SetDefault("backups.databases.schedule", string(domain.ScheduleDaily)) | 419 | v.SetDefault("backups.databases.schedule", string(domain.ScheduleDaily)) | ||
| 420 | + | v.SetDefault("backups.databases.time", "") | |||
| 417 | v.SetDefault("backups.databases.storage_dir", "") | 421 | v.SetDefault("backups.databases.storage_dir", "") | ||
| 418 | v.SetDefault("backups.databases.retention.hourly", 0) | 422 | v.SetDefault("backups.databases.retention.hourly", 0) | ||
| 419 | v.SetDefault("backups.databases.retention.daily", 0) | 423 | v.SetDefault("backups.databases.retention.daily", 0) | ||
| 420 | v.SetDefault("backups.databases.retention.weekly", 0) | 424 | v.SetDefault("backups.databases.retention.weekly", 0) | ||
| 421 | v.SetDefault("backups.databases.retention.monthly", 0) | 425 | v.SetDefault("backups.databases.retention.monthly", 0) | ||
| 422 | v.SetDefault("backups.volumes.enabled", false) | 426 | v.SetDefault("backups.volumes.enabled", false) | ||
| 423 | - | v.SetDefault("backups.volumes.interval", "24h") | 427 | + | v.SetDefault("backups.volumes.schedule", string(domain.ScheduleDaily)) |
| 428 | + | v.SetDefault("backups.volumes.time", "") | |||
| 424 | v.SetDefault("backups.volumes.compression", string(domain.VolumeBackupCompressionGzip)) | 429 | v.SetDefault("backups.volumes.compression", string(domain.VolumeBackupCompressionGzip)) | ||
| 425 | v.SetDefault("backups.volumes.timeout", "2h") | 430 | v.SetDefault("backups.volumes.timeout", "2h") | ||
| 426 | v.SetDefault("backups.volumes.max_concurrency", 2) | 431 | v.SetDefault("backups.volumes.max_concurrency", 2) | ||
M internal/app/kernel.go +2 -2
| @@ -86,8 +86,8 @@ func newKernel(configPath string, initLog kernelLoggerInit) (*Kernel, error) { | |||
| 86 | 86 | authEnabled: cfg.Auth.Enabled, | |
| 87 | 87 | configSvc: svc.configSvc, | |
| 88 | 88 | containerSvc: svc.containerSvc, | |
| 89 | - | backupSvc: svc.backupSvc, | |
| 90 | - | volumeBackupSvc: svc.volumeBackupSvc, | |
| 89 | + | backupSvc: backupServicePort(svc.backupSvc), | |
| 90 | + | volumeBackupSvc: volumeBackupServicePort(svc.volumeBackupSvc), | |
| 91 | 91 | registrySvc: svc.registrySvc, | |
| 92 | 92 | healthSvc: svc.healthSvc, | |
| 93 | 93 | logSvc: svc.logSvc, | |
| @@ -86,8 +86,8 @@ func newKernel(configPath string, initLog kernelLoggerInit) (*Kernel, error) { | |||||
| 86 | authEnabled: cfg.Auth.Enabled, | 86 | authEnabled: cfg.Auth.Enabled, | ||
| 87 | configSvc: svc.configSvc, | 87 | configSvc: svc.configSvc, | ||
| 88 | containerSvc: svc.containerSvc, | 88 | containerSvc: svc.containerSvc, | ||
| 89 | - | backupSvc: svc.backupSvc, | 89 | + | backupSvc: backupServicePort(svc.backupSvc), |
| 90 | - | volumeBackupSvc: svc.volumeBackupSvc, | 90 | + | volumeBackupSvc: volumeBackupServicePort(svc.volumeBackupSvc), |
| 91 | registrySvc: svc.registrySvc, | 91 | registrySvc: svc.registrySvc, | ||
| 92 | healthSvc: svc.healthSvc, | 92 | healthSvc: svc.healthSvc, | ||
| 93 | logSvc: svc.logSvc, | 93 | logSvc: svc.logSvc, | ||
M internal/app/run_backup_test.go +56 -5
| @@ -2,6 +2,7 @@ package app | |||
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | 4 | "testing" | |
| 5 | + | "time" | |
| 5 | 6 | ||
| 6 | 7 | "github.com/spf13/viper" | |
| 7 | 8 | "github.com/stretchr/testify/assert" | |
| @@ -144,7 +145,8 @@ func TestValidateVolumeBackupConfig(t *testing.T) { | |||
| 144 | 145 | t.Run("enabled accepts s3 config", func(t *testing.T) { | |
| 145 | 146 | var cfg Config | |
| 146 | 147 | cfg.Backups.Volumes.Enabled = true | |
| 147 | - | cfg.Backups.Volumes.Interval = "6h" | |
| 148 | + | cfg.Backups.Volumes.Schedule = "weekly" | |
| 149 | + | cfg.Backups.Volumes.Time = "03:30" | |
| 148 | 150 | cfg.Backups.Volumes.Timeout = "30m" | |
| 149 | 151 | cfg.Backups.Volumes.Compression = "gzip" | |
| 150 | 152 | cfg.Backups.Volumes.MaxConcurrency = 3 | |
| @@ -158,6 +160,10 @@ func TestValidateVolumeBackupConfig(t *testing.T) { | |||
| 158 | 160 | volumeCfg, err := validateVolumeBackupConfig(cfg) | |
| 159 | 161 | require.NoError(t, err) | |
| 160 | 162 | assert.True(t, volumeCfg.Enabled) | |
| 163 | + | assert.Equal(t, domain.ScheduleWeekly, volumeCfg.Schedule.Preset) | |
| 164 | + | require.NotNil(t, volumeCfg.Schedule.At) | |
| 165 | + | assert.Equal(t, domain.TimeOfDay{Hour: 3, Minute: 30}, *volumeCfg.Schedule.At) | |
| 166 | + | assert.Equal(t, time.Local, volumeCfg.Schedule.Location) | |
| 161 | 167 | assert.Equal(t, domain.VolumeBackupCompressionGzip, volumeCfg.Compression) | |
| 162 | 168 | assert.Equal(t, 7, volumeCfg.Retention.Keep) | |
| 163 | 169 | assert.Equal(t, 3, volumeCfg.MaxConcurrency) | |
| @@ -165,6 +171,35 @@ func TestValidateVolumeBackupConfig(t *testing.T) { | |||
| 165 | 171 | assert.Equal(t, "prod/gordon", volumeCfg.S3Prefix) | |
| 166 | 172 | }) | |
| 167 | 173 | ||
| 174 | + | t.Run("schedule defaults to daily at the preset time", func(t *testing.T) { | |
| 175 | + | var cfg Config | |
| 176 | + | ||
| 177 | + | volumeCfg, err := validateVolumeBackupConfig(cfg) | |
| 178 | + | require.NoError(t, err) | |
| 179 | + | assert.Equal(t, domain.ScheduleDaily, volumeCfg.Schedule.Preset) | |
| 180 | + | assert.Nil(t, volumeCfg.Schedule.At) | |
| 181 | + | }) | |
| 182 | + | ||
| 183 | + | t.Run("schedule rejects invalid time", func(t *testing.T) { | |
| 184 | + | for _, raw := range []string{"3h30", "24:00", "03:60", "3:5", "night"} { | |
| 185 | + | var cfg Config | |
| 186 | + | cfg.Backups.Volumes.Time = raw | |
| 187 | + | ||
| 188 | + | _, err := validateVolumeBackupConfig(cfg) | |
| 189 | + | require.Error(t, err, raw) | |
| 190 | + | assert.Contains(t, err.Error(), "backups.volumes.time") | |
| 191 | + | } | |
| 192 | + | }) | |
| 193 | + | ||
| 194 | + | t.Run("schedule rejects unknown preset", func(t *testing.T) { | |
| 195 | + | var cfg Config | |
| 196 | + | cfg.Backups.Volumes.Schedule = "nightly" | |
| 197 | + | ||
| 198 | + | _, err := validateVolumeBackupConfig(cfg) | |
| 199 | + | require.Error(t, err) | |
| 200 | + | assert.Contains(t, err.Error(), "backups.volumes.schedule") | |
| 201 | + | }) | |
| 202 | + | ||
| 168 | 203 | t.Run("storage class is normalized and passed through", func(t *testing.T) { | |
| 169 | 204 | var cfg Config | |
| 170 | 205 | cfg.Backups.Volumes.S3.StorageClass = " onezone_ia " | |
| @@ -220,7 +255,9 @@ func TestLoadConfigSetsBackupDefaults(t *testing.T) { | |||
| 220 | 255 | require.NoError(t, loadConfig(v, "")) | |
| 221 | 256 | ||
| 222 | 257 | assert.Equal(t, string(domain.ScheduleDaily), v.GetString("backups.databases.schedule")) | |
| 223 | - | assert.Equal(t, "24h", v.GetString("backups.volumes.interval")) | |
| 258 | + | assert.Equal(t, string(domain.ScheduleDaily), v.GetString("backups.volumes.schedule")) | |
| 259 | + | assert.Empty(t, v.GetString("backups.volumes.time")) | |
| 260 | + | assert.Empty(t, v.GetString("backups.databases.time")) | |
| 224 | 261 | assert.Equal(t, string(domain.VolumeBackupCompressionGzip), v.GetString("backups.volumes.compression")) | |
| 225 | 262 | assert.Equal(t, 14, v.GetInt("backups.volumes.retention.keep")) | |
| 226 | 263 | } | |
| @@ -240,16 +277,30 @@ func TestResolveBackupSchedule(t *testing.T) { | |||
| 240 | 277 | ||
| 241 | 278 | for _, tt := range tests { | |
| 242 | 279 | t.Run(tt.input, func(t *testing.T) { | |
| 243 | - | got, err := resolveBackupSchedule(tt.input) | |
| 280 | + | got, err := resolveCronSchedule(tt.input, "", "backups.databases") | |
| 244 | 281 | require.NoError(t, err) | |
| 245 | - | assert.Equal(t, tt.want, got) | |
| 282 | + | assert.Equal(t, tt.want, got.Preset) | |
| 246 | 283 | }) | |
| 247 | 284 | } | |
| 248 | 285 | }) | |
| 249 | 286 | ||
| 250 | 287 | t.Run("invalid value", func(t *testing.T) { | |
| 251 | - | _, err := resolveBackupSchedule("every-minute") | |
| 288 | + | _, err := resolveCronSchedule("every-minute", "", "backups.databases") | |
| 252 | 289 | require.Error(t, err) | |
| 253 | 290 | assert.Contains(t, err.Error(), "backups.databases.schedule") | |
| 254 | 291 | }) | |
| 292 | + | ||
| 293 | + | t.Run("time is read in server local time", func(t *testing.T) { | |
| 294 | + | got, err := resolveCronSchedule("daily", "04:15", "backups.databases") | |
| 295 | + | require.NoError(t, err) | |
| 296 | + | require.NotNil(t, got.At) | |
| 297 | + | assert.Equal(t, domain.TimeOfDay{Hour: 4, Minute: 15}, *got.At) | |
| 298 | + | assert.Equal(t, time.Local, got.Location) | |
| 299 | + | }) | |
| 300 | + | ||
| 301 | + | t.Run("invalid time names the database key", func(t *testing.T) { | |
| 302 | + | _, err := resolveCronSchedule("daily", "25:00", "backups.databases") | |
| 303 | + | require.Error(t, err) | |
| 304 | + | assert.Contains(t, err.Error(), "backups.databases.time") | |
| 305 | + | }) | |
| 255 | 306 | } | |
| @@ -2,6 +2,7 @@ package app | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "testing" | 4 | "testing" | ||
| 5 | + | "time" | |||
| 5 | 6 | ||||
| 6 | "github.com/spf13/viper" | 7 | "github.com/spf13/viper" | ||
| 7 | "github.com/stretchr/testify/assert" | 8 | "github.com/stretchr/testify/assert" | ||
| @@ -144,7 +145,8 @@ func TestValidateVolumeBackupConfig(t *testing.T) { | |||||
| 144 | t.Run("enabled accepts s3 config", func(t *testing.T) { | 145 | t.Run("enabled accepts s3 config", func(t *testing.T) { | ||
| 145 | var cfg Config | 146 | var cfg Config | ||
| 146 | cfg.Backups.Volumes.Enabled = true | 147 | cfg.Backups.Volumes.Enabled = true | ||
| 147 | - | cfg.Backups.Volumes.Interval = "6h" | 148 | + | cfg.Backups.Volumes.Schedule = "weekly" |
| 149 | + | cfg.Backups.Volumes.Time = "03:30" | |||
| 148 | cfg.Backups.Volumes.Timeout = "30m" | 150 | cfg.Backups.Volumes.Timeout = "30m" | ||
| 149 | cfg.Backups.Volumes.Compression = "gzip" | 151 | cfg.Backups.Volumes.Compression = "gzip" | ||
| 150 | cfg.Backups.Volumes.MaxConcurrency = 3 | 152 | cfg.Backups.Volumes.MaxConcurrency = 3 | ||
| @@ -158,6 +160,10 @@ func TestValidateVolumeBackupConfig(t *testing.T) { | |||||
| 158 | volumeCfg, err := validateVolumeBackupConfig(cfg) | 160 | volumeCfg, err := validateVolumeBackupConfig(cfg) | ||
| 159 | require.NoError(t, err) | 161 | require.NoError(t, err) | ||
| 160 | assert.True(t, volumeCfg.Enabled) | 162 | assert.True(t, volumeCfg.Enabled) | ||
| 163 | + | assert.Equal(t, domain.ScheduleWeekly, volumeCfg.Schedule.Preset) | |||
| 164 | + | require.NotNil(t, volumeCfg.Schedule.At) | |||
| 165 | + | assert.Equal(t, domain.TimeOfDay{Hour: 3, Minute: 30}, *volumeCfg.Schedule.At) | |||
| 166 | + | assert.Equal(t, time.Local, volumeCfg.Schedule.Location) | |||
| 161 | assert.Equal(t, domain.VolumeBackupCompressionGzip, volumeCfg.Compression) | 167 | assert.Equal(t, domain.VolumeBackupCompressionGzip, volumeCfg.Compression) | ||
| 162 | assert.Equal(t, 7, volumeCfg.Retention.Keep) | 168 | assert.Equal(t, 7, volumeCfg.Retention.Keep) | ||
| 163 | assert.Equal(t, 3, volumeCfg.MaxConcurrency) | 169 | assert.Equal(t, 3, volumeCfg.MaxConcurrency) | ||
| @@ -165,6 +171,35 @@ func TestValidateVolumeBackupConfig(t *testing.T) { | |||||
| 165 | assert.Equal(t, "prod/gordon", volumeCfg.S3Prefix) | 171 | assert.Equal(t, "prod/gordon", volumeCfg.S3Prefix) | ||
| 166 | }) | 172 | }) | ||
| 167 | 173 | ||||
| 174 | + | t.Run("schedule defaults to daily at the preset time", func(t *testing.T) { | |||
| 175 | + | var cfg Config | |||
| 176 | + | ||||
| 177 | + | volumeCfg, err := validateVolumeBackupConfig(cfg) | |||
| 178 | + | require.NoError(t, err) | |||
| 179 | + | assert.Equal(t, domain.ScheduleDaily, volumeCfg.Schedule.Preset) | |||
| 180 | + | assert.Nil(t, volumeCfg.Schedule.At) | |||
| 181 | + | }) | |||
| 182 | + | ||||
| 183 | + | t.Run("schedule rejects invalid time", func(t *testing.T) { | |||
| 184 | + | for _, raw := range []string{"3h30", "24:00", "03:60", "3:5", "night"} { | |||
| 185 | + | var cfg Config | |||
| 186 | + | cfg.Backups.Volumes.Time = raw | |||
| 187 | + | ||||
| 188 | + | _, err := validateVolumeBackupConfig(cfg) | |||
| 189 | + | require.Error(t, err, raw) | |||
| 190 | + | assert.Contains(t, err.Error(), "backups.volumes.time") | |||
| 191 | + | } | |||
| 192 | + | }) | |||
| 193 | + | ||||
| 194 | + | t.Run("schedule rejects unknown preset", func(t *testing.T) { | |||
| 195 | + | var cfg Config | |||
| 196 | + | cfg.Backups.Volumes.Schedule = "nightly" | |||
| 197 | + | ||||
| 198 | + | _, err := validateVolumeBackupConfig(cfg) | |||
| 199 | + | require.Error(t, err) | |||
| 200 | + | assert.Contains(t, err.Error(), "backups.volumes.schedule") | |||
| 201 | + | }) | |||
| 202 | + | ||||
| 168 | t.Run("storage class is normalized and passed through", func(t *testing.T) { | 203 | t.Run("storage class is normalized and passed through", func(t *testing.T) { | ||
| 169 | var cfg Config | 204 | var cfg Config | ||
| 170 | cfg.Backups.Volumes.S3.StorageClass = " onezone_ia " | 205 | cfg.Backups.Volumes.S3.StorageClass = " onezone_ia " | ||
| @@ -220,7 +255,9 @@ func TestLoadConfigSetsBackupDefaults(t *testing.T) { | |||||
| 220 | require.NoError(t, loadConfig(v, "")) | 255 | require.NoError(t, loadConfig(v, "")) | ||
| 221 | 256 | ||||
| 222 | assert.Equal(t, string(domain.ScheduleDaily), v.GetString("backups.databases.schedule")) | 257 | assert.Equal(t, string(domain.ScheduleDaily), v.GetString("backups.databases.schedule")) | ||
| 223 | - | assert.Equal(t, "24h", v.GetString("backups.volumes.interval")) | 258 | + | assert.Equal(t, string(domain.ScheduleDaily), v.GetString("backups.volumes.schedule")) |
| 259 | + | assert.Empty(t, v.GetString("backups.volumes.time")) | |||
| 260 | + | assert.Empty(t, v.GetString("backups.databases.time")) | |||
| 224 | assert.Equal(t, string(domain.VolumeBackupCompressionGzip), v.GetString("backups.volumes.compression")) | 261 | assert.Equal(t, string(domain.VolumeBackupCompressionGzip), v.GetString("backups.volumes.compression")) | ||
| 225 | assert.Equal(t, 14, v.GetInt("backups.volumes.retention.keep")) | 262 | assert.Equal(t, 14, v.GetInt("backups.volumes.retention.keep")) | ||
| 226 | } | 263 | } | ||
| @@ -240,16 +277,30 @@ func TestResolveBackupSchedule(t *testing.T) { | |||||
| 240 | 277 | ||||
| 241 | for _, tt := range tests { | 278 | for _, tt := range tests { | ||
| 242 | t.Run(tt.input, func(t *testing.T) { | 279 | t.Run(tt.input, func(t *testing.T) { | ||
| 243 | - | got, err := resolveBackupSchedule(tt.input) | 280 | + | got, err := resolveCronSchedule(tt.input, "", "backups.databases") |
| 244 | require.NoError(t, err) | 281 | require.NoError(t, err) | ||
| 245 | - | assert.Equal(t, tt.want, got) | 282 | + | assert.Equal(t, tt.want, got.Preset) |
| 246 | }) | 283 | }) | ||
| 247 | } | 284 | } | ||
| 248 | }) | 285 | }) | ||
| 249 | 286 | ||||
| 250 | t.Run("invalid value", func(t *testing.T) { | 287 | t.Run("invalid value", func(t *testing.T) { | ||
| 251 | - | _, err := resolveBackupSchedule("every-minute") | 288 | + | _, err := resolveCronSchedule("every-minute", "", "backups.databases") |
| 252 | require.Error(t, err) | 289 | require.Error(t, err) | ||
| 253 | assert.Contains(t, err.Error(), "backups.databases.schedule") | 290 | assert.Contains(t, err.Error(), "backups.databases.schedule") | ||
| 254 | }) | 291 | }) | ||
| 292 | + | ||||
| 293 | + | t.Run("time is read in server local time", func(t *testing.T) { | |||
| 294 | + | got, err := resolveCronSchedule("daily", "04:15", "backups.databases") | |||
| 295 | + | require.NoError(t, err) | |||
| 296 | + | require.NotNil(t, got.At) | |||
| 297 | + | assert.Equal(t, domain.TimeOfDay{Hour: 4, Minute: 15}, *got.At) | |||
| 298 | + | assert.Equal(t, time.Local, got.Location) | |||
| 299 | + | }) | |||
| 300 | + | ||||
| 301 | + | t.Run("invalid time names the database key", func(t *testing.T) { | |||
| 302 | + | _, err := resolveCronSchedule("daily", "25:00", "backups.databases") | |||
| 303 | + | require.Error(t, err) | |||
| 304 | + | assert.Contains(t, err.Error(), "backups.databases.time") | |||
| 305 | + | }) | |||
| 255 | } | 306 | } | ||
M internal/app/run_entrypoint_migration_test.go +10 -0
| @@ -37,6 +37,16 @@ func TestInitConfigRejectsRetiredAppKeys(t *testing.T) { | |||
| 37 | 37 | require.ErrorContains(t, err, `"routes"`) | |
| 38 | 38 | } | |
| 39 | 39 | ||
| 40 | + | func TestInitConfigRejectsRetiredVolumeBackupInterval(t *testing.T) { | |
| 41 | + | configPath := filepath.Join(t.TempDir(), "gordon.toml") | |
| 42 | + | require.NoError(t, os.WriteFile(configPath, []byte("[backups.volumes]\ninterval = \"24h\"\n"), 0o600)) | |
| 43 | + | ||
| 44 | + | _, _, err := initConfig(configPath) | |
| 45 | + | ||
| 46 | + | require.ErrorContains(t, err, "config-retired") | |
| 47 | + | require.ErrorContains(t, err, "backups.volumes.schedule") | |
| 48 | + | } | |
| 49 | + | ||
| 40 | 50 | func TestInitConfigRejectsRetiredEnvSection(t *testing.T) { | |
| 41 | 51 | configPath := filepath.Join(t.TempDir(), "gordon.toml") | |
| 42 | 52 | require.NoError(t, os.WriteFile(configPath, []byte("[env]\ndir = \"/tmp/env\"\n"), 0o600)) | |
| @@ -37,6 +37,16 @@ func TestInitConfigRejectsRetiredAppKeys(t *testing.T) { | |||||
| 37 | require.ErrorContains(t, err, `"routes"`) | 37 | require.ErrorContains(t, err, `"routes"`) | ||
| 38 | } | 38 | } | ||
| 39 | 39 | ||||
| 40 | + | func TestInitConfigRejectsRetiredVolumeBackupInterval(t *testing.T) { | |||
| 41 | + | configPath := filepath.Join(t.TempDir(), "gordon.toml") | |||
| 42 | + | require.NoError(t, os.WriteFile(configPath, []byte("[backups.volumes]\ninterval = \"24h\"\n"), 0o600)) | |||
| 43 | + | ||||
| 44 | + | _, _, err := initConfig(configPath) | |||
| 45 | + | ||||
| 46 | + | require.ErrorContains(t, err, "config-retired") | |||
| 47 | + | require.ErrorContains(t, err, "backups.volumes.schedule") | |||
| 48 | + | } | |||
| 49 | + | ||||
| 40 | func TestInitConfigRejectsRetiredEnvSection(t *testing.T) { | 50 | func TestInitConfigRejectsRetiredEnvSection(t *testing.T) { | ||
| 41 | configPath := filepath.Join(t.TempDir(), "gordon.toml") | 51 | configPath := filepath.Join(t.TempDir(), "gordon.toml") | ||
| 42 | require.NoError(t, os.WriteFile(configPath, []byte("[env]\ndir = \"/tmp/env\"\n"), 0o600)) | 52 | require.NoError(t, os.WriteFile(configPath, []byte("[env]\ndir = \"/tmp/env\"\n"), 0o600)) | ||
M internal/app/schedulers.go +55 -10
| @@ -4,6 +4,7 @@ import ( | |||
| 4 | 4 | "context" | |
| 5 | 5 | "fmt" | |
| 6 | 6 | "strings" | |
| 7 | + | "time" | |
| 7 | 8 | ||
| 8 | 9 | "github.com/bnema/zerowrap" | |
| 9 | 10 | "github.com/spf13/viper" | |
| @@ -61,22 +62,23 @@ func startBackupScheduler(ctx context.Context, cfg Config, svc *services, log ze | |||
| 61 | 62 | return nil, nil | |
| 62 | 63 | } | |
| 63 | 64 | ||
| 64 | - | preset, err := resolveBackupSchedule(dbCfg.Schedule) | |
| 65 | + | schedule, err := resolveCronSchedule(dbCfg.Schedule, dbCfg.Time, "backups.databases") | |
| 65 | 66 | if err != nil { | |
| 66 | 67 | return nil, err | |
| 67 | 68 | } | |
| 69 | + | preset := schedule.Preset | |
| 68 | 70 | ||
| 69 | 71 | scheduler := cronSvc.NewScheduler(log) | |
| 70 | 72 | err = scheduler.Add( | |
| 71 | 73 | "backup-scheduler", | |
| 72 | 74 | "Backups", | |
| 73 | - | domain.CronSchedule{Preset: preset}, | |
| 75 | + | schedule, | |
| 74 | 76 | func(jobCtx context.Context) error { | |
| 75 | 77 | if err := svc.backupSvc.RunForSchedule(jobCtx, preset); err != nil { | |
| 76 | 78 | return err | |
| 77 | 79 | } | |
| 78 | 80 | log.Info(). | |
| 79 | - | Str("schedule", string(preset)). | |
| 81 | + | Str("schedule", describeSchedule(schedule)). | |
| 80 | 82 | Msg("scheduled backup run complete") | |
| 81 | 83 | return nil | |
| 82 | 84 | }, | |
| @@ -87,14 +89,21 @@ func startBackupScheduler(ctx context.Context, cfg Config, svc *services, log ze | |||
| 87 | 89 | ||
| 88 | 90 | scheduler.Start(ctx) | |
| 89 | 91 | log.Info(). | |
| 90 | - | Str("schedule", string(preset)). | |
| 92 | + | Str("schedule", describeSchedule(schedule)). | |
| 93 | + | Time("next_run", nextRunOf(scheduler, "backup-scheduler")). | |
| 91 | 94 | Msg("backup scheduler enabled") | |
| 92 | 95 | ||
| 93 | 96 | return scheduler, nil | |
| 94 | 97 | } | |
| 95 | 98 | ||
| 96 | - | func resolveBackupSchedule(raw string) (domain.BackupSchedule, error) { | |
| 97 | - | return resolveSchedulePreset(raw, "backups.databases.schedule", domain.ScheduleDaily) | |
| 99 | + | // nextRunOf returns the next planned run of one registered job. | |
| 100 | + | func nextRunOf(scheduler *cronSvc.Scheduler, id string) time.Time { | |
| 101 | + | for _, entry := range scheduler.List() { | |
| 102 | + | if entry.ID == id { | |
| 103 | + | return entry.NextRun | |
| 104 | + | } | |
| 105 | + | } | |
| 106 | + | return time.Time{} | |
| 98 | 107 | } | |
| 99 | 108 | ||
| 100 | 109 | func startVolumeBackupScheduler(ctx context.Context, cfg Config, svc *services, log zerowrap.Logger) (*cronSvc.Scheduler, error) { | |
| @@ -111,13 +120,13 @@ func startVolumeBackupScheduler(ctx context.Context, cfg Config, svc *services, | |||
| 111 | 120 | err = scheduler.Add( | |
| 112 | 121 | "volume-backup-scheduler", | |
| 113 | 122 | "Volume Backups", | |
| 114 | - | domain.CronSchedule{Interval: volumeCfg.Interval}, | |
| 123 | + | volumeCfg.Schedule, | |
| 115 | 124 | func(jobCtx context.Context) error { | |
| 116 | - | if err := svc.volumeBackupSvc.RunVolumeBackupsForSchedule(jobCtx, ""); err != nil { | |
| 125 | + | if err := svc.volumeBackupSvc.RunVolumeBackupsForSchedule(jobCtx, volumeCfg.Schedule.Preset); err != nil { | |
| 117 | 126 | return err | |
| 118 | 127 | } | |
| 119 | 128 | log.Info(). | |
| 120 | - | Dur("interval", volumeCfg.Interval). | |
| 129 | + | Str("schedule", describeSchedule(volumeCfg.Schedule)). | |
| 121 | 130 | Msg("scheduled volume backup run complete") | |
| 122 | 131 | return nil | |
| 123 | 132 | }, | |
| @@ -128,7 +137,8 @@ func startVolumeBackupScheduler(ctx context.Context, cfg Config, svc *services, | |||
| 128 | 137 | ||
| 129 | 138 | scheduler.Start(ctx) | |
| 130 | 139 | log.Info(). | |
| 131 | - | Dur("interval", volumeCfg.Interval). | |
| 140 | + | Str("schedule", describeSchedule(volumeCfg.Schedule)). | |
| 141 | + | Time("next_run", nextRunOf(scheduler, "volume-backup-scheduler")). | |
| 132 | 142 | Msg("volume backup scheduler enabled") | |
| 133 | 143 | ||
| 134 | 144 | return scheduler, nil | |
| @@ -209,6 +219,41 @@ func resolveImagePruneSchedule(raw string) (domain.BackupSchedule, error) { | |||
| 209 | 219 | return resolveSchedulePreset(raw, "images.prune.schedule", domain.ScheduleDaily) | |
| 210 | 220 | } | |
| 211 | 221 | ||
| 222 | + | // resolveCronSchedule builds a backup schedule from a preset and an | |
| 223 | + | // optional HH:MM time read in the server's local timezone. prefix names | |
| 224 | + | // the config section (e.g. "backups.volumes") for error messages. | |
| 225 | + | func resolveCronSchedule(rawSchedule, rawTime, prefix string) (domain.CronSchedule, error) { | |
| 226 | + | preset, err := resolveSchedulePreset(rawSchedule, prefix+".schedule", domain.ScheduleDaily) | |
| 227 | + | if err != nil { | |
| 228 | + | return domain.CronSchedule{}, err | |
| 229 | + | } | |
| 230 | + | schedule := domain.CronSchedule{Preset: preset, Location: time.Local} | |
| 231 | + | if strings.TrimSpace(rawTime) == "" { | |
| 232 | + | return schedule, nil | |
| 233 | + | } | |
| 234 | + | at, err := domain.ParseTimeOfDay(rawTime) | |
| 235 | + | if err != nil { | |
| 236 | + | return domain.CronSchedule{}, fmt.Errorf("%s.time: %w", prefix, err) | |
| 237 | + | } | |
| 238 | + | schedule.At = &at | |
| 239 | + | return schedule, nil | |
| 240 | + | } | |
| 241 | + | ||
| 242 | + | // describeSchedule renders a schedule for logs, e.g. "daily at 03:30 Europe/Paris". | |
| 243 | + | func describeSchedule(s domain.CronSchedule) string { | |
| 244 | + | at := "default time" | |
| 245 | + | if s.At != nil { | |
| 246 | + | at = s.At.String() | |
| 247 | + | } | |
| 248 | + | loc := "UTC" | |
| 249 | + | if s.Location != nil { | |
| 250 | + | // time.Local reports "Local"; the current zone abbreviation is | |
| 251 | + | // what an operator can read (e.g. CEST). | |
| 252 | + | loc, _ = time.Now().In(s.Location).Zone() | |
| 253 | + | } | |
| 254 | + | return fmt.Sprintf("%s at %s %s", s.Preset, at, loc) | |
| 255 | + | } | |
| 256 | + | ||
| 212 | 257 | func resolveSchedulePreset(raw, name string, defaultVal domain.BackupSchedule) (domain.BackupSchedule, error) { | |
| 213 | 258 | schedule := domain.BackupSchedule(strings.ToLower(strings.TrimSpace(raw))) | |
| 214 | 259 | if schedule == "" { | |
| @@ -4,6 +4,7 @@ import ( | |||||
| 4 | "context" | 4 | "context" | ||
| 5 | "fmt" | 5 | "fmt" | ||
| 6 | "strings" | 6 | "strings" | ||
| 7 | + | "time" | |||
| 7 | 8 | ||||
| 8 | "github.com/bnema/zerowrap" | 9 | "github.com/bnema/zerowrap" | ||
| 9 | "github.com/spf13/viper" | 10 | "github.com/spf13/viper" | ||
| @@ -61,22 +62,23 @@ func startBackupScheduler(ctx context.Context, cfg Config, svc *services, log ze | |||||
| 61 | return nil, nil | 62 | return nil, nil | ||
| 62 | } | 63 | } | ||
| 63 | 64 | ||||
| 64 | - | preset, err := resolveBackupSchedule(dbCfg.Schedule) | 65 | + | schedule, err := resolveCronSchedule(dbCfg.Schedule, dbCfg.Time, "backups.databases") |
| 65 | if err != nil { | 66 | if err != nil { | ||
| 66 | return nil, err | 67 | return nil, err | ||
| 67 | } | 68 | } | ||
| 69 | + | preset := schedule.Preset | |||
| 68 | 70 | ||||
| 69 | scheduler := cronSvc.NewScheduler(log) | 71 | scheduler := cronSvc.NewScheduler(log) | ||
| 70 | err = scheduler.Add( | 72 | err = scheduler.Add( | ||
| 71 | "backup-scheduler", | 73 | "backup-scheduler", | ||
| 72 | "Backups", | 74 | "Backups", | ||
| 73 | - | domain.CronSchedule{Preset: preset}, | 75 | + | schedule, |
| 74 | func(jobCtx context.Context) error { | 76 | func(jobCtx context.Context) error { | ||
| 75 | if err := svc.backupSvc.RunForSchedule(jobCtx, preset); err != nil { | 77 | if err := svc.backupSvc.RunForSchedule(jobCtx, preset); err != nil { | ||
| 76 | return err | 78 | return err | ||
| 77 | } | 79 | } | ||
| 78 | log.Info(). | 80 | log.Info(). | ||
| 79 | - | Str("schedule", string(preset)). | 81 | + | Str("schedule", describeSchedule(schedule)). |
| 80 | Msg("scheduled backup run complete") | 82 | Msg("scheduled backup run complete") | ||
| 81 | return nil | 83 | return nil | ||
| 82 | }, | 84 | }, | ||
| @@ -87,14 +89,21 @@ func startBackupScheduler(ctx context.Context, cfg Config, svc *services, log ze | |||||
| 87 | 89 | ||||
| 88 | scheduler.Start(ctx) | 90 | scheduler.Start(ctx) | ||
| 89 | log.Info(). | 91 | log.Info(). | ||
| 90 | - | Str("schedule", string(preset)). | 92 | + | Str("schedule", describeSchedule(schedule)). |
| 93 | + | Time("next_run", nextRunOf(scheduler, "backup-scheduler")). | |||
| 91 | Msg("backup scheduler enabled") | 94 | Msg("backup scheduler enabled") | ||
| 92 | 95 | ||||
| 93 | return scheduler, nil | 96 | return scheduler, nil | ||
| 94 | } | 97 | } | ||
| 95 | 98 | ||||
| 96 | - | func resolveBackupSchedule(raw string) (domain.BackupSchedule, error) { | 99 | + | // nextRunOf returns the next planned run of one registered job. |
| 97 | - | return resolveSchedulePreset(raw, "backups.databases.schedule", domain.ScheduleDaily) | 100 | + | func nextRunOf(scheduler *cronSvc.Scheduler, id string) time.Time { |
| 101 | + | for _, entry := range scheduler.List() { | |||
| 102 | + | if entry.ID == id { | |||
| 103 | + | return entry.NextRun | |||
| 104 | + | } | |||
| 105 | + | } | |||
| 106 | + | return time.Time{} | |||
| 98 | } | 107 | } | ||
| 99 | 108 | ||||
| 100 | func startVolumeBackupScheduler(ctx context.Context, cfg Config, svc *services, log zerowrap.Logger) (*cronSvc.Scheduler, error) { | 109 | func startVolumeBackupScheduler(ctx context.Context, cfg Config, svc *services, log zerowrap.Logger) (*cronSvc.Scheduler, error) { | ||
| @@ -111,13 +120,13 @@ func startVolumeBackupScheduler(ctx context.Context, cfg Config, svc *services, | |||||
| 111 | err = scheduler.Add( | 120 | err = scheduler.Add( | ||
| 112 | "volume-backup-scheduler", | 121 | "volume-backup-scheduler", | ||
| 113 | "Volume Backups", | 122 | "Volume Backups", | ||
| 114 | - | domain.CronSchedule{Interval: volumeCfg.Interval}, | 123 | + | volumeCfg.Schedule, |
| 115 | func(jobCtx context.Context) error { | 124 | func(jobCtx context.Context) error { | ||
| 116 | - | if err := svc.volumeBackupSvc.RunVolumeBackupsForSchedule(jobCtx, ""); err != nil { | 125 | + | if err := svc.volumeBackupSvc.RunVolumeBackupsForSchedule(jobCtx, volumeCfg.Schedule.Preset); err != nil { |
| 117 | return err | 126 | return err | ||
| 118 | } | 127 | } | ||
| 119 | log.Info(). | 128 | log.Info(). | ||
| 120 | - | Dur("interval", volumeCfg.Interval). | 129 | + | Str("schedule", describeSchedule(volumeCfg.Schedule)). |
| 121 | Msg("scheduled volume backup run complete") | 130 | Msg("scheduled volume backup run complete") | ||
| 122 | return nil | 131 | return nil | ||
| 123 | }, | 132 | }, | ||
| @@ -128,7 +137,8 @@ func startVolumeBackupScheduler(ctx context.Context, cfg Config, svc *services, | |||||
| 128 | 137 | ||||
| 129 | scheduler.Start(ctx) | 138 | scheduler.Start(ctx) | ||
| 130 | log.Info(). | 139 | log.Info(). | ||
| 131 | - | Dur("interval", volumeCfg.Interval). | 140 | + | Str("schedule", describeSchedule(volumeCfg.Schedule)). |
| 141 | + | Time("next_run", nextRunOf(scheduler, "volume-backup-scheduler")). | |||
| 132 | Msg("volume backup scheduler enabled") | 142 | Msg("volume backup scheduler enabled") | ||
| 133 | 143 | ||||
| 134 | return scheduler, nil | 144 | return scheduler, nil | ||
| @@ -209,6 +219,41 @@ func resolveImagePruneSchedule(raw string) (domain.BackupSchedule, error) { | |||||
| 209 | return resolveSchedulePreset(raw, "images.prune.schedule", domain.ScheduleDaily) | 219 | return resolveSchedulePreset(raw, "images.prune.schedule", domain.ScheduleDaily) | ||
| 210 | } | 220 | } | ||
| 211 | 221 | ||||
| 222 | + | // resolveCronSchedule builds a backup schedule from a preset and an | |||
| 223 | + | // optional HH:MM time read in the server's local timezone. prefix names | |||
| 224 | + | // the config section (e.g. "backups.volumes") for error messages. | |||
| 225 | + | func resolveCronSchedule(rawSchedule, rawTime, prefix string) (domain.CronSchedule, error) { | |||
| 226 | + | preset, err := resolveSchedulePreset(rawSchedule, prefix+".schedule", domain.ScheduleDaily) | |||
| 227 | + | if err != nil { | |||
| 228 | + | return domain.CronSchedule{}, err | |||
| 229 | + | } | |||
| 230 | + | schedule := domain.CronSchedule{Preset: preset, Location: time.Local} | |||
| 231 | + | if strings.TrimSpace(rawTime) == "" { | |||
| 232 | + | return schedule, nil | |||
| 233 | + | } | |||
| 234 | + | at, err := domain.ParseTimeOfDay(rawTime) | |||
| 235 | + | if err != nil { | |||
| 236 | + | return domain.CronSchedule{}, fmt.Errorf("%s.time: %w", prefix, err) | |||
| 237 | + | } | |||
| 238 | + | schedule.At = &at | |||
| 239 | + | return schedule, nil | |||
| 240 | + | } | |||
| 241 | + | ||||
| 242 | + | // describeSchedule renders a schedule for logs, e.g. "daily at 03:30 Europe/Paris". | |||
| 243 | + | func describeSchedule(s domain.CronSchedule) string { | |||
| 244 | + | at := "default time" | |||
| 245 | + | if s.At != nil { | |||
| 246 | + | at = s.At.String() | |||
| 247 | + | } | |||
| 248 | + | loc := "UTC" | |||
| 249 | + | if s.Location != nil { | |||
| 250 | + | // time.Local reports "Local"; the current zone abbreviation is | |||
| 251 | + | // what an operator can read (e.g. CEST). | |||
| 252 | + | loc, _ = time.Now().In(s.Location).Zone() | |||
| 253 | + | } | |||
| 254 | + | return fmt.Sprintf("%s at %s %s", s.Preset, at, loc) | |||
| 255 | + | } | |||
| 256 | + | ||||
| 212 | func resolveSchedulePreset(raw, name string, defaultVal domain.BackupSchedule) (domain.BackupSchedule, error) { | 257 | func resolveSchedulePreset(raw, name string, defaultVal domain.BackupSchedule) (domain.BackupSchedule, error) { | ||
| 213 | schedule := domain.BackupSchedule(strings.ToLower(strings.TrimSpace(raw))) | 258 | schedule := domain.BackupSchedule(strings.ToLower(strings.TrimSpace(raw))) | ||
| 214 | if schedule == "" { | 259 | if schedule == "" { | ||
M internal/app/services.go +20 -2
| @@ -431,8 +431,8 @@ func (si *serviceInit) initHandlers() { | |||
| 431 | 431 | RegistrySvc: si.svc.registrySvc, | |
| 432 | 432 | ReloadTrigger: si.svc.reloadCoordinator, | |
| 433 | 433 | Log: si.log, | |
| 434 | - | BackupSvc: si.svc.backupSvc, | |
| 435 | - | VolumeBackupSvc: si.svc.volumeBackupSvc, | |
| 434 | + | BackupSvc: backupServicePort(si.svc.backupSvc), | |
| 435 | + | VolumeBackupSvc: volumeBackupServicePort(si.svc.volumeBackupSvc), | |
| 436 | 436 | ImageSvc: si.svc.imageSvc, | |
| 437 | 437 | VolumeSvc: si.svc.volumeSvc, | |
| 438 | 438 | PublicTLSSvc: si.svc.publicTLSSvc, | |
| @@ -442,6 +442,24 @@ func (si *serviceInit) initHandlers() { | |||
| 442 | 442 | }) | |
| 443 | 443 | } | |
| 444 | 444 | ||
| 445 | + | // backupServicePort returns the database backup service as a port, or an | |
| 446 | + | // untyped nil when backups are disabled. Storing a nil *backup.Service in | |
| 447 | + | // the interface would make it non-nil and handlers would call into it. | |
| 448 | + | func backupServicePort(svc *backup.Service) in.BackupService { | |
| 449 | + | if svc == nil { | |
| 450 | + | return nil | |
| 451 | + | } | |
| 452 | + | return svc | |
| 453 | + | } | |
| 454 | + | ||
| 455 | + | // volumeBackupServicePort is backupServicePort for volume backups. | |
| 456 | + | func volumeBackupServicePort(svc *backup.VolumeService) in.VolumeBackupService { | |
| 457 | + | if svc == nil { | |
| 458 | + | return nil | |
| 459 | + | } | |
| 460 | + | return svc | |
| 461 | + | } | |
| 462 | + | ||
| 445 | 463 | // caInfoProvider returns the internal CA as a CAInfoService, or an untyped nil | |
| 446 | 464 | // when internal TLS is disabled so the admin handler sees a nil interface. | |
| 447 | 465 | func (si *serviceInit) caInfoProvider() in.CAInfoService { | |
| @@ -431,8 +431,8 @@ func (si *serviceInit) initHandlers() { | |||||
| 431 | RegistrySvc: si.svc.registrySvc, | 431 | RegistrySvc: si.svc.registrySvc, | ||
| 432 | ReloadTrigger: si.svc.reloadCoordinator, | 432 | ReloadTrigger: si.svc.reloadCoordinator, | ||
| 433 | Log: si.log, | 433 | Log: si.log, | ||
| 434 | - | BackupSvc: si.svc.backupSvc, | 434 | + | BackupSvc: backupServicePort(si.svc.backupSvc), |
| 435 | - | VolumeBackupSvc: si.svc.volumeBackupSvc, | 435 | + | VolumeBackupSvc: volumeBackupServicePort(si.svc.volumeBackupSvc), |
| 436 | ImageSvc: si.svc.imageSvc, | 436 | ImageSvc: si.svc.imageSvc, | ||
| 437 | VolumeSvc: si.svc.volumeSvc, | 437 | VolumeSvc: si.svc.volumeSvc, | ||
| 438 | PublicTLSSvc: si.svc.publicTLSSvc, | 438 | PublicTLSSvc: si.svc.publicTLSSvc, | ||
| @@ -442,6 +442,24 @@ func (si *serviceInit) initHandlers() { | |||||
| 442 | }) | 442 | }) | ||
| 443 | } | 443 | } | ||
| 444 | 444 | ||||
| 445 | + | // backupServicePort returns the database backup service as a port, or an | |||
| 446 | + | // untyped nil when backups are disabled. Storing a nil *backup.Service in | |||
| 447 | + | // the interface would make it non-nil and handlers would call into it. | |||
| 448 | + | func backupServicePort(svc *backup.Service) in.BackupService { | |||
| 449 | + | if svc == nil { | |||
| 450 | + | return nil | |||
| 451 | + | } | |||
| 452 | + | return svc | |||
| 453 | + | } | |||
| 454 | + | ||||
| 455 | + | // volumeBackupServicePort is backupServicePort for volume backups. | |||
| 456 | + | func volumeBackupServicePort(svc *backup.VolumeService) in.VolumeBackupService { | |||
| 457 | + | if svc == nil { | |||
| 458 | + | return nil | |||
| 459 | + | } | |||
| 460 | + | return svc | |||
| 461 | + | } | |||
| 462 | + | ||||
| 445 | // caInfoProvider returns the internal CA as a CAInfoService, or an untyped nil | 463 | // caInfoProvider returns the internal CA as a CAInfoService, or an untyped nil | ||
| 446 | // when internal TLS is disabled so the admin handler sees a nil interface. | 464 | // when internal TLS is disabled so the admin handler sees a nil interface. | ||
| 447 | func (si *serviceInit) caInfoProvider() in.CAInfoService { | 465 | func (si *serviceInit) caInfoProvider() in.CAInfoService { | ||
A internal/app/services_backup_ports_test.go +47 -0
| @@ -0,0 +1,47 @@ | |||
| 1 | + | package app | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "net/http" | |
| 6 | + | "net/http/httptest" | |
| 7 | + | "testing" | |
| 8 | + | ||
| 9 | + | "github.com/bnema/zerowrap" | |
| 10 | + | "github.com/stretchr/testify/assert" | |
| 11 | + | "github.com/stretchr/testify/require" | |
| 12 | + | ||
| 13 | + | "github.com/bnema/gordon/internal/adapters/in/http/admin" | |
| 14 | + | "github.com/bnema/gordon/internal/domain" | |
| 15 | + | "github.com/bnema/gordon/internal/usecase/backup" | |
| 16 | + | ) | |
| 17 | + | ||
| 18 | + | func TestBackupServicePortsReturnUntypedNilWhenDisabled(t *testing.T) { | |
| 19 | + | assert.True(t, backupServicePort(nil) == nil) | |
| 20 | + | assert.True(t, volumeBackupServicePort(nil) == nil) | |
| 21 | + | ||
| 22 | + | assert.True(t, backupServicePort(&backup.Service{}) != nil) | |
| 23 | + | assert.True(t, volumeBackupServicePort(&backup.VolumeService{}) != nil) | |
| 24 | + | } | |
| 25 | + | ||
| 26 | + | // TestAdminBackupRoutesWhenBackupsDisabled proves the admin API answers | |
| 27 | + | // 503 instead of panicking when database or volume backups are disabled. | |
| 28 | + | func TestAdminBackupRoutesWhenBackupsDisabled(t *testing.T) { | |
| 29 | + | var disabled *backup.Service | |
| 30 | + | var disabledVolumes *backup.VolumeService | |
| 31 | + | handler := admin.NewHandler(admin.HandlerDeps{ | |
| 32 | + | Log: zerowrap.Default(), | |
| 33 | + | BackupSvc: backupServicePort(disabled), | |
| 34 | + | VolumeBackupSvc: volumeBackupServicePort(disabledVolumes), | |
| 35 | + | }) | |
| 36 | + | ||
| 37 | + | for _, path := range []string{"/admin/backups/status", "/admin/backups/volumes/status"} { | |
| 38 | + | t.Run(path, func(t *testing.T) { | |
| 39 | + | req := httptest.NewRequest(http.MethodGet, path, nil) | |
| 40 | + | req = req.WithContext(context.WithValue(req.Context(), domain.ContextKeyScopes, []string{"admin:*:*"})) | |
| 41 | + | rec := httptest.NewRecorder() | |
| 42 | + | ||
| 43 | + | require.NotPanics(t, func() { handler.ServeHTTP(rec, req) }) | |
| 44 | + | assert.Equal(t, http.StatusServiceUnavailable, rec.Code) | |
| 45 | + | }) | |
| 46 | + | } | |
| 47 | + | } | |
| @@ -0,0 +1,47 @@ | |||||
| 1 | + | package app | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "net/http" | |||
| 6 | + | "net/http/httptest" | |||
| 7 | + | "testing" | |||
| 8 | + | ||||
| 9 | + | "github.com/bnema/zerowrap" | |||
| 10 | + | "github.com/stretchr/testify/assert" | |||
| 11 | + | "github.com/stretchr/testify/require" | |||
| 12 | + | ||||
| 13 | + | "github.com/bnema/gordon/internal/adapters/in/http/admin" | |||
| 14 | + | "github.com/bnema/gordon/internal/domain" | |||
| 15 | + | "github.com/bnema/gordon/internal/usecase/backup" | |||
| 16 | + | ) | |||
| 17 | + | ||||
| 18 | + | func TestBackupServicePortsReturnUntypedNilWhenDisabled(t *testing.T) { | |||
| 19 | + | assert.True(t, backupServicePort(nil) == nil) | |||
| 20 | + | assert.True(t, volumeBackupServicePort(nil) == nil) | |||
| 21 | + | ||||
| 22 | + | assert.True(t, backupServicePort(&backup.Service{}) != nil) | |||
| 23 | + | assert.True(t, volumeBackupServicePort(&backup.VolumeService{}) != nil) | |||
| 24 | + | } | |||
| 25 | + | ||||
| 26 | + | // TestAdminBackupRoutesWhenBackupsDisabled proves the admin API answers | |||
| 27 | + | // 503 instead of panicking when database or volume backups are disabled. | |||
| 28 | + | func TestAdminBackupRoutesWhenBackupsDisabled(t *testing.T) { | |||
| 29 | + | var disabled *backup.Service | |||
| 30 | + | var disabledVolumes *backup.VolumeService | |||
| 31 | + | handler := admin.NewHandler(admin.HandlerDeps{ | |||
| 32 | + | Log: zerowrap.Default(), | |||
| 33 | + | BackupSvc: backupServicePort(disabled), | |||
| 34 | + | VolumeBackupSvc: volumeBackupServicePort(disabledVolumes), | |||
| 35 | + | }) | |||
| 36 | + | ||||
| 37 | + | for _, path := range []string{"/admin/backups/status", "/admin/backups/volumes/status"} { | |||
| 38 | + | t.Run(path, func(t *testing.T) { | |||
| 39 | + | req := httptest.NewRequest(http.MethodGet, path, nil) | |||
| 40 | + | req = req.WithContext(context.WithValue(req.Context(), domain.ContextKeyScopes, []string{"admin:*:*"})) | |||
| 41 | + | rec := httptest.NewRecorder() | |||
| 42 | + | ||||
| 43 | + | require.NotPanics(t, func() { handler.ServeHTTP(rec, req) }) | |||
| 44 | + | assert.Equal(t, http.StatusServiceUnavailable, rec.Code) | |||
| 45 | + | }) | |||
| 46 | + | } | |||
| 47 | + | } | |||
M internal/domain/app_spec.go +10 -0
| @@ -1074,6 +1074,16 @@ func SameAppService(a, b AppService) bool { | |||
| 1074 | 1074 | return len(diffService(a.Name, a, b)) == 0 | |
| 1075 | 1075 | } | |
| 1076 | 1076 | ||
| 1077 | + | // SameAppServiceRuntime reports whether two service specs describe the same | |
| 1078 | + | // running container. It is SameAppService minus the settings that never reach | |
| 1079 | + | // the container: backup declarations only select backup targets, so a | |
| 1080 | + | // backup-only change must not force a replacement. DiffAppSpec keeps | |
| 1081 | + | // reporting those changes. | |
| 1082 | + | func SameAppServiceRuntime(a, b AppService) bool { | |
| 1083 | + | a.Backup, b.Backup = AppBackup{}, AppBackup{} | |
| 1084 | + | return SameAppService(a, b) | |
| 1085 | + | } | |
| 1086 | + | ||
| 1077 | 1087 | // diffService compares two services field by field. | |
| 1078 | 1088 | func diffService(name string, desired, effective AppService) []string { | |
| 1079 | 1089 | var changed []string | |
| @@ -1074,6 +1074,16 @@ func SameAppService(a, b AppService) bool { | |||||
| 1074 | return len(diffService(a.Name, a, b)) == 0 | 1074 | return len(diffService(a.Name, a, b)) == 0 | ||
| 1075 | } | 1075 | } | ||
| 1076 | 1076 | ||||
| 1077 | + | // SameAppServiceRuntime reports whether two service specs describe the same | |||
| 1078 | + | // running container. It is SameAppService minus the settings that never reach | |||
| 1079 | + | // the container: backup declarations only select backup targets, so a | |||
| 1080 | + | // backup-only change must not force a replacement. DiffAppSpec keeps | |||
| 1081 | + | // reporting those changes. | |||
| 1082 | + | func SameAppServiceRuntime(a, b AppService) bool { | |||
| 1083 | + | a.Backup, b.Backup = AppBackup{}, AppBackup{} | |||
| 1084 | + | return SameAppService(a, b) | |||
| 1085 | + | } | |||
| 1086 | + | ||||
| 1077 | // diffService compares two services field by field. | 1087 | // diffService compares two services field by field. | ||
| 1078 | func diffService(name string, desired, effective AppService) []string { | 1088 | func diffService(name string, desired, effective AppService) []string { | ||
| 1079 | var changed []string | 1089 | var changed []string | ||
M internal/domain/app_spec_test.go +17 -0
| @@ -296,3 +296,20 @@ func TestActiveAppSpec_EmptyActiveHasEmptyEnv(t *testing.T) { | |||
| 296 | 296 | assert.Empty(t, spec.Env) | |
| 297 | 297 | assert.NotNil(t, spec.Env) | |
| 298 | 298 | } | |
| 299 | + | ||
| 300 | + | func TestAppService_BackupOnlyChange(t *testing.T) { | |
| 301 | + | base := validSpec() | |
| 302 | + | changed := validSpec() | |
| 303 | + | changed.Services[0].Backup = domain.AppBackup{Volume: []string{"web-data"}} | |
| 304 | + | ||
| 305 | + | // The user-facing diff keeps reporting the backup change. | |
| 306 | + | assert.Equal(t, []string{"service/web/backup"}, domain.DiffAppSpec(changed, base).Changed) | |
| 307 | + | assert.False(t, domain.SameAppService(changed.Services[0], base.Services[0])) | |
| 308 | + | ||
| 309 | + | // The runtime comparison ignores it. | |
| 310 | + | assert.True(t, domain.SameAppServiceRuntime(changed.Services[0], base.Services[0])) | |
| 311 | + | ||
| 312 | + | // A real runtime change is still detected alongside a backup change. | |
| 313 | + | changed.Services[0].Command = []string{"serve"} | |
| 314 | + | assert.False(t, domain.SameAppServiceRuntime(changed.Services[0], base.Services[0])) | |
| 315 | + | } | |
| @@ -296,3 +296,20 @@ func TestActiveAppSpec_EmptyActiveHasEmptyEnv(t *testing.T) { | |||||
| 296 | assert.Empty(t, spec.Env) | 296 | assert.Empty(t, spec.Env) | ||
| 297 | assert.NotNil(t, spec.Env) | 297 | assert.NotNil(t, spec.Env) | ||
| 298 | } | 298 | } | ||
| 299 | + | ||||
| 300 | + | func TestAppService_BackupOnlyChange(t *testing.T) { | |||
| 301 | + | base := validSpec() | |||
| 302 | + | changed := validSpec() | |||
| 303 | + | changed.Services[0].Backup = domain.AppBackup{Volume: []string{"web-data"}} | |||
| 304 | + | ||||
| 305 | + | // The user-facing diff keeps reporting the backup change. | |||
| 306 | + | assert.Equal(t, []string{"service/web/backup"}, domain.DiffAppSpec(changed, base).Changed) | |||
| 307 | + | assert.False(t, domain.SameAppService(changed.Services[0], base.Services[0])) | |||
| 308 | + | ||||
| 309 | + | // The runtime comparison ignores it. | |||
| 310 | + | assert.True(t, domain.SameAppServiceRuntime(changed.Services[0], base.Services[0])) | |||
| 311 | + | ||||
| 312 | + | // A real runtime change is still detected alongside a backup change. | |||
| 313 | + | changed.Services[0].Command = []string{"serve"} | |||
| 314 | + | assert.False(t, domain.SameAppServiceRuntime(changed.Services[0], base.Services[0])) | |||
| 315 | + | } | |||
M internal/domain/backup.go +1 -1
| @@ -127,7 +127,7 @@ type DatabaseBackupConfig = BackupConfig | |||
| 127 | 127 | // VolumeBackupConfig is the volume backup configuration. | |
| 128 | 128 | type VolumeBackupConfig struct { | |
| 129 | 129 | Enabled bool | |
| 130 | - | Interval time.Duration | |
| 130 | + | Schedule CronSchedule | |
| 131 | 131 | Compression VolumeBackupCompression | |
| 132 | 132 | Retention VolumeBackupRetentionPolicy | |
| 133 | 133 | Timeout time.Duration | |
| @@ -127,7 +127,7 @@ type DatabaseBackupConfig = BackupConfig | |||||
| 127 | // VolumeBackupConfig is the volume backup configuration. | 127 | // VolumeBackupConfig is the volume backup configuration. | ||
| 128 | type VolumeBackupConfig struct { | 128 | type VolumeBackupConfig struct { | ||
| 129 | Enabled bool | 129 | Enabled bool | ||
| 130 | - | Interval time.Duration | 130 | + | Schedule CronSchedule |
| 131 | Compression VolumeBackupCompression | 131 | Compression VolumeBackupCompression | ||
| 132 | Retention VolumeBackupRetentionPolicy | 132 | Retention VolumeBackupRetentionPolicy | ||
| 133 | Timeout time.Duration | 133 | Timeout time.Duration | ||
M internal/domain/cron.go +38 -6
| @@ -1,15 +1,47 @@ | |||
| 1 | 1 | package domain | |
| 2 | 2 | ||
| 3 | - | import "time" | |
| 3 | + | import ( | |
| 4 | + | "fmt" | |
| 5 | + | "strconv" | |
| 6 | + | "strings" | |
| 7 | + | "time" | |
| 8 | + | ) | |
| 4 | 9 | ||
| 5 | - | // CronSchedule represents a recurring schedule. | |
| 10 | + | // CronSchedule represents a recurring schedule: a preset, an optional time | |
| 11 | + | // of day, and the timezone that time is read in. | |
| 6 | 12 | // | |
| 7 | - | // Preset schedules currently use fixed UTC times by design (MVP scope). | |
| 8 | - | // Timezone-aware scheduling is deferred until a timezone field is added. | |
| 9 | - | // When timezone support lands, host system timezone should be the default. | |
| 13 | + | // When At is nil the preset keeps its built-in time (daily 02:00, weekly | |
| 14 | + | // Sunday 03:00, monthly day 1 04:00, hourly on the hour). When Location is | |
| 15 | + | // nil times are read in UTC. | |
| 10 | 16 | type CronSchedule struct { | |
| 11 | 17 | Preset BackupSchedule | |
| 12 | - | Interval time.Duration | |
| 18 | + | At *TimeOfDay | |
| 19 | + | Location *time.Location | |
| 20 | + | } | |
| 21 | + | ||
| 22 | + | // TimeOfDay is a wall-clock time. Hourly schedules use only Minute. | |
| 23 | + | type TimeOfDay struct { | |
| 24 | + | Hour int | |
| 25 | + | Minute int | |
| 26 | + | } | |
| 27 | + | ||
| 28 | + | // String formats the time as HH:MM. | |
| 29 | + | func (t TimeOfDay) String() string { | |
| 30 | + | return fmt.Sprintf("%02d:%02d", t.Hour, t.Minute) | |
| 31 | + | } | |
| 32 | + | ||
| 33 | + | // ParseTimeOfDay parses a 24-hour "HH:MM" wall-clock time. | |
| 34 | + | func ParseTimeOfDay(raw string) (TimeOfDay, error) { | |
| 35 | + | hh, mm, ok := strings.Cut(strings.TrimSpace(raw), ":") | |
| 36 | + | if !ok || len(hh) == 0 || len(hh) > 2 || len(mm) != 2 { | |
| 37 | + | return TimeOfDay{}, fmt.Errorf("time must use 24-hour HH:MM format, got %q", raw) | |
| 38 | + | } | |
| 39 | + | hour, errH := strconv.Atoi(hh) | |
| 40 | + | minute, errM := strconv.Atoi(mm) | |
| 41 | + | if errH != nil || errM != nil || hour < 0 || hour > 23 || minute < 0 || minute > 59 { | |
| 42 | + | return TimeOfDay{}, fmt.Errorf("time must use 24-hour HH:MM format, got %q", raw) | |
| 43 | + | } | |
| 44 | + | return TimeOfDay{Hour: hour, Minute: minute}, nil | |
| 13 | 45 | } | |
| 14 | 46 | ||
| 15 | 47 | // CronEntry represents a registered cron job. | |
| @@ -1,15 +1,47 @@ | |||||
| 1 | package domain | 1 | package domain | ||
| 2 | 2 | ||||
| 3 | - | import "time" | 3 | + | import ( |
| 4 | + | "fmt" | |||
| 5 | + | "strconv" | |||
| 6 | + | "strings" | |||
| 7 | + | "time" | |||
| 8 | + | ) | |||
| 4 | 9 | ||||
| 5 | - | // CronSchedule represents a recurring schedule. | 10 | + | // CronSchedule represents a recurring schedule: a preset, an optional time |
| 11 | + | // of day, and the timezone that time is read in. | |||
| 6 | // | 12 | // | ||
| 7 | - | // Preset schedules currently use fixed UTC times by design (MVP scope). | 13 | + | // When At is nil the preset keeps its built-in time (daily 02:00, weekly |
| 8 | - | // Timezone-aware scheduling is deferred until a timezone field is added. | 14 | + | // Sunday 03:00, monthly day 1 04:00, hourly on the hour). When Location is |
| 9 | - | // When timezone support lands, host system timezone should be the default. | 15 | + | // nil times are read in UTC. |
| 10 | type CronSchedule struct { | 16 | type CronSchedule struct { | ||
| 11 | Preset BackupSchedule | 17 | Preset BackupSchedule | ||
| 12 | - | Interval time.Duration | 18 | + | At *TimeOfDay |
| 19 | + | Location *time.Location | |||
| 20 | + | } | |||
| 21 | + | ||||
| 22 | + | // TimeOfDay is a wall-clock time. Hourly schedules use only Minute. | |||
| 23 | + | type TimeOfDay struct { | |||
| 24 | + | Hour int | |||
| 25 | + | Minute int | |||
| 26 | + | } | |||
| 27 | + | ||||
| 28 | + | // String formats the time as HH:MM. | |||
| 29 | + | func (t TimeOfDay) String() string { | |||
| 30 | + | return fmt.Sprintf("%02d:%02d", t.Hour, t.Minute) | |||
| 31 | + | } | |||
| 32 | + | ||||
| 33 | + | // ParseTimeOfDay parses a 24-hour "HH:MM" wall-clock time. | |||
| 34 | + | func ParseTimeOfDay(raw string) (TimeOfDay, error) { | |||
| 35 | + | hh, mm, ok := strings.Cut(strings.TrimSpace(raw), ":") | |||
| 36 | + | if !ok || len(hh) == 0 || len(hh) > 2 || len(mm) != 2 { | |||
| 37 | + | return TimeOfDay{}, fmt.Errorf("time must use 24-hour HH:MM format, got %q", raw) | |||
| 38 | + | } | |||
| 39 | + | hour, errH := strconv.Atoi(hh) | |||
| 40 | + | minute, errM := strconv.Atoi(mm) | |||
| 41 | + | if errH != nil || errM != nil || hour < 0 || hour > 23 || minute < 0 || minute > 59 { | |||
| 42 | + | return TimeOfDay{}, fmt.Errorf("time must use 24-hour HH:MM format, got %q", raw) | |||
| 43 | + | } | |||
| 44 | + | return TimeOfDay{Hour: hour, Minute: minute}, nil | |||
| 13 | } | 45 | } | ||
| 14 | 46 | ||||
| 15 | // CronEntry represents a registered cron job. | 47 | // CronEntry represents a registered cron job. | ||
A internal/usecase/backup/volume_job_id_test.go +17 -0
| @@ -0,0 +1,17 @@ | |||
| 1 | + | package backup | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "regexp" | |
| 5 | + | "testing" | |
| 6 | + | ||
| 7 | + | "github.com/stretchr/testify/assert" | |
| 8 | + | ) | |
| 9 | + | ||
| 10 | + | // TestNewVolumeBackupJobIDIsShortAndUnique proves the volume job ID adds | |
| 11 | + | // only a short random suffix: the storage key already holds the timestamp. | |
| 12 | + | func TestNewVolumeBackupJobIDIsShortAndUnique(t *testing.T) { | |
| 13 | + | a, b := newVolumeBackupJobID(), newVolumeBackupJobID() | |
| 14 | + | ||
| 15 | + | assert.Regexp(t, regexp.MustCompile(`^[0-9a-f]{8}$`), a) | |
| 16 | + | assert.NotEqual(t, a, b) | |
| 17 | + | } | |
| @@ -0,0 +1,17 @@ | |||||
| 1 | + | package backup | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "regexp" | |||
| 5 | + | "testing" | |||
| 6 | + | ||||
| 7 | + | "github.com/stretchr/testify/assert" | |||
| 8 | + | ) | |||
| 9 | + | ||||
| 10 | + | // TestNewVolumeBackupJobIDIsShortAndUnique proves the volume job ID adds | |||
| 11 | + | // only a short random suffix: the storage key already holds the timestamp. | |||
| 12 | + | func TestNewVolumeBackupJobIDIsShortAndUnique(t *testing.T) { | |||
| 13 | + | a, b := newVolumeBackupJobID(), newVolumeBackupJobID() | |||
| 14 | + | ||||
| 15 | + | assert.Regexp(t, regexp.MustCompile(`^[0-9a-f]{8}$`), a) | |||
| 16 | + | assert.NotEqual(t, a, b) | |||
| 17 | + | } | |||
M internal/usecase/backup/volume_service.go +40 -3
| @@ -2,6 +2,8 @@ package backup | |||
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | + | "crypto/rand" | |
| 6 | + | "encoding/hex" | |
| 5 | 7 | "fmt" | |
| 6 | 8 | "io" | |
| 7 | 9 | "sort" | |
| @@ -165,10 +167,10 @@ func (s *VolumeService) RunVolumeBackupsForSchedule(ctx context.Context, schedul | |||
| 165 | 167 | if err != nil { | |
| 166 | 168 | return err | |
| 167 | 169 | } | |
| 170 | + | jobs := s.runVolumeBackupsConcurrently(ctx, targets) | |
| 168 | 171 | var firstErr error | |
| 169 | 172 | apps := map[string]struct{}{} | |
| 170 | - | for _, target := range targets { | |
| 171 | - | job := s.runVolumeBackup(ctx, target) | |
| 173 | + | for _, job := range jobs { | |
| 172 | 174 | if job.Status != domain.BackupStatusCompleted { | |
| 173 | 175 | if firstErr == nil { | |
| 174 | 176 | firstErr = fmt.Errorf("volume backup %s/%s: %s", job.App, job.VolumeName, job.Error) | |
| @@ -177,6 +179,8 @@ func (s *VolumeService) RunVolumeBackupsForSchedule(ctx context.Context, schedul | |||
| 177 | 179 | } | |
| 178 | 180 | apps[job.App] = struct{}{} | |
| 179 | 181 | } | |
| 182 | + | // Retention runs only after every archive finished, so a pass never | |
| 183 | + | // deletes an app's older archive while its replacement is uploading. | |
| 180 | 184 | for _, app := range sortedSet(apps) { | |
| 181 | 185 | if _, err := s.storage.ApplyVolumeRetention(ctx, app, s.config.Retention); err != nil { | |
| 182 | 186 | if firstErr == nil { | |
| @@ -187,6 +191,29 @@ func (s *VolumeService) RunVolumeBackupsForSchedule(ctx context.Context, schedul | |||
| 187 | 191 | return firstErr | |
| 188 | 192 | } | |
| 189 | 193 | ||
| 194 | + | // runVolumeBackupsConcurrently runs every target with at most | |
| 195 | + | // MaxConcurrency exports in flight and returns the jobs in target order. | |
| 196 | + | func (s *VolumeService) runVolumeBackupsConcurrently(ctx context.Context, targets []domain.VolumeBackupTarget) []domain.VolumeBackupJob { | |
| 197 | + | limit := s.config.MaxConcurrency | |
| 198 | + | if limit < 1 { | |
| 199 | + | limit = 1 | |
| 200 | + | } | |
| 201 | + | jobs := make([]domain.VolumeBackupJob, len(targets)) | |
| 202 | + | sem := make(chan struct{}, limit) | |
| 203 | + | var wg sync.WaitGroup | |
| 204 | + | for i, target := range targets { | |
| 205 | + | wg.Add(1) | |
| 206 | + | sem <- struct{}{} | |
| 207 | + | go func(i int, target domain.VolumeBackupTarget) { | |
| 208 | + | defer wg.Done() | |
| 209 | + | defer func() { <-sem }() | |
| 210 | + | jobs[i] = s.runVolumeBackup(ctx, target) | |
| 211 | + | }(i, target) | |
| 212 | + | } | |
| 213 | + | wg.Wait() | |
| 214 | + | return jobs | |
| 215 | + | } | |
| 216 | + | ||
| 190 | 217 | func (s *VolumeService) runVolumeBackup(ctx context.Context, target domain.VolumeBackupTarget) domain.VolumeBackupJob { | |
| 191 | 218 | ctx = zerowrap.CtxWithFields(ctx, map[string]any{ | |
| 192 | 219 | "app": target.App, | |
| @@ -198,7 +225,7 @@ func (s *VolumeService) runVolumeBackup(ctx context.Context, target domain.Volum | |||
| 198 | 225 | log := zerowrap.FromCtx(ctx) | |
| 199 | 226 | started := time.Now().UTC() | |
| 200 | 227 | job := domain.VolumeBackupJob{ | |
| 201 | - | ID: newBackupJobID(started), | |
| 228 | + | ID: newVolumeBackupJobID(), | |
| 202 | 229 | App: target.App, | |
| 203 | 230 | Service: target.Service, | |
| 204 | 231 | VolumeName: target.VolumeName, | |
| @@ -251,6 +278,16 @@ func (s *VolumeService) runVolumeBackup(ctx context.Context, target domain.Volum | |||
| 251 | 278 | return job | |
| 252 | 279 | } | |
| 253 | 280 | ||
| 281 | + | // newVolumeBackupJobID returns a short random job ID. Storage keys already | |
| 282 | + | // carry the start timestamp, so the ID only has to make each run unique. | |
| 283 | + | func newVolumeBackupJobID() string { | |
| 284 | + | random := make([]byte, 4) | |
| 285 | + | if _, err := rand.Read(random); err != nil { | |
| 286 | + | return fmt.Sprintf("%x", time.Now().UnixNano()) | |
| 287 | + | } | |
| 288 | + | return hex.EncodeToString(random) | |
| 289 | + | } | |
| 290 | + | ||
| 254 | 291 | // volumeTargetKey reports the (service, volume) identity of one target. | |
| 255 | 292 | func volumeTargetKey(target domain.VolumeBackupTarget) (string, string) { | |
| 256 | 293 | return target.Service, target.VolumeName | |
| @@ -2,6 +2,8 @@ package backup | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | + | "crypto/rand" | |||
| 6 | + | "encoding/hex" | |||
| 5 | "fmt" | 7 | "fmt" | ||
| 6 | "io" | 8 | "io" | ||
| 7 | "sort" | 9 | "sort" | ||
| @@ -165,10 +167,10 @@ func (s *VolumeService) RunVolumeBackupsForSchedule(ctx context.Context, schedul | |||||
| 165 | if err != nil { | 167 | if err != nil { | ||
| 166 | return err | 168 | return err | ||
| 167 | } | 169 | } | ||
| 170 | + | jobs := s.runVolumeBackupsConcurrently(ctx, targets) | |||
| 168 | var firstErr error | 171 | var firstErr error | ||
| 169 | apps := map[string]struct{}{} | 172 | apps := map[string]struct{}{} | ||
| 170 | - | for _, target := range targets { | 173 | + | for _, job := range jobs { |
| 171 | - | job := s.runVolumeBackup(ctx, target) | |||
| 172 | if job.Status != domain.BackupStatusCompleted { | 174 | if job.Status != domain.BackupStatusCompleted { | ||
| 173 | if firstErr == nil { | 175 | if firstErr == nil { | ||
| 174 | firstErr = fmt.Errorf("volume backup %s/%s: %s", job.App, job.VolumeName, job.Error) | 176 | firstErr = fmt.Errorf("volume backup %s/%s: %s", job.App, job.VolumeName, job.Error) | ||
| @@ -177,6 +179,8 @@ func (s *VolumeService) RunVolumeBackupsForSchedule(ctx context.Context, schedul | |||||
| 177 | } | 179 | } | ||
| 178 | apps[job.App] = struct{}{} | 180 | apps[job.App] = struct{}{} | ||
| 179 | } | 181 | } | ||
| 182 | + | // Retention runs only after every archive finished, so a pass never | |||
| 183 | + | // deletes an app's older archive while its replacement is uploading. | |||
| 180 | for _, app := range sortedSet(apps) { | 184 | for _, app := range sortedSet(apps) { | ||
| 181 | if _, err := s.storage.ApplyVolumeRetention(ctx, app, s.config.Retention); err != nil { | 185 | if _, err := s.storage.ApplyVolumeRetention(ctx, app, s.config.Retention); err != nil { | ||
| 182 | if firstErr == nil { | 186 | if firstErr == nil { | ||
| @@ -187,6 +191,29 @@ func (s *VolumeService) RunVolumeBackupsForSchedule(ctx context.Context, schedul | |||||
| 187 | return firstErr | 191 | return firstErr | ||
| 188 | } | 192 | } | ||
| 189 | 193 | ||||
| 194 | + | // runVolumeBackupsConcurrently runs every target with at most | |||
| 195 | + | // MaxConcurrency exports in flight and returns the jobs in target order. | |||
| 196 | + | func (s *VolumeService) runVolumeBackupsConcurrently(ctx context.Context, targets []domain.VolumeBackupTarget) []domain.VolumeBackupJob { | |||
| 197 | + | limit := s.config.MaxConcurrency | |||
| 198 | + | if limit < 1 { | |||
| 199 | + | limit = 1 | |||
| 200 | + | } | |||
| 201 | + | jobs := make([]domain.VolumeBackupJob, len(targets)) | |||
| 202 | + | sem := make(chan struct{}, limit) | |||
| 203 | + | var wg sync.WaitGroup | |||
| 204 | + | for i, target := range targets { | |||
| 205 | + | wg.Add(1) | |||
| 206 | + | sem <- struct{}{} | |||
| 207 | + | go func(i int, target domain.VolumeBackupTarget) { | |||
| 208 | + | defer wg.Done() | |||
| 209 | + | defer func() { <-sem }() | |||
| 210 | + | jobs[i] = s.runVolumeBackup(ctx, target) | |||
| 211 | + | }(i, target) | |||
| 212 | + | } | |||
| 213 | + | wg.Wait() | |||
| 214 | + | return jobs | |||
| 215 | + | } | |||
| 216 | + | ||||
| 190 | func (s *VolumeService) runVolumeBackup(ctx context.Context, target domain.VolumeBackupTarget) domain.VolumeBackupJob { | 217 | func (s *VolumeService) runVolumeBackup(ctx context.Context, target domain.VolumeBackupTarget) domain.VolumeBackupJob { | ||
| 191 | ctx = zerowrap.CtxWithFields(ctx, map[string]any{ | 218 | ctx = zerowrap.CtxWithFields(ctx, map[string]any{ | ||
| 192 | "app": target.App, | 219 | "app": target.App, | ||
| @@ -198,7 +225,7 @@ func (s *VolumeService) runVolumeBackup(ctx context.Context, target domain.Volum | |||||
| 198 | log := zerowrap.FromCtx(ctx) | 225 | log := zerowrap.FromCtx(ctx) | ||
| 199 | started := time.Now().UTC() | 226 | started := time.Now().UTC() | ||
| 200 | job := domain.VolumeBackupJob{ | 227 | job := domain.VolumeBackupJob{ | ||
| 201 | - | ID: newBackupJobID(started), | 228 | + | ID: newVolumeBackupJobID(), |
| 202 | App: target.App, | 229 | App: target.App, | ||
| 203 | Service: target.Service, | 230 | Service: target.Service, | ||
| 204 | VolumeName: target.VolumeName, | 231 | VolumeName: target.VolumeName, | ||
| @@ -251,6 +278,16 @@ func (s *VolumeService) runVolumeBackup(ctx context.Context, target domain.Volum | |||||
| 251 | return job | 278 | return job | ||
| 252 | } | 279 | } | ||
| 253 | 280 | ||||
| 281 | + | // newVolumeBackupJobID returns a short random job ID. Storage keys already | |||
| 282 | + | // carry the start timestamp, so the ID only has to make each run unique. | |||
| 283 | + | func newVolumeBackupJobID() string { | |||
| 284 | + | random := make([]byte, 4) | |||
| 285 | + | if _, err := rand.Read(random); err != nil { | |||
| 286 | + | return fmt.Sprintf("%x", time.Now().UnixNano()) | |||
| 287 | + | } | |||
| 288 | + | return hex.EncodeToString(random) | |||
| 289 | + | } | |||
| 290 | + | ||||
| 254 | // volumeTargetKey reports the (service, volume) identity of one target. | 291 | // volumeTargetKey reports the (service, volume) identity of one target. | ||
| 255 | func volumeTargetKey(target domain.VolumeBackupTarget) (string, string) { | 292 | func volumeTargetKey(target domain.VolumeBackupTarget) (string, string) { | ||
| 256 | return target.Service, target.VolumeName | 293 | return target.Service, target.VolumeName | ||
M internal/usecase/backup/volume_service_test.go +74 -0
| @@ -4,7 +4,9 @@ import ( | |||
| 4 | 4 | "bytes" | |
| 5 | 5 | "context" | |
| 6 | 6 | "errors" | |
| 7 | + | "fmt" | |
| 7 | 8 | "io" | |
| 9 | + | "sync" | |
| 8 | 10 | "testing" | |
| 9 | 11 | "time" | |
| 10 | 12 | ||
| @@ -22,12 +24,33 @@ func testLogger() zerowrap.Logger { | |||
| 22 | 24 | } | |
| 23 | 25 | ||
| 24 | 26 | type fakeVolumeArchiveExporter struct { | |
| 27 | + | mu sync.Mutex | |
| 25 | 28 | err error | |
| 26 | 29 | requested []domain.VolumeArchiveRequest | |
| 30 | + | // hold, when set, keeps each export open until released so tests can | |
| 31 | + | // observe how many run at once. | |
| 32 | + | hold chan struct{} | |
| 33 | + | inFlight int | |
| 34 | + | peak int | |
| 27 | 35 | } | |
| 28 | 36 | ||
| 29 | 37 | func (f *fakeVolumeArchiveExporter) ExportVolumeArchive(_ context.Context, request domain.VolumeArchiveRequest) (*domain.VolumeArchiveResult, error) { | |
| 38 | + | f.mu.Lock() | |
| 30 | 39 | f.requested = append(f.requested, request) | |
| 40 | + | f.inFlight++ | |
| 41 | + | if f.inFlight > f.peak { | |
| 42 | + | f.peak = f.inFlight | |
| 43 | + | } | |
| 44 | + | hold := f.hold | |
| 45 | + | f.mu.Unlock() | |
| 46 | + | defer func() { | |
| 47 | + | f.mu.Lock() | |
| 48 | + | f.inFlight-- | |
| 49 | + | f.mu.Unlock() | |
| 50 | + | }() | |
| 51 | + | if hold != nil { | |
| 52 | + | <-hold | |
| 53 | + | } | |
| 31 | 54 | if f.err != nil { | |
| 32 | 55 | return nil, f.err | |
| 33 | 56 | } | |
| @@ -35,6 +58,7 @@ func (f *fakeVolumeArchiveExporter) ExportVolumeArchive(_ context.Context, reque | |||
| 35 | 58 | } | |
| 36 | 59 | ||
| 37 | 60 | type fakeVolumeBackupStorage struct { | |
| 61 | + | mu sync.Mutex | |
| 38 | 62 | stored []domain.VolumeBackupJob | |
| 39 | 63 | retention []string | |
| 40 | 64 | jobs []domain.VolumeBackupJob | |
| @@ -49,7 +73,9 @@ func (f *fakeVolumeBackupStorage) StoreVolumeArchive(_ context.Context, job doma | |||
| 49 | 73 | if f.storeErr != nil { | |
| 50 | 74 | return "", f.storeErr | |
| 51 | 75 | } | |
| 76 | + | f.mu.Lock() | |
| 52 | 77 | f.stored = append(f.stored, job) | |
| 78 | + | f.mu.Unlock() | |
| 53 | 79 | return "s3://bucket/" + job.App + "/" + job.VolumeName, nil | |
| 54 | 80 | } | |
| 55 | 81 | ||
| @@ -194,6 +220,54 @@ func TestVolumeService_RunVolumeBackupsForSchedule_RunsEveryDeclaredTarget(t *te | |||
| 194 | 220 | assert.Equal(t, []string{"shop"}, storage.retention) | |
| 195 | 221 | } | |
| 196 | 222 | ||
| 223 | + | // TestVolumeService_RunVolumeBackupsForSchedule_HonorsMaxConcurrency proves | |
| 224 | + | // the scheduled pass runs targets in parallel, never more than | |
| 225 | + | // MaxConcurrency at once, and still backs up every target. | |
| 226 | + | func TestVolumeService_RunVolumeBackupsForSchedule_HonorsMaxConcurrency(t *testing.T) { | |
| 227 | + | for _, limit := range []int{1, 2} { | |
| 228 | + | t.Run(fmt.Sprintf("max_concurrency=%d", limit), func(t *testing.T) { | |
| 229 | + | ctx := context.Background() | |
| 230 | + | state := outmocks.NewMockAppStateReader(t) | |
| 231 | + | state.EXPECT().ListApps(mock.Anything).Return([]string{"shop"}, nil).Once() | |
| 232 | + | state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once() | |
| 233 | + | state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{ | |
| 234 | + | "a": volumeSpec("a", "data-a", "/a"), | |
| 235 | + | "b": volumeSpec("b", "data-b", "/b"), | |
| 236 | + | "c": volumeSpec("c", "data-c", "/c"), | |
| 237 | + | }, "ctr"), true, nil).Once() | |
| 238 | + | ||
| 239 | + | hold := make(chan struct{}) | |
| 240 | + | exporter := &fakeVolumeArchiveExporter{hold: hold} | |
| 241 | + | storage := &fakeVolumeBackupStorage{} | |
| 242 | + | svc := NewVolumeService(exporter, storage, domain.VolumeBackupConfig{ | |
| 243 | + | Enabled: true, | |
| 244 | + | Compression: domain.VolumeBackupCompressionGzip, | |
| 245 | + | Retention: domain.VolumeBackupRetentionPolicy{Keep: 2}, | |
| 246 | + | Timeout: time.Minute, | |
| 247 | + | MaxConcurrency: limit, | |
| 248 | + | HelperImage: "helper:latest", | |
| 249 | + | }, testLogger()).WithAppState(state) | |
| 250 | + | ||
| 251 | + | done := make(chan error, 1) | |
| 252 | + | go func() { done <- svc.RunVolumeBackupsForSchedule(ctx, domain.ScheduleDaily) }() | |
| 253 | + | ||
| 254 | + | // Wait until the limit is reached, then confirm it is never exceeded. | |
| 255 | + | require.Eventually(t, func() bool { | |
| 256 | + | exporter.mu.Lock() | |
| 257 | + | defer exporter.mu.Unlock() | |
| 258 | + | return exporter.inFlight == limit | |
| 259 | + | }, 2*time.Second, time.Millisecond) | |
| 260 | + | time.Sleep(20 * time.Millisecond) | |
| 261 | + | close(hold) | |
| 262 | + | ||
| 263 | + | require.NoError(t, <-done) | |
| 264 | + | assert.Equal(t, limit, exporter.peak) | |
| 265 | + | assert.Len(t, storage.stored, 3) | |
| 266 | + | assert.Equal(t, []string{"shop"}, storage.retention) | |
| 267 | + | }) | |
| 268 | + | } | |
| 269 | + | } | |
| 270 | + | ||
| 197 | 271 | // TestVolumeService_DisabledDoesNothing proves a disabled backup | |
| 198 | 272 | // configuration runs nothing at all. | |
| 199 | 273 | func TestVolumeService_DisabledDoesNothing(t *testing.T) { | |
| @@ -4,7 +4,9 @@ import ( | |||||
| 4 | "bytes" | 4 | "bytes" | ||
| 5 | "context" | 5 | "context" | ||
| 6 | "errors" | 6 | "errors" | ||
| 7 | + | "fmt" | |||
| 7 | "io" | 8 | "io" | ||
| 9 | + | "sync" | |||
| 8 | "testing" | 10 | "testing" | ||
| 9 | "time" | 11 | "time" | ||
| 10 | 12 | ||||
| @@ -22,12 +24,33 @@ func testLogger() zerowrap.Logger { | |||||
| 22 | } | 24 | } | ||
| 23 | 25 | ||||
| 24 | type fakeVolumeArchiveExporter struct { | 26 | type fakeVolumeArchiveExporter struct { | ||
| 27 | + | mu sync.Mutex | |||
| 25 | err error | 28 | err error | ||
| 26 | requested []domain.VolumeArchiveRequest | 29 | requested []domain.VolumeArchiveRequest | ||
| 30 | + | // hold, when set, keeps each export open until released so tests can | |||
| 31 | + | // observe how many run at once. | |||
| 32 | + | hold chan struct{} | |||
| 33 | + | inFlight int | |||
| 34 | + | peak int | |||
| 27 | } | 35 | } | ||
| 28 | 36 | ||||
| 29 | func (f *fakeVolumeArchiveExporter) ExportVolumeArchive(_ context.Context, request domain.VolumeArchiveRequest) (*domain.VolumeArchiveResult, error) { | 37 | func (f *fakeVolumeArchiveExporter) ExportVolumeArchive(_ context.Context, request domain.VolumeArchiveRequest) (*domain.VolumeArchiveResult, error) { | ||
| 38 | + | f.mu.Lock() | |||
| 30 | f.requested = append(f.requested, request) | 39 | f.requested = append(f.requested, request) | ||
| 40 | + | f.inFlight++ | |||
| 41 | + | if f.inFlight > f.peak { | |||
| 42 | + | f.peak = f.inFlight | |||
| 43 | + | } | |||
| 44 | + | hold := f.hold | |||
| 45 | + | f.mu.Unlock() | |||
| 46 | + | defer func() { | |||
| 47 | + | f.mu.Lock() | |||
| 48 | + | f.inFlight-- | |||
| 49 | + | f.mu.Unlock() | |||
| 50 | + | }() | |||
| 51 | + | if hold != nil { | |||
| 52 | + | <-hold | |||
| 53 | + | } | |||
| 31 | if f.err != nil { | 54 | if f.err != nil { | ||
| 32 | return nil, f.err | 55 | return nil, f.err | ||
| 33 | } | 56 | } | ||
| @@ -35,6 +58,7 @@ func (f *fakeVolumeArchiveExporter) ExportVolumeArchive(_ context.Context, reque | |||||
| 35 | } | 58 | } | ||
| 36 | 59 | ||||
| 37 | type fakeVolumeBackupStorage struct { | 60 | type fakeVolumeBackupStorage struct { | ||
| 61 | + | mu sync.Mutex | |||
| 38 | stored []domain.VolumeBackupJob | 62 | stored []domain.VolumeBackupJob | ||
| 39 | retention []string | 63 | retention []string | ||
| 40 | jobs []domain.VolumeBackupJob | 64 | jobs []domain.VolumeBackupJob | ||
| @@ -49,7 +73,9 @@ func (f *fakeVolumeBackupStorage) StoreVolumeArchive(_ context.Context, job doma | |||||
| 49 | if f.storeErr != nil { | 73 | if f.storeErr != nil { | ||
| 50 | return "", f.storeErr | 74 | return "", f.storeErr | ||
| 51 | } | 75 | } | ||
| 76 | + | f.mu.Lock() | |||
| 52 | f.stored = append(f.stored, job) | 77 | f.stored = append(f.stored, job) | ||
| 78 | + | f.mu.Unlock() | |||
| 53 | return "s3://bucket/" + job.App + "/" + job.VolumeName, nil | 79 | return "s3://bucket/" + job.App + "/" + job.VolumeName, nil | ||
| 54 | } | 80 | } | ||
| 55 | 81 | ||||
| @@ -194,6 +220,54 @@ func TestVolumeService_RunVolumeBackupsForSchedule_RunsEveryDeclaredTarget(t *te | |||||
| 194 | assert.Equal(t, []string{"shop"}, storage.retention) | 220 | assert.Equal(t, []string{"shop"}, storage.retention) | ||
| 195 | } | 221 | } | ||
| 196 | 222 | ||||
| 223 | + | // TestVolumeService_RunVolumeBackupsForSchedule_HonorsMaxConcurrency proves | |||
| 224 | + | // the scheduled pass runs targets in parallel, never more than | |||
| 225 | + | // MaxConcurrency at once, and still backs up every target. | |||
| 226 | + | func TestVolumeService_RunVolumeBackupsForSchedule_HonorsMaxConcurrency(t *testing.T) { | |||
| 227 | + | for _, limit := range []int{1, 2} { | |||
| 228 | + | t.Run(fmt.Sprintf("max_concurrency=%d", limit), func(t *testing.T) { | |||
| 229 | + | ctx := context.Background() | |||
| 230 | + | state := outmocks.NewMockAppStateReader(t) | |||
| 231 | + | state.EXPECT().ListApps(mock.Anything).Return([]string{"shop"}, nil).Once() | |||
| 232 | + | state.EXPECT().LoadIntent(mock.Anything, "shop").Return(domain.AppStopIntent{App: "shop"}, nil).Once() | |||
| 233 | + | state.EXPECT().LoadActive(mock.Anything, "shop").Return(activeWith("shop", map[string]domain.AppService{ | |||
| 234 | + | "a": volumeSpec("a", "data-a", "/a"), | |||
| 235 | + | "b": volumeSpec("b", "data-b", "/b"), | |||
| 236 | + | "c": volumeSpec("c", "data-c", "/c"), | |||
| 237 | + | }, "ctr"), true, nil).Once() | |||
| 238 | + | ||||
| 239 | + | hold := make(chan struct{}) | |||
| 240 | + | exporter := &fakeVolumeArchiveExporter{hold: hold} | |||
| 241 | + | storage := &fakeVolumeBackupStorage{} | |||
| 242 | + | svc := NewVolumeService(exporter, storage, domain.VolumeBackupConfig{ | |||
| 243 | + | Enabled: true, | |||
| 244 | + | Compression: domain.VolumeBackupCompressionGzip, | |||
| 245 | + | Retention: domain.VolumeBackupRetentionPolicy{Keep: 2}, | |||
| 246 | + | Timeout: time.Minute, | |||
| 247 | + | MaxConcurrency: limit, | |||
| 248 | + | HelperImage: "helper:latest", | |||
| 249 | + | }, testLogger()).WithAppState(state) | |||
| 250 | + | ||||
| 251 | + | done := make(chan error, 1) | |||
| 252 | + | go func() { done <- svc.RunVolumeBackupsForSchedule(ctx, domain.ScheduleDaily) }() | |||
| 253 | + | ||||
| 254 | + | // Wait until the limit is reached, then confirm it is never exceeded. | |||
| 255 | + | require.Eventually(t, func() bool { | |||
| 256 | + | exporter.mu.Lock() | |||
| 257 | + | defer exporter.mu.Unlock() | |||
| 258 | + | return exporter.inFlight == limit | |||
| 259 | + | }, 2*time.Second, time.Millisecond) | |||
| 260 | + | time.Sleep(20 * time.Millisecond) | |||
| 261 | + | close(hold) | |||
| 262 | + | ||||
| 263 | + | require.NoError(t, <-done) | |||
| 264 | + | assert.Equal(t, limit, exporter.peak) | |||
| 265 | + | assert.Len(t, storage.stored, 3) | |||
| 266 | + | assert.Equal(t, []string{"shop"}, storage.retention) | |||
| 267 | + | }) | |||
| 268 | + | } | |||
| 269 | + | } | |||
| 270 | + | ||||
| 197 | // TestVolumeService_DisabledDoesNothing proves a disabled backup | 271 | // TestVolumeService_DisabledDoesNothing proves a disabled backup | ||
| 198 | // configuration runs nothing at all. | 272 | // configuration runs nothing at all. | ||
| 199 | func TestVolumeService_DisabledDoesNothing(t *testing.T) { | 273 | func TestVolumeService_DisabledDoesNothing(t *testing.T) { | ||
M internal/usecase/cron/scheduler.go +76 -41
| @@ -12,7 +12,8 @@ import ( | |||
| 12 | 12 | "github.com/bnema/zerowrap" | |
| 13 | 13 | ) | |
| 14 | 14 | ||
| 15 | - | // Scheduler runs recurring jobs based on backup schedules. | |
| 15 | + | // Scheduler runs recurring jobs on preset schedules. Times are wall-clock | |
| 16 | + | // times in each schedule's Location; nowFn returns an absolute instant. | |
| 16 | 17 | type Scheduler struct { | |
| 17 | 18 | entries map[string]*entry | |
| 18 | 19 | mu sync.RWMutex | |
| @@ -275,55 +276,89 @@ func (s *Scheduler) dueEntries(now time.Time) []*entry { | |||
| 275 | 276 | return entries | |
| 276 | 277 | } | |
| 277 | 278 | ||
| 278 | - | // calculateNextRun computes the next execution time using UTC-only preset rules. | |
| 279 | - | // | |
| 280 | - | // Current MVP behavior is intentionally fixed to UTC because domain.CronSchedule | |
| 281 | - | // has no timezone field yet: | |
| 282 | - | // - daily: 02:00 UTC | |
| 283 | - | // - weekly: Sunday 03:00 UTC | |
| 284 | - | // - monthly: first day of month 04:00 UTC | |
| 279 | + | // presetDefaultTimes are the built-in times of day used when a schedule | |
| 280 | + | // sets no explicit time. | |
| 281 | + | var presetDefaultTimes = map[domain.BackupSchedule]domain.TimeOfDay{ | |
| 282 | + | domain.ScheduleHourly: {Hour: 0, Minute: 0}, | |
| 283 | + | domain.ScheduleDaily: {Hour: 2, Minute: 0}, | |
| 284 | + | domain.ScheduleWeekly: {Hour: 3, Minute: 0}, | |
| 285 | + | domain.ScheduleMonthly: {Hour: 4, Minute: 0}, | |
| 286 | + | } | |
| 287 | + | ||
| 288 | + | // calculateNextRun returns the first scheduled instant strictly after now. | |
| 285 | 289 | // | |
| 286 | - | // This means users in non-UTC timezones should account for UTC offsets. | |
| 290 | + | // Times are wall-clock times in schedule.Location (UTC when nil), so a | |
| 291 | + | // daily 03:30 stays 03:30 across daylight-saving changes. A wall time that | |
| 292 | + | // does not exist on a given day (spring-forward gap) runs at the end of the | |
| 293 | + | // gap. A wall time that happens twice (fall-back) runs once, at its first | |
| 294 | + | // occurrence; hourly schedules follow real elapsed hours. | |
| 287 | 295 | // | |
| 288 | - | // TODO: add timezone support on domain.CronSchedule once timezone-aware | |
| 289 | - | // scheduling is introduced. Default timezone should be the host system timezone | |
| 290 | - | // when no explicit timezone is configured. | |
| 296 | + | // The result only ever depends on now: a run missed while the process was | |
| 297 | + | // down is skipped, never caught up. | |
| 298 | + | // - hourly: every hour at At.Minute | |
| 299 | + | // - daily: every day at At | |
| 300 | + | // - weekly: every Sunday at At | |
| 301 | + | // - monthly: the first day of every month at At | |
| 291 | 302 | func calculateNextRun(now time.Time, schedule domain.CronSchedule) (time.Time, error) { | |
| 292 | - | now = now.UTC() | |
| 293 | - | if schedule.Interval > 0 && schedule.Preset != "" { | |
| 294 | - | return time.Time{}, fmt.Errorf("cron schedule cannot set both interval and preset") | |
| 303 | + | at, ok := presetDefaultTimes[schedule.Preset] | |
| 304 | + | if !ok { | |
| 305 | + | return time.Time{}, fmt.Errorf("unsupported schedule preset: %q", schedule.Preset) | |
| 306 | + | } | |
| 307 | + | if schedule.At != nil { | |
| 308 | + | at = *schedule.At | |
| 295 | 309 | } | |
| 296 | - | if schedule.Interval > 0 { | |
| 297 | - | return now.Add(schedule.Interval), nil | |
| 310 | + | loc := schedule.Location | |
| 311 | + | if loc == nil { | |
| 312 | + | loc = time.UTC | |
| 298 | 313 | } | |
| 314 | + | local := now.In(loc) | |
| 315 | + | y, m, d := local.Date() | |
| 299 | 316 | ||
| 300 | 317 | switch schedule.Preset { | |
| 301 | 318 | case domain.ScheduleHourly: | |
| 302 | - | next := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.UTC) | |
| 303 | - | if !next.After(now) { | |
| 304 | - | next = next.Add(time.Hour) | |
| 305 | - | } | |
| 306 | - | return next, nil | |
| 319 | + | return nextMinuteOfHour(now, at.Minute, loc), nil | |
| 307 | 320 | case domain.ScheduleDaily: | |
| 308 | - | next := time.Date(now.Year(), now.Month(), now.Day(), 2, 0, 0, 0, time.UTC) | |
| 309 | - | if !next.After(now) { | |
| 310 | - | next = next.AddDate(0, 0, 1) | |
| 311 | - | } | |
| 312 | - | return next, nil | |
| 321 | + | return firstAfter(now, func(i int) time.Time { return wallTime(y, m, d+i, at, loc) }), nil | |
| 313 | 322 | case domain.ScheduleWeekly: | |
| 314 | - | daysUntilSunday := (7 - int(now.Weekday())) % 7 | |
| 315 | - | next := time.Date(now.Year(), now.Month(), now.Day(), 3, 0, 0, 0, time.UTC).AddDate(0, 0, daysUntilSunday) | |
| 316 | - | if !next.After(now) { | |
| 317 | - | next = next.AddDate(0, 0, 7) | |
| 318 | - | } | |
| 319 | - | return next, nil | |
| 320 | - | case domain.ScheduleMonthly: | |
| 321 | - | next := time.Date(now.Year(), now.Month(), 1, 4, 0, 0, 0, time.UTC) | |
| 322 | - | if !next.After(now) { | |
| 323 | - | next = next.AddDate(0, 1, 0) | |
| 324 | - | } | |
| 325 | - | return next, nil | |
| 326 | - | default: | |
| 327 | - | return time.Time{}, fmt.Errorf("unsupported schedule preset: %q", schedule.Preset) | |
| 323 | + | sunday := d + (7-int(local.Weekday()))%7 | |
| 324 | + | return firstAfter(now, func(i int) time.Time { return wallTime(y, m, sunday+7*i, at, loc) }), nil | |
| 325 | + | default: // domain.ScheduleMonthly | |
| 326 | + | return firstAfter(now, func(i int) time.Time { return wallTime(y, m+time.Month(i), 1, at, loc) }), nil | |
| 327 | + | } | |
| 328 | + | } | |
| 329 | + | ||
| 330 | + | // firstAfter returns candidate(0) or candidate(1), whichever is the first | |
| 331 | + | // strictly after now. | |
| 332 | + | func firstAfter(now time.Time, candidate func(i int) time.Time) time.Time { | |
| 333 | + | if next := candidate(0); next.After(now) { | |
| 334 | + | return next | |
| 335 | + | } | |
| 336 | + | return candidate(1) | |
| 337 | + | } | |
| 338 | + | ||
| 339 | + | // nextMinuteOfHour returns the first whole minute strictly after now whose | |
| 340 | + | // wall-clock minute in loc equals minute. Walking real minutes keeps hourly | |
| 341 | + | // runs one hour apart across DST changes. | |
| 342 | + | func nextMinuteOfHour(now time.Time, minute int, loc *time.Location) time.Time { | |
| 343 | + | t := now.Truncate(time.Minute).Add(time.Minute) | |
| 344 | + | for i := 0; i < 120 && t.In(loc).Minute() != minute; i++ { | |
| 345 | + | t = t.Add(time.Minute) | |
| 346 | + | } | |
| 347 | + | return t | |
| 348 | + | } | |
| 349 | + | ||
| 350 | + | // wallTime returns the instant of the given wall-clock time in loc. Day and | |
| 351 | + | // month overflow are normalized like time.Date. When the wall time falls in | |
| 352 | + | // a spring-forward gap, the end of the gap is returned. | |
| 353 | + | func wallTime(year int, month time.Month, day int, at domain.TimeOfDay, loc *time.Location) time.Time { | |
| 354 | + | t := time.Date(year, month, day, at.Hour, at.Minute, 0, 0, loc) | |
| 355 | + | if t.Hour() == at.Hour && t.Minute() == at.Minute { | |
| 356 | + | return t | |
| 328 | 357 | } | |
| 358 | + | // Nonexistent wall time: interpret it with the offset in force before | |
| 359 | + | // the gap, which lands after the transition, then snap back to it. | |
| 360 | + | naive := time.Date(year, month, day, at.Hour, at.Minute, 0, 0, time.UTC) | |
| 361 | + | _, offsetBefore := naive.Add(-12 * time.Hour).In(loc).Zone() | |
| 362 | + | start, _ := naive.Add(-time.Duration(offsetBefore) * time.Second).In(loc).ZoneBounds() | |
| 363 | + | return start.In(loc) | |
| 329 | 364 | } | |
| @@ -12,7 +12,8 @@ import ( | |||||
| 12 | "github.com/bnema/zerowrap" | 12 | "github.com/bnema/zerowrap" | ||
| 13 | ) | 13 | ) | ||
| 14 | 14 | ||||
| 15 | - | // Scheduler runs recurring jobs based on backup schedules. | 15 | + | // Scheduler runs recurring jobs on preset schedules. Times are wall-clock |
| 16 | + | // times in each schedule's Location; nowFn returns an absolute instant. | |||
| 16 | type Scheduler struct { | 17 | type Scheduler struct { | ||
| 17 | entries map[string]*entry | 18 | entries map[string]*entry | ||
| 18 | mu sync.RWMutex | 19 | mu sync.RWMutex | ||
| @@ -275,55 +276,89 @@ func (s *Scheduler) dueEntries(now time.Time) []*entry { | |||||
| 275 | return entries | 276 | return entries | ||
| 276 | } | 277 | } | ||
| 277 | 278 | ||||
| 278 | - | // calculateNextRun computes the next execution time using UTC-only preset rules. | 279 | + | // presetDefaultTimes are the built-in times of day used when a schedule |
| 279 | - | // | 280 | + | // sets no explicit time. |
| 280 | - | // Current MVP behavior is intentionally fixed to UTC because domain.CronSchedule | 281 | + | var presetDefaultTimes = map[domain.BackupSchedule]domain.TimeOfDay{ |
| 281 | - | // has no timezone field yet: | 282 | + | domain.ScheduleHourly: {Hour: 0, Minute: 0}, |
| 282 | - | // - daily: 02:00 UTC | 283 | + | domain.ScheduleDaily: {Hour: 2, Minute: 0}, |
| 283 | - | // - weekly: Sunday 03:00 UTC | 284 | + | domain.ScheduleWeekly: {Hour: 3, Minute: 0}, |
| 284 | - | // - monthly: first day of month 04:00 UTC | 285 | + | domain.ScheduleMonthly: {Hour: 4, Minute: 0}, |
| 286 | + | } | |||
| 287 | + | ||||
| 288 | + | // calculateNextRun returns the first scheduled instant strictly after now. | |||
| 285 | // | 289 | // | ||
| 286 | - | // This means users in non-UTC timezones should account for UTC offsets. | 290 | + | // Times are wall-clock times in schedule.Location (UTC when nil), so a |
| 291 | + | // daily 03:30 stays 03:30 across daylight-saving changes. A wall time that | |||
| 292 | + | // does not exist on a given day (spring-forward gap) runs at the end of the | |||
| 293 | + | // gap. A wall time that happens twice (fall-back) runs once, at its first | |||
| 294 | + | // occurrence; hourly schedules follow real elapsed hours. | |||
| 287 | // | 295 | // | ||
| 288 | - | // TODO: add timezone support on domain.CronSchedule once timezone-aware | 296 | + | // The result only ever depends on now: a run missed while the process was |
| 289 | - | // scheduling is introduced. Default timezone should be the host system timezone | 297 | + | // down is skipped, never caught up. |
| 290 | - | // when no explicit timezone is configured. | 298 | + | // - hourly: every hour at At.Minute |
| 299 | + | // - daily: every day at At | |||
| 300 | + | // - weekly: every Sunday at At | |||
| 301 | + | // - monthly: the first day of every month at At | |||
| 291 | func calculateNextRun(now time.Time, schedule domain.CronSchedule) (time.Time, error) { | 302 | func calculateNextRun(now time.Time, schedule domain.CronSchedule) (time.Time, error) { | ||
| 292 | - | now = now.UTC() | 303 | + | at, ok := presetDefaultTimes[schedule.Preset] |
| 293 | - | if schedule.Interval > 0 && schedule.Preset != "" { | 304 | + | if !ok { |
| 294 | - | return time.Time{}, fmt.Errorf("cron schedule cannot set both interval and preset") | 305 | + | return time.Time{}, fmt.Errorf("unsupported schedule preset: %q", schedule.Preset) |
| 306 | + | } | |||
| 307 | + | if schedule.At != nil { | |||
| 308 | + | at = *schedule.At | |||
| 295 | } | 309 | } | ||
| 296 | - | if schedule.Interval > 0 { | 310 | + | loc := schedule.Location |
| 297 | - | return now.Add(schedule.Interval), nil | 311 | + | if loc == nil { |
| 312 | + | loc = time.UTC | |||
| 298 | } | 313 | } | ||
| 314 | + | local := now.In(loc) | |||
| 315 | + | y, m, d := local.Date() | |||
| 299 | 316 | ||||
| 300 | switch schedule.Preset { | 317 | switch schedule.Preset { | ||
| 301 | case domain.ScheduleHourly: | 318 | case domain.ScheduleHourly: | ||
| 302 | - | next := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, time.UTC) | 319 | + | return nextMinuteOfHour(now, at.Minute, loc), nil |
| 303 | - | if !next.After(now) { | |||
| 304 | - | next = next.Add(time.Hour) | |||
| 305 | - | } | |||
| 306 | - | return next, nil | |||
| 307 | case domain.ScheduleDaily: | 320 | case domain.ScheduleDaily: | ||
| 308 | - | next := time.Date(now.Year(), now.Month(), now.Day(), 2, 0, 0, 0, time.UTC) | 321 | + | return firstAfter(now, func(i int) time.Time { return wallTime(y, m, d+i, at, loc) }), nil |
| 309 | - | if !next.After(now) { | |||
| 310 | - | next = next.AddDate(0, 0, 1) | |||
| 311 | - | } | |||
| 312 | - | return next, nil | |||
| 313 | case domain.ScheduleWeekly: | 322 | case domain.ScheduleWeekly: | ||
| 314 | - | daysUntilSunday := (7 - int(now.Weekday())) % 7 | 323 | + | sunday := d + (7-int(local.Weekday()))%7 |
| 315 | - | next := time.Date(now.Year(), now.Month(), now.Day(), 3, 0, 0, 0, time.UTC).AddDate(0, 0, daysUntilSunday) | 324 | + | return firstAfter(now, func(i int) time.Time { return wallTime(y, m, sunday+7*i, at, loc) }), nil |
| 316 | - | if !next.After(now) { | 325 | + | default: // domain.ScheduleMonthly |
| 317 | - | next = next.AddDate(0, 0, 7) | 326 | + | return firstAfter(now, func(i int) time.Time { return wallTime(y, m+time.Month(i), 1, at, loc) }), nil |
| 318 | - | } | 327 | + | } |
| 319 | - | return next, nil | 328 | + | } |
| 320 | - | case domain.ScheduleMonthly: | 329 | + | |
| 321 | - | next := time.Date(now.Year(), now.Month(), 1, 4, 0, 0, 0, time.UTC) | 330 | + | // firstAfter returns candidate(0) or candidate(1), whichever is the first |
| 322 | - | if !next.After(now) { | 331 | + | // strictly after now. |
| 323 | - | next = next.AddDate(0, 1, 0) | 332 | + | func firstAfter(now time.Time, candidate func(i int) time.Time) time.Time { |
| 324 | - | } | 333 | + | if next := candidate(0); next.After(now) { |
| 325 | - | return next, nil | 334 | + | return next |
| 326 | - | default: | 335 | + | } |
| 327 | - | return time.Time{}, fmt.Errorf("unsupported schedule preset: %q", schedule.Preset) | 336 | + | return candidate(1) |
| 337 | + | } | |||
| 338 | + | ||||
| 339 | + | // nextMinuteOfHour returns the first whole minute strictly after now whose | |||
| 340 | + | // wall-clock minute in loc equals minute. Walking real minutes keeps hourly | |||
| 341 | + | // runs one hour apart across DST changes. | |||
| 342 | + | func nextMinuteOfHour(now time.Time, minute int, loc *time.Location) time.Time { | |||
| 343 | + | t := now.Truncate(time.Minute).Add(time.Minute) | |||
| 344 | + | for i := 0; i < 120 && t.In(loc).Minute() != minute; i++ { | |||
| 345 | + | t = t.Add(time.Minute) | |||
| 346 | + | } | |||
| 347 | + | return t | |||
| 348 | + | } | |||
| 349 | + | ||||
| 350 | + | // wallTime returns the instant of the given wall-clock time in loc. Day and | |||
| 351 | + | // month overflow are normalized like time.Date. When the wall time falls in | |||
| 352 | + | // a spring-forward gap, the end of the gap is returned. | |||
| 353 | + | func wallTime(year int, month time.Month, day int, at domain.TimeOfDay, loc *time.Location) time.Time { | |||
| 354 | + | t := time.Date(year, month, day, at.Hour, at.Minute, 0, 0, loc) | |||
| 355 | + | if t.Hour() == at.Hour && t.Minute() == at.Minute { | |||
| 356 | + | return t | |||
| 328 | } | 357 | } | ||
| 358 | + | // Nonexistent wall time: interpret it with the offset in force before | |||
| 359 | + | // the gap, which lands after the transition, then snap back to it. | |||
| 360 | + | naive := time.Date(year, month, day, at.Hour, at.Minute, 0, 0, time.UTC) | |||
| 361 | + | _, offsetBefore := naive.Add(-12 * time.Hour).In(loc).Zone() | |||
| 362 | + | start, _ := naive.Add(-time.Duration(offsetBefore) * time.Second).In(loc).ZoneBounds() | |||
| 363 | + | return start.In(loc) | |||
| 329 | } | 364 | } | ||
M internal/usecase/cron/scheduler_test.go +88 -7
| @@ -51,20 +51,101 @@ func TestCalculateNextRunPresets(t *testing.T) { | |||
| 51 | 51 | } | |
| 52 | 52 | } | |
| 53 | 53 | ||
| 54 | - | func TestCalculateNextRunInterval(t *testing.T) { | |
| 55 | - | now := time.Date(2026, 2, 7, 12, 34, 20, 0, time.UTC) | |
| 54 | + | func TestCalculateNextRunAtLocalTime(t *testing.T) { | |
| 55 | + | paris, err := time.LoadLocation("Europe/Paris") | |
| 56 | + | require.NoError(t, err) | |
| 57 | + | at := &domain.TimeOfDay{Hour: 3, Minute: 30} | |
| 56 | 58 | ||
| 57 | - | next, err := calculateNextRun(now, domain.CronSchedule{Interval: 6 * time.Hour}) | |
| 59 | + | tests := []struct { | |
| 60 | + | name string | |
| 61 | + | preset domain.BackupSchedule | |
| 62 | + | now time.Time | |
| 63 | + | expected time.Time | |
| 64 | + | }{ | |
| 65 | + | {"daily later today", domain.ScheduleDaily, | |
| 66 | + | time.Date(2026, 10, 17, 1, 0, 0, 0, paris), time.Date(2026, 10, 17, 3, 30, 0, 0, paris)}, | |
| 67 | + | {"daily already passed", domain.ScheduleDaily, | |
| 68 | + | time.Date(2026, 10, 17, 7, 24, 0, 0, paris), time.Date(2026, 10, 18, 3, 30, 0, 0, paris)}, | |
| 69 | + | {"daily exactly now runs tomorrow", domain.ScheduleDaily, | |
| 70 | + | time.Date(2026, 10, 17, 3, 30, 0, 0, paris), time.Date(2026, 10, 18, 3, 30, 0, 0, paris)}, | |
| 71 | + | // 2026-10-25: Paris leaves summer time at 03:00, so 03:30 is CET. | |
| 72 | + | {"daily across DST end keeps wall clock", domain.ScheduleDaily, | |
| 73 | + | time.Date(2026, 10, 24, 12, 0, 0, 0, paris), time.Date(2026, 10, 25, 3, 30, 0, 0, paris)}, | |
| 74 | + | {"weekly next sunday", domain.ScheduleWeekly, | |
| 75 | + | time.Date(2026, 10, 17, 12, 0, 0, 0, paris), time.Date(2026, 10, 18, 3, 30, 0, 0, paris)}, | |
| 76 | + | {"monthly next first day", domain.ScheduleMonthly, | |
| 77 | + | time.Date(2026, 10, 17, 12, 0, 0, 0, paris), time.Date(2026, 11, 1, 3, 30, 0, 0, paris)}, | |
| 78 | + | {"hourly uses minute only", domain.ScheduleHourly, | |
| 79 | + | time.Date(2026, 10, 17, 12, 40, 0, 0, paris), time.Date(2026, 10, 17, 13, 30, 0, 0, paris)}, | |
| 80 | + | } | |
| 81 | + | for _, tt := range tests { | |
| 82 | + | t.Run(tt.name, func(t *testing.T) { | |
| 83 | + | next, err := calculateNextRun(tt.now, domain.CronSchedule{Preset: tt.preset, At: at, Location: paris}) | |
| 84 | + | require.NoError(t, err) | |
| 85 | + | assert.True(t, tt.expected.Equal(next), "want %s, got %s", tt.expected, next.In(paris)) | |
| 86 | + | }) | |
| 87 | + | } | |
| 88 | + | } | |
| 58 | 89 | ||
| 90 | + | func TestCalculateNextRunDSTGapSkipsForward(t *testing.T) { | |
| 91 | + | paris, err := time.LoadLocation("Europe/Paris") | |
| 92 | + | require.NoError(t, err) | |
| 93 | + | // 2026-03-29: Paris skips 02:00-03:00. A 02:30 schedule runs once, at | |
| 94 | + | // the first valid instant after the gap, never twice or never. | |
| 95 | + | now := time.Date(2026, 3, 28, 12, 0, 0, 0, paris) | |
| 96 | + | next, err := calculateNextRun(now, domain.CronSchedule{ | |
| 97 | + | Preset: domain.ScheduleDaily, At: &domain.TimeOfDay{Hour: 2, Minute: 30}, Location: paris, | |
| 98 | + | }) | |
| 59 | 99 | require.NoError(t, err) | |
| 60 | - | assert.Equal(t, now.Add(6*time.Hour), next) | |
| 100 | + | // First valid instant after the gap: 03:00 CEST = 01:00 UTC. | |
| 101 | + | assert.True(t, time.Date(2026, 3, 29, 1, 0, 0, 0, time.UTC).Equal(next), "got %s", next.In(paris)) | |
| 61 | 102 | } | |
| 62 | 103 | ||
| 63 | - | func TestCalculateNextRunRejectsIntervalAndPreset(t *testing.T) { | |
| 64 | - | _, err := calculateNextRun(time.Now(), domain.CronSchedule{Preset: domain.ScheduleDaily, Interval: time.Hour}) | |
| 104 | + | func TestCalculateNextRunDSTEdgeCasesNewYork(t *testing.T) { | |
| 105 | + | ny, err := time.LoadLocation("America/New_York") | |
| 106 | + | require.NoError(t, err) | |
| 107 | + | at := &domain.TimeOfDay{Hour: 2, Minute: 30} | |
| 108 | + | // 2026-03-08: New York jumps from 02:00 EST to 03:00 EDT (07:00 UTC). | |
| 109 | + | gapEnd := time.Date(2026, 3, 8, 7, 0, 0, 0, time.UTC) | |
| 110 | + | // 2026-11-01: 01:00-02:00 happens twice (EDT then EST). | |
| 111 | + | firstOneFortyFive := time.Date(2026, 11, 1, 5, 45, 0, 0, time.UTC) // 01:45 EDT | |
| 112 | + | secondOneFortyFive := time.Date(2026, 11, 1, 6, 45, 0, 0, time.UTC) // 01:45 EST | |
| 113 | + | ||
| 114 | + | tests := []struct { | |
| 115 | + | name string | |
| 116 | + | preset domain.BackupSchedule | |
| 117 | + | at *domain.TimeOfDay | |
| 118 | + | now time.Time | |
| 119 | + | expected time.Time | |
| 120 | + | }{ | |
| 121 | + | {"daily in gap from same morning runs at gap end", domain.ScheduleDaily, at, | |
| 122 | + | time.Date(2026, 3, 8, 6, 45, 0, 0, time.UTC), gapEnd}, // now 01:45 EST | |
| 123 | + | {"daily in gap from previous day runs at gap end", domain.ScheduleDaily, at, | |
| 124 | + | time.Date(2026, 3, 7, 17, 0, 0, 0, time.UTC), gapEnd}, | |
| 125 | + | {"daily after gap end runs next day", domain.ScheduleDaily, at, | |
| 126 | + | gapEnd, time.Date(2026, 3, 9, 6, 30, 0, 0, time.UTC)}, | |
| 127 | + | {"weekly in gap runs at gap end", domain.ScheduleWeekly, at, | |
| 128 | + | time.Date(2026, 3, 7, 17, 0, 0, 0, time.UTC), gapEnd}, | |
| 129 | + | {"hourly in repeated hour, first pass", domain.ScheduleHourly, &domain.TimeOfDay{Minute: 30}, | |
| 130 | + | firstOneFortyFive, time.Date(2026, 11, 1, 6, 30, 0, 0, time.UTC)}, // 01:30 EST | |
| 131 | + | {"hourly in repeated hour, second pass", domain.ScheduleHourly, &domain.TimeOfDay{Minute: 30}, | |
| 132 | + | secondOneFortyFive, time.Date(2026, 11, 1, 7, 30, 0, 0, time.UTC)}, // 02:30 EST | |
| 133 | + | {"daily in repeated hour runs next day", domain.ScheduleDaily, &domain.TimeOfDay{Hour: 1, Minute: 30}, | |
| 134 | + | secondOneFortyFive, time.Date(2026, 11, 2, 6, 30, 0, 0, time.UTC)}, | |
| 135 | + | } | |
| 136 | + | for _, tt := range tests { | |
| 137 | + | t.Run(tt.name, func(t *testing.T) { | |
| 138 | + | next, err := calculateNextRun(tt.now, domain.CronSchedule{Preset: tt.preset, At: tt.at, Location: ny}) | |
| 139 | + | require.NoError(t, err) | |
| 140 | + | assert.True(t, next.After(tt.now), "next %s must be after now %s", next, tt.now) | |
| 141 | + | assert.True(t, tt.expected.Equal(next), "want %s, got %s", tt.expected.In(ny), next.In(ny)) | |
| 142 | + | }) | |
| 143 | + | } | |
| 144 | + | } | |
| 65 | 145 | ||
| 146 | + | func TestCalculateNextRunRejectsMissingPreset(t *testing.T) { | |
| 147 | + | _, err := calculateNextRun(time.Now(), domain.CronSchedule{}) | |
| 66 | 148 | require.Error(t, err) | |
| 67 | - | assert.Contains(t, err.Error(), "both interval and preset") | |
| 68 | 149 | } | |
| 69 | 150 | ||
| 70 | 151 | func TestSchedulerAddListAndRunNow(t *testing.T) { | |
| @@ -51,20 +51,101 @@ func TestCalculateNextRunPresets(t *testing.T) { | |||||
| 51 | } | 51 | } | ||
| 52 | } | 52 | } | ||
| 53 | 53 | ||||
| 54 | - | func TestCalculateNextRunInterval(t *testing.T) { | 54 | + | func TestCalculateNextRunAtLocalTime(t *testing.T) { |
| 55 | - | now := time.Date(2026, 2, 7, 12, 34, 20, 0, time.UTC) | 55 | + | paris, err := time.LoadLocation("Europe/Paris") |
| 56 | + | require.NoError(t, err) | |||
| 57 | + | at := &domain.TimeOfDay{Hour: 3, Minute: 30} | |||
| 56 | 58 | ||||
| 57 | - | next, err := calculateNextRun(now, domain.CronSchedule{Interval: 6 * time.Hour}) | 59 | + | tests := []struct { |
| 60 | + | name string | |||
| 61 | + | preset domain.BackupSchedule | |||
| 62 | + | now time.Time | |||
| 63 | + | expected time.Time | |||
| 64 | + | }{ | |||
| 65 | + | {"daily later today", domain.ScheduleDaily, | |||
| 66 | + | time.Date(2026, 10, 17, 1, 0, 0, 0, paris), time.Date(2026, 10, 17, 3, 30, 0, 0, paris)}, | |||
| 67 | + | {"daily already passed", domain.ScheduleDaily, | |||
| 68 | + | time.Date(2026, 10, 17, 7, 24, 0, 0, paris), time.Date(2026, 10, 18, 3, 30, 0, 0, paris)}, | |||
| 69 | + | {"daily exactly now runs tomorrow", domain.ScheduleDaily, | |||
| 70 | + | time.Date(2026, 10, 17, 3, 30, 0, 0, paris), time.Date(2026, 10, 18, 3, 30, 0, 0, paris)}, | |||
| 71 | + | // 2026-10-25: Paris leaves summer time at 03:00, so 03:30 is CET. | |||
| 72 | + | {"daily across DST end keeps wall clock", domain.ScheduleDaily, | |||
| 73 | + | time.Date(2026, 10, 24, 12, 0, 0, 0, paris), time.Date(2026, 10, 25, 3, 30, 0, 0, paris)}, | |||
| 74 | + | {"weekly next sunday", domain.ScheduleWeekly, | |||
| 75 | + | time.Date(2026, 10, 17, 12, 0, 0, 0, paris), time.Date(2026, 10, 18, 3, 30, 0, 0, paris)}, | |||
| 76 | + | {"monthly next first day", domain.ScheduleMonthly, | |||
| 77 | + | time.Date(2026, 10, 17, 12, 0, 0, 0, paris), time.Date(2026, 11, 1, 3, 30, 0, 0, paris)}, | |||
| 78 | + | {"hourly uses minute only", domain.ScheduleHourly, | |||
| 79 | + | time.Date(2026, 10, 17, 12, 40, 0, 0, paris), time.Date(2026, 10, 17, 13, 30, 0, 0, paris)}, | |||
| 80 | + | } | |||
| 81 | + | for _, tt := range tests { | |||
| 82 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 83 | + | next, err := calculateNextRun(tt.now, domain.CronSchedule{Preset: tt.preset, At: at, Location: paris}) | |||
| 84 | + | require.NoError(t, err) | |||
| 85 | + | assert.True(t, tt.expected.Equal(next), "want %s, got %s", tt.expected, next.In(paris)) | |||
| 86 | + | }) | |||
| 87 | + | } | |||
| 88 | + | } | |||
| 58 | 89 | ||||
| 90 | + | func TestCalculateNextRunDSTGapSkipsForward(t *testing.T) { | |||
| 91 | + | paris, err := time.LoadLocation("Europe/Paris") | |||
| 92 | + | require.NoError(t, err) | |||
| 93 | + | // 2026-03-29: Paris skips 02:00-03:00. A 02:30 schedule runs once, at | |||
| 94 | + | // the first valid instant after the gap, never twice or never. | |||
| 95 | + | now := time.Date(2026, 3, 28, 12, 0, 0, 0, paris) | |||
| 96 | + | next, err := calculateNextRun(now, domain.CronSchedule{ | |||
| 97 | + | Preset: domain.ScheduleDaily, At: &domain.TimeOfDay{Hour: 2, Minute: 30}, Location: paris, | |||
| 98 | + | }) | |||
| 59 | require.NoError(t, err) | 99 | require.NoError(t, err) | ||
| 60 | - | assert.Equal(t, now.Add(6*time.Hour), next) | 100 | + | // First valid instant after the gap: 03:00 CEST = 01:00 UTC. |
| 101 | + | assert.True(t, time.Date(2026, 3, 29, 1, 0, 0, 0, time.UTC).Equal(next), "got %s", next.In(paris)) | |||
| 61 | } | 102 | } | ||
| 62 | 103 | ||||
| 63 | - | func TestCalculateNextRunRejectsIntervalAndPreset(t *testing.T) { | 104 | + | func TestCalculateNextRunDSTEdgeCasesNewYork(t *testing.T) { |
| 64 | - | _, err := calculateNextRun(time.Now(), domain.CronSchedule{Preset: domain.ScheduleDaily, Interval: time.Hour}) | 105 | + | ny, err := time.LoadLocation("America/New_York") |
| 106 | + | require.NoError(t, err) | |||
| 107 | + | at := &domain.TimeOfDay{Hour: 2, Minute: 30} | |||
| 108 | + | // 2026-03-08: New York jumps from 02:00 EST to 03:00 EDT (07:00 UTC). | |||
| 109 | + | gapEnd := time.Date(2026, 3, 8, 7, 0, 0, 0, time.UTC) | |||
| 110 | + | // 2026-11-01: 01:00-02:00 happens twice (EDT then EST). | |||
| 111 | + | firstOneFortyFive := time.Date(2026, 11, 1, 5, 45, 0, 0, time.UTC) // 01:45 EDT | |||
| 112 | + | secondOneFortyFive := time.Date(2026, 11, 1, 6, 45, 0, 0, time.UTC) // 01:45 EST | |||
| 113 | + | ||||
| 114 | + | tests := []struct { | |||
| 115 | + | name string | |||
| 116 | + | preset domain.BackupSchedule | |||
| 117 | + | at *domain.TimeOfDay | |||
| 118 | + | now time.Time | |||
| 119 | + | expected time.Time | |||
| 120 | + | }{ | |||
| 121 | + | {"daily in gap from same morning runs at gap end", domain.ScheduleDaily, at, | |||
| 122 | + | time.Date(2026, 3, 8, 6, 45, 0, 0, time.UTC), gapEnd}, // now 01:45 EST | |||
| 123 | + | {"daily in gap from previous day runs at gap end", domain.ScheduleDaily, at, | |||
| 124 | + | time.Date(2026, 3, 7, 17, 0, 0, 0, time.UTC), gapEnd}, | |||
| 125 | + | {"daily after gap end runs next day", domain.ScheduleDaily, at, | |||
| 126 | + | gapEnd, time.Date(2026, 3, 9, 6, 30, 0, 0, time.UTC)}, | |||
| 127 | + | {"weekly in gap runs at gap end", domain.ScheduleWeekly, at, | |||
| 128 | + | time.Date(2026, 3, 7, 17, 0, 0, 0, time.UTC), gapEnd}, | |||
| 129 | + | {"hourly in repeated hour, first pass", domain.ScheduleHourly, &domain.TimeOfDay{Minute: 30}, | |||
| 130 | + | firstOneFortyFive, time.Date(2026, 11, 1, 6, 30, 0, 0, time.UTC)}, // 01:30 EST | |||
| 131 | + | {"hourly in repeated hour, second pass", domain.ScheduleHourly, &domain.TimeOfDay{Minute: 30}, | |||
| 132 | + | secondOneFortyFive, time.Date(2026, 11, 1, 7, 30, 0, 0, time.UTC)}, // 02:30 EST | |||
| 133 | + | {"daily in repeated hour runs next day", domain.ScheduleDaily, &domain.TimeOfDay{Hour: 1, Minute: 30}, | |||
| 134 | + | secondOneFortyFive, time.Date(2026, 11, 2, 6, 30, 0, 0, time.UTC)}, | |||
| 135 | + | } | |||
| 136 | + | for _, tt := range tests { | |||
| 137 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 138 | + | next, err := calculateNextRun(tt.now, domain.CronSchedule{Preset: tt.preset, At: tt.at, Location: ny}) | |||
| 139 | + | require.NoError(t, err) | |||
| 140 | + | assert.True(t, next.After(tt.now), "next %s must be after now %s", next, tt.now) | |||
| 141 | + | assert.True(t, tt.expected.Equal(next), "want %s, got %s", tt.expected.In(ny), next.In(ny)) | |||
| 142 | + | }) | |||
| 143 | + | } | |||
| 144 | + | } | |||
| 65 | 145 | ||||
| 146 | + | func TestCalculateNextRunRejectsMissingPreset(t *testing.T) { | |||
| 147 | + | _, err := calculateNextRun(time.Now(), domain.CronSchedule{}) | |||
| 66 | require.Error(t, err) | 148 | require.Error(t, err) | ||
| 67 | - | assert.Contains(t, err.Error(), "both interval and preset") | |||
| 68 | } | 149 | } | ||
| 69 | 150 | ||||
| 70 | func TestSchedulerAddListAndRunNow(t *testing.T) { | 151 | func TestSchedulerAddListAndRunNow(t *testing.T) { | ||
M internal/usecase/deployment/deploy_unchanged_test.go +59 -0
| @@ -22,6 +22,8 @@ type unchangedSeed struct { | |||
| 22 | 22 | binds map[int]int | |
| 23 | 23 | desiredEnv map[string]string | |
| 24 | 24 | inhibitedBy string | |
| 25 | + | // mutateDesired alters the desired service after the ACTIVE one is seeded. | |
| 26 | + | mutateDesired func(*domain.AppService) | |
| 25 | 27 | } | |
| 26 | 28 | ||
| 27 | 29 | // seedUnchangedApp seeds an app whose ACTIVE service runs exactly what the | |
| @@ -50,6 +52,9 @@ func seedUnchangedApp(t *testing.T, ctx context.Context, store *appstate.Store, | |||
| 50 | 52 | if seed.desiredEnv != nil { | |
| 51 | 53 | desired.Spec.Env = seed.desiredEnv | |
| 52 | 54 | } | |
| 55 | + | if seed.mutateDesired != nil { | |
| 56 | + | seed.mutateDesired(&desired.Spec.Services[0]) | |
| 57 | + | } | |
| 53 | 58 | seedRevision(t, ctx, store, "intent-1", "rev-0", desired) | |
| 54 | 59 | if seed.inhibitedBy != "" { | |
| 55 | 60 | require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{ | |
| @@ -123,6 +128,60 @@ func TestDeploy_SkipsServiceAlreadyRunningTheSameImage(t *testing.T) { | |||
| 123 | 128 | assert.True(t, active.Converged, "an unchanged deploy still converges ACTIVE on the deployed revision") | |
| 124 | 129 | } | |
| 125 | 130 | ||
| 131 | + | // TestDeploy_KeepsContainerOnBackupOnlyChange proves a change limited to the | |
| 132 | + | // service backup declaration keeps the running container, and that ACTIVE | |
| 133 | + | // records the new declaration and revision so backup target resolution sees it. | |
| 134 | + | func TestDeploy_KeepsContainerOnBackupOnlyChange(t *testing.T) { | |
| 135 | + | svc, runtime, _, ctx, store := unchangedDeployService(t, unchangedSeed{ | |
| 136 | + | binds: map[int]int{8080: 18080}, | |
| 137 | + | mutateDesired: func(s *domain.AppService) { | |
| 138 | + | s.Backup = domain.AppBackup{Volume: []string{"web-data"}} | |
| 139 | + | }, | |
| 140 | + | }) | |
| 141 | + | runtime.EXPECT().InspectContainer(mock.Anything, "c-old"). | |
| 142 | + | Return(&domain.Container{ID: "c-old", Status: string(domain.ContainerStatusRunning)}, nil).Once() | |
| 143 | + | ||
| 144 | + | result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog", Op: "op-backup"}) | |
| 145 | + | require.NoError(t, err) | |
| 146 | + | assert.Equal(t, deployment.ServiceResultUnchanged, result.Services["web"].Result) | |
| 147 | + | assert.Equal(t, "c-old", result.Services["web"].After) | |
| 148 | + | runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything) | |
| 149 | + | runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything) | |
| 150 | + | ||
| 151 | + | active, ok, err := store.LoadActive(ctx, "blog") | |
| 152 | + | require.NoError(t, err) | |
| 153 | + | require.True(t, ok) | |
| 154 | + | web := active.Services["web"] | |
| 155 | + | assert.Equal(t, "c-old", web.Container) | |
| 156 | + | assert.Equal(t, "op-created", web.ActivatedBy) | |
| 157 | + | assert.Equal(t, []string{"web-data"}, web.Spec.Backup.Volume, "ACTIVE records the new backup declaration") | |
| 158 | + | assert.NotEqual(t, "rev-0", web.EffectiveRevision, "ACTIVE records the new effective revision") | |
| 159 | + | assert.Equal(t, active.ConvergedRevision, web.EffectiveRevision) | |
| 160 | + | } | |
| 161 | + | ||
| 162 | + | // TestDeploy_ReplacesWhenRuntimeSettingsChangeAlongsideBackup proves a real | |
| 163 | + | // runtime change still replaces the container even when backup also changed. | |
| 164 | + | func TestDeploy_ReplacesWhenRuntimeSettingsChangeAlongsideBackup(t *testing.T) { | |
| 165 | + | svc, runtime, _, ctx, store := unchangedDeployService(t, unchangedSeed{ | |
| 166 | + | binds: map[int]int{8080: 18080}, | |
| 167 | + | mutateDesired: func(s *domain.AppService) { | |
| 168 | + | s.Backup = domain.AppBackup{Volume: []string{"web-data"}} | |
| 169 | + | s.Command = []string{"serve", "--fast"} | |
| 170 | + | }, | |
| 171 | + | }) | |
| 172 | + | expectReplacement(runtime) | |
| 173 | + | ||
| 174 | + | result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog", Op: "op-backup-cmd"}) | |
| 175 | + | require.NoError(t, err) | |
| 176 | + | assert.Equal(t, "deployed", result.Services["web"].Result) | |
| 177 | + | assert.Equal(t, "c-new", result.Services["web"].After) | |
| 178 | + | ||
| 179 | + | active, ok, err := store.LoadActive(ctx, "blog") | |
| 180 | + | require.NoError(t, err) | |
| 181 | + | require.True(t, ok) | |
| 182 | + | assert.Equal(t, []string{"web-data"}, active.Services["web"].Spec.Backup.Volume) | |
| 183 | + | } | |
| 184 | + | ||
| 126 | 185 | // TestDeploy_ReplacesSameDigestWhenTheServiceIsNotServing proves the skip | |
| 127 | 186 | // never keeps a container that is not fully published: missing routing | |
| 128 | 187 | // binds (a withdrawn generation), a recovery inhibition left by a failed | |
| @@ -22,6 +22,8 @@ type unchangedSeed struct { | |||||
| 22 | binds map[int]int | 22 | binds map[int]int | ||
| 23 | desiredEnv map[string]string | 23 | desiredEnv map[string]string | ||
| 24 | inhibitedBy string | 24 | inhibitedBy string | ||
| 25 | + | // mutateDesired alters the desired service after the ACTIVE one is seeded. | |||
| 26 | + | mutateDesired func(*domain.AppService) | |||
| 25 | } | 27 | } | ||
| 26 | 28 | ||||
| 27 | // seedUnchangedApp seeds an app whose ACTIVE service runs exactly what the | 29 | // seedUnchangedApp seeds an app whose ACTIVE service runs exactly what the | ||
| @@ -50,6 +52,9 @@ func seedUnchangedApp(t *testing.T, ctx context.Context, store *appstate.Store, | |||||
| 50 | if seed.desiredEnv != nil { | 52 | if seed.desiredEnv != nil { | ||
| 51 | desired.Spec.Env = seed.desiredEnv | 53 | desired.Spec.Env = seed.desiredEnv | ||
| 52 | } | 54 | } | ||
| 55 | + | if seed.mutateDesired != nil { | |||
| 56 | + | seed.mutateDesired(&desired.Spec.Services[0]) | |||
| 57 | + | } | |||
| 53 | seedRevision(t, ctx, store, "intent-1", "rev-0", desired) | 58 | seedRevision(t, ctx, store, "intent-1", "rev-0", desired) | ||
| 54 | if seed.inhibitedBy != "" { | 59 | if seed.inhibitedBy != "" { | ||
| 55 | require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{ | 60 | require.NoError(t, store.SaveRecoveryInhibition(ctx, domain.AppRecoveryInhibition{ | ||
| @@ -123,6 +128,60 @@ func TestDeploy_SkipsServiceAlreadyRunningTheSameImage(t *testing.T) { | |||||
| 123 | assert.True(t, active.Converged, "an unchanged deploy still converges ACTIVE on the deployed revision") | 128 | assert.True(t, active.Converged, "an unchanged deploy still converges ACTIVE on the deployed revision") | ||
| 124 | } | 129 | } | ||
| 125 | 130 | ||||
| 131 | + | // TestDeploy_KeepsContainerOnBackupOnlyChange proves a change limited to the | |||
| 132 | + | // service backup declaration keeps the running container, and that ACTIVE | |||
| 133 | + | // records the new declaration and revision so backup target resolution sees it. | |||
| 134 | + | func TestDeploy_KeepsContainerOnBackupOnlyChange(t *testing.T) { | |||
| 135 | + | svc, runtime, _, ctx, store := unchangedDeployService(t, unchangedSeed{ | |||
| 136 | + | binds: map[int]int{8080: 18080}, | |||
| 137 | + | mutateDesired: func(s *domain.AppService) { | |||
| 138 | + | s.Backup = domain.AppBackup{Volume: []string{"web-data"}} | |||
| 139 | + | }, | |||
| 140 | + | }) | |||
| 141 | + | runtime.EXPECT().InspectContainer(mock.Anything, "c-old"). | |||
| 142 | + | Return(&domain.Container{ID: "c-old", Status: string(domain.ContainerStatusRunning)}, nil).Once() | |||
| 143 | + | ||||
| 144 | + | result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog", Op: "op-backup"}) | |||
| 145 | + | require.NoError(t, err) | |||
| 146 | + | assert.Equal(t, deployment.ServiceResultUnchanged, result.Services["web"].Result) | |||
| 147 | + | assert.Equal(t, "c-old", result.Services["web"].After) | |||
| 148 | + | runtime.AssertNotCalled(t, "StopContainer", mock.Anything, mock.Anything, mock.Anything) | |||
| 149 | + | runtime.AssertNotCalled(t, "CreateContainer", mock.Anything, mock.Anything) | |||
| 150 | + | ||||
| 151 | + | active, ok, err := store.LoadActive(ctx, "blog") | |||
| 152 | + | require.NoError(t, err) | |||
| 153 | + | require.True(t, ok) | |||
| 154 | + | web := active.Services["web"] | |||
| 155 | + | assert.Equal(t, "c-old", web.Container) | |||
| 156 | + | assert.Equal(t, "op-created", web.ActivatedBy) | |||
| 157 | + | assert.Equal(t, []string{"web-data"}, web.Spec.Backup.Volume, "ACTIVE records the new backup declaration") | |||
| 158 | + | assert.NotEqual(t, "rev-0", web.EffectiveRevision, "ACTIVE records the new effective revision") | |||
| 159 | + | assert.Equal(t, active.ConvergedRevision, web.EffectiveRevision) | |||
| 160 | + | } | |||
| 161 | + | ||||
| 162 | + | // TestDeploy_ReplacesWhenRuntimeSettingsChangeAlongsideBackup proves a real | |||
| 163 | + | // runtime change still replaces the container even when backup also changed. | |||
| 164 | + | func TestDeploy_ReplacesWhenRuntimeSettingsChangeAlongsideBackup(t *testing.T) { | |||
| 165 | + | svc, runtime, _, ctx, store := unchangedDeployService(t, unchangedSeed{ | |||
| 166 | + | binds: map[int]int{8080: 18080}, | |||
| 167 | + | mutateDesired: func(s *domain.AppService) { | |||
| 168 | + | s.Backup = domain.AppBackup{Volume: []string{"web-data"}} | |||
| 169 | + | s.Command = []string{"serve", "--fast"} | |||
| 170 | + | }, | |||
| 171 | + | }) | |||
| 172 | + | expectReplacement(runtime) | |||
| 173 | + | ||||
| 174 | + | result, err := svc.Deploy(ctx, deployment.DeployInput{App: "blog", Op: "op-backup-cmd"}) | |||
| 175 | + | require.NoError(t, err) | |||
| 176 | + | assert.Equal(t, "deployed", result.Services["web"].Result) | |||
| 177 | + | assert.Equal(t, "c-new", result.Services["web"].After) | |||
| 178 | + | ||||
| 179 | + | active, ok, err := store.LoadActive(ctx, "blog") | |||
| 180 | + | require.NoError(t, err) | |||
| 181 | + | require.True(t, ok) | |||
| 182 | + | assert.Equal(t, []string{"web-data"}, active.Services["web"].Spec.Backup.Volume) | |||
| 183 | + | } | |||
| 184 | + | ||||
| 126 | // TestDeploy_ReplacesSameDigestWhenTheServiceIsNotServing proves the skip | 185 | // TestDeploy_ReplacesSameDigestWhenTheServiceIsNotServing proves the skip | ||
| 127 | // never keeps a container that is not fully published: missing routing | 186 | // never keeps a container that is not fully published: missing routing | ||
| 128 | // binds (a withdrawn generation), a recovery inhibition left by a failed | 187 | // binds (a withdrawn generation), a recovery inhibition left by a failed | ||
M internal/usecase/deployment/execute.go +3 -1
| @@ -489,7 +489,9 @@ func (s *Service) unchangedRefusal(ctx context.Context, app string, p pinnedServ | |||
| 489 | 489 | switch { | |
| 490 | 490 | case len(p.spec.Binds) > 0 || len(p.spec.Devices) > 0: | |
| 491 | 491 | return "host binds or devices" | |
| 492 | - | case !domain.SameAppService(p.spec, eff.Spec): | |
| 492 | + | case !domain.SameAppServiceRuntime(p.spec, eff.Spec): | |
| 493 | + | // Backup declarations never reach the container, so a backup-only | |
| 494 | + | // change keeps it; publishService still records the new spec in ACTIVE. | |
| 493 | 495 | return "spec changed" | |
| 494 | 496 | case !bindsCover(eff, backendPorts(p.spec)): | |
| 495 | 497 | return "backend binds missing" | |
| @@ -489,7 +489,9 @@ func (s *Service) unchangedRefusal(ctx context.Context, app string, p pinnedServ | |||||
| 489 | switch { | 489 | switch { | ||
| 490 | case len(p.spec.Binds) > 0 || len(p.spec.Devices) > 0: | 490 | case len(p.spec.Binds) > 0 || len(p.spec.Devices) > 0: | ||
| 491 | return "host binds or devices" | 491 | return "host binds or devices" | ||
| 492 | - | case !domain.SameAppService(p.spec, eff.Spec): | 492 | + | case !domain.SameAppServiceRuntime(p.spec, eff.Spec): |
| 493 | + | // Backup declarations never reach the container, so a backup-only | |||
| 494 | + | // change keeps it; publishService still records the new spec in ACTIVE. | |||
| 493 | return "spec changed" | 495 | return "spec changed" | ||
| 494 | case !bindsCover(eff, backendPorts(p.spec)): | 496 | case !bindsCover(eff, backendPorts(p.spec)): | ||
| 495 | return "backend binds missing" | 497 | return "backend binds missing" | ||