detached 5c8a62f

Merge pull request #68 from bnema/fix/readiness-and-backup-detect

Improve deploy readiness handling and push UX
commit
5c8a62f98c94784fbd1ca209393071b2e16a7d30
author
bnema · · 2026-02-08 17:19 UTC
committer
GitHub · · 2026-02-08 17:19 UTC
merge of
37ac30f + aa1feb1 · diff against the first parent
signature
unsigned
23 files · +1512 -227
Unified
UnifiedSplit
M docs/config/server.md +24 -0
@@ -69,6 +69,30 @@ With TLS enabled:
6969 - `port` remains available for plain HTTP unless you restrict it externally
7070 - If `tls_cert_file`/`tls_key_file` are empty, Gordon auto-generates a self-signed pair in `{data_dir}/tls/`
7171
72+For rootless user services, do not bind directly to `443`. Use an unprivileged TLS port (for example `8443`) and forward `443` to it at the firewall or edge proxy.
73+
74+```toml
75+[server]
76+tls_enabled = true
77+tls_port = 8443
78+```
79+
80+With firewalld and Tailscale:
81+
82+```bash
83+# Verify tailscale0 is in trusted zone
84+sudo firewall-cmd --get-zone-of-interface=tailscale0
85+
86+# If needed, place tailscale0 in trusted
87+sudo firewall-cmd --permanent --zone=trusted --add-interface=tailscale0
88+sudo firewall-cmd --reload
89+
90+# Forward 443 -> 8443 permanently (IPv4 + IPv6)
91+sudo firewall-cmd --permanent --zone=trusted --add-rich-rule='rule family=ipv4 forward-port port=443 protocol=tcp to-port=8443'
92+sudo firewall-cmd --permanent --zone=trusted --add-rich-rule='rule family=ipv6 forward-port port=443 protocol=tcp to-port=8443'
93+sudo firewall-cmd --reload
94+```
95+
7296 ### Registry Port
7397
7498 The `registry_port` setting controls where Gordon accepts image pushes:
@@ -69,6 +69,30 @@ With TLS enabled:
69 - `port` remains available for plain HTTP unless you restrict it externally69 - `port` remains available for plain HTTP unless you restrict it externally
70 - If `tls_cert_file`/`tls_key_file` are empty, Gordon auto-generates a self-signed pair in `{data_dir}/tls/`70 - If `tls_cert_file`/`tls_key_file` are empty, Gordon auto-generates a self-signed pair in `{data_dir}/tls/`
71 71
72+For rootless user services, do not bind directly to `443`. Use an unprivileged TLS port (for example `8443`) and forward `443` to it at the firewall or edge proxy.
73+
74+```toml
75+[server]
76+tls_enabled = true
77+tls_port = 8443
78+```
79+
80+With firewalld and Tailscale:
81+
82+```bash
83+# Verify tailscale0 is in trusted zone
84+sudo firewall-cmd --get-zone-of-interface=tailscale0
85+
86+# If needed, place tailscale0 in trusted
87+sudo firewall-cmd --permanent --zone=trusted --add-interface=tailscale0
88+sudo firewall-cmd --reload
89+
90+# Forward 443 -> 8443 permanently (IPv4 + IPv6)
91+sudo firewall-cmd --permanent --zone=trusted --add-rich-rule='rule family=ipv4 forward-port port=443 protocol=tcp to-port=8443'
92+sudo firewall-cmd --permanent --zone=trusted --add-rich-rule='rule family=ipv6 forward-port port=443 protocol=tcp to-port=8443'
93+sudo firewall-cmd --reload
94+```
95+
72 ### Registry Port96 ### Registry Port
73 97
74 The `registry_port` setting controls where Gordon accepts image pushes:98 The `registry_port` setting controls where Gordon accepts image pushes:
M internal/adapters/in/cli/push.go +98 -1
@@ -8,6 +8,7 @@ import (
88 "regexp"
99 "strings"
1010
11+ tea "github.com/charmbracelet/bubbletea"
1112 "github.com/spf13/cobra"
1213
1314 "github.com/bnema/gordon/internal/adapters/in/cli/remote"
@@ -236,7 +237,7 @@ func deployAfterPush(ctx context.Context, client *remote.Client, pushDomain stri
236237 }
237238 }
238239
239- result, err := client.Deploy(ctx, pushDomain)
240+ result, err := deployWithSpinner(ctx, client, pushDomain)
240241 if err != nil {
241242 return fmt.Errorf("failed to deploy: %w", err)
242243 }
@@ -248,6 +249,102 @@ func deployAfterPush(ctx context.Context, client *remote.Client, pushDomain stri
248249 return nil
249250 }
250251
252+func deployWithSpinner(ctx context.Context, client *remote.Client, pushDomain string) (*remote.DeployResult, error) {
253+ if !isInteractiveTerminal() {
254+ fmt.Printf("Deploying %s...\n", pushDomain)
255+ return client.Deploy(ctx, pushDomain)
256+ }
257+
258+ done := make(chan deployOutcome, 1)
259+ go func() {
260+ result, err := client.Deploy(ctx, pushDomain)
261+ done <- deployOutcome{result: result, err: err}
262+ }()
263+
264+ model := newDeploySpinnerModel(pushDomain, done)
265+ final, err := tea.NewProgram(model, tea.WithContext(ctx)).Run()
266+ fmt.Print("\r\033[K")
267+ if err != nil {
268+ return nil, err
269+ }
270+
271+ deployModel, ok := final.(deploySpinnerModel)
272+ if !ok {
273+ return nil, fmt.Errorf("spinner exited with unexpected model type %T", final)
274+ }
275+ if !deployModel.finished {
276+ return nil, fmt.Errorf("deploy spinner exited before deploy result was received")
277+ }
278+
279+ return deployModel.outcome.result, deployModel.outcome.err
280+}
281+
282+type deployOutcome struct {
283+ result *remote.DeployResult
284+ err error
285+}
286+
287+type deployDoneMsg deployOutcome
288+
289+type deploySpinnerModel struct {
290+ spinner components.SpinnerModel
291+ done <-chan deployOutcome
292+ outcome deployOutcome
293+ finished bool
294+}
295+
296+func newDeploySpinnerModel(pushDomain string, done <-chan deployOutcome) deploySpinnerModel {
297+ return deploySpinnerModel{
298+ spinner: components.NewSpinner(
299+ components.WithMessage(fmt.Sprintf("Deploying %s...", pushDomain)),
300+ components.WithSpinnerType(components.SpinnerMiniDot),
301+ ),
302+ done: done,
303+ }
304+}
305+
306+func (m deploySpinnerModel) Init() tea.Cmd {
307+ return tea.Batch(m.spinner.Init(), waitForDeployDone(m.done))
308+}
309+
310+func (m deploySpinnerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
311+ switch msg := msg.(type) {
312+ case deployDoneMsg:
313+ m.outcome = deployOutcome(msg)
314+ m.finished = true
315+ return m, tea.Quit
316+ default:
317+ updated, cmd := m.spinner.Update(msg)
318+ spinnerModel, ok := updated.(components.SpinnerModel)
319+ if ok {
320+ m.spinner = spinnerModel
321+ }
322+ return m, cmd
323+ }
324+}
325+
326+func (m deploySpinnerModel) View() string {
327+ return m.spinner.View()
328+}
329+
330+func waitForDeployDone(done <-chan deployOutcome) tea.Cmd {
331+ return func() tea.Msg {
332+ return deployDoneMsg(<-done)
333+ }
334+}
335+
336+func isInteractiveTerminal() bool {
337+ term := os.Getenv("TERM")
338+ if term == "" || term == "dumb" {
339+ return false
340+ }
341+ info, err := os.Stdout.Stat()
342+ if err != nil {
343+ return false
344+ }
345+ return (info.Mode() & os.ModeCharDevice) != 0
346+}
347+
251348 // parseImageRef splits "registry/name:tag" into components.
252349 func parseImageRef(image string) (registry, name, tag string) {
253350 // Handle "reg.bnema.dev/jp-bot:latest" or "reg.bnema.dev/jp-bot"
@@ -8,6 +8,7 @@ import (
8 "regexp"8 "regexp"
9 "strings"9 "strings"
10 10
11+ tea "github.com/charmbracelet/bubbletea"
11 "github.com/spf13/cobra"12 "github.com/spf13/cobra"
12 13
13 "github.com/bnema/gordon/internal/adapters/in/cli/remote"14 "github.com/bnema/gordon/internal/adapters/in/cli/remote"
@@ -236,7 +237,7 @@ func deployAfterPush(ctx context.Context, client *remote.Client, pushDomain stri
236 }237 }
237 }238 }
238 239
239- result, err := client.Deploy(ctx, pushDomain)240+ result, err := deployWithSpinner(ctx, client, pushDomain)
240 if err != nil {241 if err != nil {
241 return fmt.Errorf("failed to deploy: %w", err)242 return fmt.Errorf("failed to deploy: %w", err)
242 }243 }
@@ -248,6 +249,102 @@ func deployAfterPush(ctx context.Context, client *remote.Client, pushDomain stri
248 return nil249 return nil
249 }250 }
250 251
252+func deployWithSpinner(ctx context.Context, client *remote.Client, pushDomain string) (*remote.DeployResult, error) {
253+ if !isInteractiveTerminal() {
254+ fmt.Printf("Deploying %s...\n", pushDomain)
255+ return client.Deploy(ctx, pushDomain)
256+ }
257+
258+ done := make(chan deployOutcome, 1)
259+ go func() {
260+ result, err := client.Deploy(ctx, pushDomain)
261+ done <- deployOutcome{result: result, err: err}
262+ }()
263+
264+ model := newDeploySpinnerModel(pushDomain, done)
265+ final, err := tea.NewProgram(model, tea.WithContext(ctx)).Run()
266+ fmt.Print("\r\033[K")
267+ if err != nil {
268+ return nil, err
269+ }
270+
271+ deployModel, ok := final.(deploySpinnerModel)
272+ if !ok {
273+ return nil, fmt.Errorf("spinner exited with unexpected model type %T", final)
274+ }
275+ if !deployModel.finished {
276+ return nil, fmt.Errorf("deploy spinner exited before deploy result was received")
277+ }
278+
279+ return deployModel.outcome.result, deployModel.outcome.err
280+}
281+
282+type deployOutcome struct {
283+ result *remote.DeployResult
284+ err error
285+}
286+
287+type deployDoneMsg deployOutcome
288+
289+type deploySpinnerModel struct {
290+ spinner components.SpinnerModel
291+ done <-chan deployOutcome
292+ outcome deployOutcome
293+ finished bool
294+}
295+
296+func newDeploySpinnerModel(pushDomain string, done <-chan deployOutcome) deploySpinnerModel {
297+ return deploySpinnerModel{
298+ spinner: components.NewSpinner(
299+ components.WithMessage(fmt.Sprintf("Deploying %s...", pushDomain)),
300+ components.WithSpinnerType(components.SpinnerMiniDot),
301+ ),
302+ done: done,
303+ }
304+}
305+
306+func (m deploySpinnerModel) Init() tea.Cmd {
307+ return tea.Batch(m.spinner.Init(), waitForDeployDone(m.done))
308+}
309+
310+func (m deploySpinnerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
311+ switch msg := msg.(type) {
312+ case deployDoneMsg:
313+ m.outcome = deployOutcome(msg)
314+ m.finished = true
315+ return m, tea.Quit
316+ default:
317+ updated, cmd := m.spinner.Update(msg)
318+ spinnerModel, ok := updated.(components.SpinnerModel)
319+ if ok {
320+ m.spinner = spinnerModel
321+ }
322+ return m, cmd
323+ }
324+}
325+
326+func (m deploySpinnerModel) View() string {
327+ return m.spinner.View()
328+}
329+
330+func waitForDeployDone(done <-chan deployOutcome) tea.Cmd {
331+ return func() tea.Msg {
332+ return deployDoneMsg(<-done)
333+ }
334+}
335+
336+func isInteractiveTerminal() bool {
337+ term := os.Getenv("TERM")
338+ if term == "" || term == "dumb" {
339+ return false
340+ }
341+ info, err := os.Stdout.Stat()
342+ if err != nil {
343+ return false
344+ }
345+ return (info.Mode() & os.ModeCharDevice) != 0
346+}
347+
251 // parseImageRef splits "registry/name:tag" into components.348 // parseImageRef splits "registry/name:tag" into components.
252 func parseImageRef(image string) (registry, name, tag string) {349 func parseImageRef(image string) (registry, name, tag string) {
253 // Handle "reg.bnema.dev/jp-bot:latest" or "reg.bnema.dev/jp-bot"350 // Handle "reg.bnema.dev/jp-bot:latest" or "reg.bnema.dev/jp-bot"
M internal/adapters/out/docker/runtime.go +24 -7
@@ -350,12 +350,13 @@ func (r *Runtime) InspectContainer(ctx context.Context, containerID string) (*do
350350 name := strings.TrimPrefix(resp.Name, "/")
351351
352352 return &domain.Container{
353- ID: resp.ID,
354- Image: resp.Config.Image,
355- Name: name,
356- Status: resp.State.Status,
357- Ports: ports,
358- Labels: resp.Config.Labels,
353+ ID: resp.ID,
354+ Image: resp.Config.Image,
355+ Name: name,
356+ Status: resp.State.Status,
357+ ExitCode: resp.State.ExitCode,
358+ Ports: ports,
359+ Labels: resp.Config.Labels,
359360 }, nil
360361 }
361362
@@ -688,13 +689,29 @@ func (r *Runtime) GetContainerHealthStatus(ctx context.Context, containerID stri
688689 if err != nil {
689690 return "", false, log.WrapErr(err, "failed to inspect container")
690691 }
691- if resp.State == nil || resp.State.Health == nil {
692+
693+ // Detect configured healthchecks from container config rather than relying on
694+ // State.Health presence. Some runtimes can expose an empty State.Health struct
695+ // even when no healthcheck is configured.
696+ hasHealthcheck := hasConfiguredHealthcheck(resp.Config)
697+ if !hasHealthcheck {
692698 return "", false, nil
693699 }
700+ if resp.State == nil || resp.State.Health == nil {
701+ return "", true, nil
702+ }
694703
695704 return resp.State.Health.Status, true, nil
696705 }
697706
707+func hasConfiguredHealthcheck(cfg *container.Config) bool {
708+ if cfg == nil || cfg.Healthcheck == nil || len(cfg.Healthcheck.Test) == 0 {
709+ return false
710+ }
711+ // Docker uses ["NONE"] to explicitly disable healthchecks inherited from base images.
712+ return !strings.EqualFold(strings.TrimSpace(cfg.Healthcheck.Test[0]), "NONE")
713+}
714+
698715 // GetContainerPort gets the host port for a container's internal port.
699716 func (r *Runtime) GetContainerPort(ctx context.Context, containerID string, internalPort int) (int, error) {
700717 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
@@ -350,12 +350,13 @@ func (r *Runtime) InspectContainer(ctx context.Context, containerID string) (*do
350 name := strings.TrimPrefix(resp.Name, "/")350 name := strings.TrimPrefix(resp.Name, "/")
351 351
352 return &domain.Container{352 return &domain.Container{
353- ID: resp.ID,353+ ID: resp.ID,
354- Image: resp.Config.Image,354+ Image: resp.Config.Image,
355- Name: name,355+ Name: name,
356- Status: resp.State.Status,356+ Status: resp.State.Status,
357- Ports: ports,357+ ExitCode: resp.State.ExitCode,
358- Labels: resp.Config.Labels,358+ Ports: ports,
359+ Labels: resp.Config.Labels,
359 }, nil360 }, nil
360 }361 }
361 362
@@ -688,13 +689,29 @@ func (r *Runtime) GetContainerHealthStatus(ctx context.Context, containerID stri
688 if err != nil {689 if err != nil {
689 return "", false, log.WrapErr(err, "failed to inspect container")690 return "", false, log.WrapErr(err, "failed to inspect container")
690 }691 }
691- if resp.State == nil || resp.State.Health == nil {692+
693+ // Detect configured healthchecks from container config rather than relying on
694+ // State.Health presence. Some runtimes can expose an empty State.Health struct
695+ // even when no healthcheck is configured.
696+ hasHealthcheck := hasConfiguredHealthcheck(resp.Config)
697+ if !hasHealthcheck {
692 return "", false, nil698 return "", false, nil
693 }699 }
700+ if resp.State == nil || resp.State.Health == nil {
701+ return "", true, nil
702+ }
694 703
695 return resp.State.Health.Status, true, nil704 return resp.State.Health.Status, true, nil
696 }705 }
697 706
707+func hasConfiguredHealthcheck(cfg *container.Config) bool {
708+ if cfg == nil || cfg.Healthcheck == nil || len(cfg.Healthcheck.Test) == 0 {
709+ return false
710+ }
711+ // Docker uses ["NONE"] to explicitly disable healthchecks inherited from base images.
712+ return !strings.EqualFold(strings.TrimSpace(cfg.Healthcheck.Test[0]), "NONE")
713+}
714+
698 // GetContainerPort gets the host port for a container's internal port.715 // GetContainerPort gets the host port for a container's internal port.
699 func (r *Runtime) GetContainerPort(ctx context.Context, containerID string, internalPort int) (int, error) {716 func (r *Runtime) GetContainerPort(ctx context.Context, containerID string, internalPort int) (int, error) {
700 ctx = zerowrap.CtxWithFields(ctx, map[string]any{717 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
A internal/adapters/out/docker/runtime_health_status_test.go +77 -0
@@ -0,0 +1,77 @@
1+package docker
2+
3+import (
4+ "context"
5+ "net/http"
6+ "net/http/httptest"
7+ "testing"
8+
9+ "github.com/stretchr/testify/assert"
10+ "github.com/stretchr/testify/require"
11+)
12+
13+func TestRuntime_GetContainerHealthStatus_ConfiguredHealthcheck(t *testing.T) {
14+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
15+ assert.Equal(t, http.MethodGet, r.Method)
16+ assert.Equal(t, "/v1.41/containers/test-container/json", r.URL.Path)
17+
18+ w.Header().Set("Content-Type", "application/json")
19+ _, _ = w.Write([]byte(`{
20+ "Id":"test-container",
21+ "State":{"Status":"running","Health":{"Status":"healthy","FailingStreak":0,"Log":[]}},
22+ "Config":{"Healthcheck":{"Test":["CMD-SHELL","curl -f http://localhost/health || exit 1"]}}
23+ }`))
24+ }))
25+ defer server.Close()
26+
27+ runtime := newRuntimeForHTTPServer(t, server)
28+ status, hasHealthcheck, err := runtime.GetContainerHealthStatus(context.Background(), "test-container")
29+
30+ require.NoError(t, err)
31+ assert.True(t, hasHealthcheck)
32+ assert.Equal(t, "healthy", status)
33+}
34+
35+func TestRuntime_GetContainerHealthStatus_NoConfiguredHealthcheckWithStateHealthObject(t *testing.T) {
36+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
37+ assert.Equal(t, http.MethodGet, r.Method)
38+ assert.Equal(t, "/v1.41/containers/test-container/json", r.URL.Path)
39+
40+ w.Header().Set("Content-Type", "application/json")
41+ _, _ = w.Write([]byte(`{
42+ "Id":"test-container",
43+ "State":{"Status":"running","Health":{"Status":"","FailingStreak":0,"Log":null}},
44+ "Config":{"Healthcheck":null}
45+ }`))
46+ }))
47+ defer server.Close()
48+
49+ runtime := newRuntimeForHTTPServer(t, server)
50+ status, hasHealthcheck, err := runtime.GetContainerHealthStatus(context.Background(), "test-container")
51+
52+ require.NoError(t, err)
53+ assert.False(t, hasHealthcheck)
54+ assert.Equal(t, "", status)
55+}
56+
57+func TestRuntime_GetContainerHealthStatus_DisabledHealthcheck(t *testing.T) {
58+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
59+ assert.Equal(t, http.MethodGet, r.Method)
60+ assert.Equal(t, "/v1.41/containers/test-container/json", r.URL.Path)
61+
62+ w.Header().Set("Content-Type", "application/json")
63+ _, _ = w.Write([]byte(`{
64+ "Id":"test-container",
65+ "State":{"Status":"running","Health":{"Status":"starting","FailingStreak":0,"Log":[]}},
66+ "Config":{"Healthcheck":{"Test":["NONE"]}}
67+ }`))
68+ }))
69+ defer server.Close()
70+
71+ runtime := newRuntimeForHTTPServer(t, server)
72+ status, hasHealthcheck, err := runtime.GetContainerHealthStatus(context.Background(), "test-container")
73+
74+ require.NoError(t, err)
75+ assert.False(t, hasHealthcheck)
76+ assert.Equal(t, "", status)
77+}
@@ -0,0 +1,77 @@
1+package docker
2+
3+import (
4+ "context"
5+ "net/http"
6+ "net/http/httptest"
7+ "testing"
8+
9+ "github.com/stretchr/testify/assert"
10+ "github.com/stretchr/testify/require"
11+)
12+
13+func TestRuntime_GetContainerHealthStatus_ConfiguredHealthcheck(t *testing.T) {
14+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
15+ assert.Equal(t, http.MethodGet, r.Method)
16+ assert.Equal(t, "/v1.41/containers/test-container/json", r.URL.Path)
17+
18+ w.Header().Set("Content-Type", "application/json")
19+ _, _ = w.Write([]byte(`{
20+ "Id":"test-container",
21+ "State":{"Status":"running","Health":{"Status":"healthy","FailingStreak":0,"Log":[]}},
22+ "Config":{"Healthcheck":{"Test":["CMD-SHELL","curl -f http://localhost/health || exit 1"]}}
23+ }`))
24+ }))
25+ defer server.Close()
26+
27+ runtime := newRuntimeForHTTPServer(t, server)
28+ status, hasHealthcheck, err := runtime.GetContainerHealthStatus(context.Background(), "test-container")
29+
30+ require.NoError(t, err)
31+ assert.True(t, hasHealthcheck)
32+ assert.Equal(t, "healthy", status)
33+}
34+
35+func TestRuntime_GetContainerHealthStatus_NoConfiguredHealthcheckWithStateHealthObject(t *testing.T) {
36+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
37+ assert.Equal(t, http.MethodGet, r.Method)
38+ assert.Equal(t, "/v1.41/containers/test-container/json", r.URL.Path)
39+
40+ w.Header().Set("Content-Type", "application/json")
41+ _, _ = w.Write([]byte(`{
42+ "Id":"test-container",
43+ "State":{"Status":"running","Health":{"Status":"","FailingStreak":0,"Log":null}},
44+ "Config":{"Healthcheck":null}
45+ }`))
46+ }))
47+ defer server.Close()
48+
49+ runtime := newRuntimeForHTTPServer(t, server)
50+ status, hasHealthcheck, err := runtime.GetContainerHealthStatus(context.Background(), "test-container")
51+
52+ require.NoError(t, err)
53+ assert.False(t, hasHealthcheck)
54+ assert.Equal(t, "", status)
55+}
56+
57+func TestRuntime_GetContainerHealthStatus_DisabledHealthcheck(t *testing.T) {
58+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
59+ assert.Equal(t, http.MethodGet, r.Method)
60+ assert.Equal(t, "/v1.41/containers/test-container/json", r.URL.Path)
61+
62+ w.Header().Set("Content-Type", "application/json")
63+ _, _ = w.Write([]byte(`{
64+ "Id":"test-container",
65+ "State":{"Status":"running","Health":{"Status":"starting","FailingStreak":0,"Log":[]}},
66+ "Config":{"Healthcheck":{"Test":["NONE"]}}
67+ }`))
68+ }))
69+ defer server.Close()
70+
71+ runtime := newRuntimeForHTTPServer(t, server)
72+ status, hasHealthcheck, err := runtime.GetContainerHealthStatus(context.Background(), "test-container")
73+
74+ require.NoError(t, err)
75+ assert.False(t, hasHealthcheck)
76+ assert.Equal(t, "", status)
77+}
M internal/adapters/out/filesystem/backup_storage.go +13 -0
@@ -27,6 +27,8 @@ type BackupStorage struct {
2727
2828 // NewBackupStorage creates a new filesystem backup storage.
2929 func NewBackupStorage(rootDir string, log zerowrap.Logger) (*BackupStorage, error) {
30+ rootDir = expandTilde(rootDir)
31+
3032 if err := os.MkdirAll(rootDir, 0750); err != nil {
3133 return nil, fmt.Errorf("failed to create backup directory: %w", err)
3234 }
@@ -34,6 +36,17 @@ func NewBackupStorage(rootDir string, log zerowrap.Logger) (*BackupStorage, erro
3436 return &BackupStorage{rootDir: rootDir, log: log}, nil
3537 }
3638
39+// expandTilde replaces a leading "~/" with the user's home directory.
40+func expandTilde(path string) string {
41+ if !strings.HasPrefix(path, "~/") {
42+ return path
43+ }
44+ if home, err := os.UserHomeDir(); err == nil {
45+ return filepath.Join(home, path[2:])
46+ }
47+ return path
48+}
49+
3750 // Store saves backup data and returns the absolute storage path.
3851 func (s *BackupStorage) Store(_ context.Context, domainName, dbName string, schedule domain.BackupSchedule, timestamp time.Time, data io.Reader) (string, error) {
3952 domainPart := sanitizeBackupPathComponent(domainName)
@@ -27,6 +27,8 @@ type BackupStorage struct {
27 27
28 // NewBackupStorage creates a new filesystem backup storage.28 // NewBackupStorage creates a new filesystem backup storage.
29 func NewBackupStorage(rootDir string, log zerowrap.Logger) (*BackupStorage, error) {29 func NewBackupStorage(rootDir string, log zerowrap.Logger) (*BackupStorage, error) {
30+ rootDir = expandTilde(rootDir)
31+
30 if err := os.MkdirAll(rootDir, 0750); err != nil {32 if err := os.MkdirAll(rootDir, 0750); err != nil {
31 return nil, fmt.Errorf("failed to create backup directory: %w", err)33 return nil, fmt.Errorf("failed to create backup directory: %w", err)
32 }34 }
@@ -34,6 +36,17 @@ func NewBackupStorage(rootDir string, log zerowrap.Logger) (*BackupStorage, erro
34 return &BackupStorage{rootDir: rootDir, log: log}, nil36 return &BackupStorage{rootDir: rootDir, log: log}, nil
35 }37 }
36 38
39+// expandTilde replaces a leading "~/" with the user's home directory.
40+func expandTilde(path string) string {
41+ if !strings.HasPrefix(path, "~/") {
42+ return path
43+ }
44+ if home, err := os.UserHomeDir(); err == nil {
45+ return filepath.Join(home, path[2:])
46+ }
47+ return path
48+}
49+
37 // Store saves backup data and returns the absolute storage path.50 // Store saves backup data and returns the absolute storage path.
38 func (s *BackupStorage) Store(_ context.Context, domainName, dbName string, schedule domain.BackupSchedule, timestamp time.Time, data io.Reader) (string, error) {51 func (s *BackupStorage) Store(_ context.Context, domainName, dbName string, schedule domain.BackupSchedule, timestamp time.Time, data io.Reader) (string, error) {
39 domainPart := sanitizeBackupPathComponent(domainName)52 domainPart := sanitizeBackupPathComponent(domainName)
M internal/app/run.go +6 -1
@@ -492,7 +492,7 @@ func createServices(ctx context.Context, v *viper.Viper, cfg Config, log zerowra
492492 healthSvc := health.NewService(svc.configSvc, svc.containerSvc, prober, log)
493493
494494 // Create log service for accessing logs via admin API
495- logSvc := logs.NewService(resolveLogFilePath(cfg), svc.containerSvc, svc.runtime, log)
495+ logSvc := logs.NewService(resolveLogFilePath(cfg), cfg.Logging.File.Enabled, svc.containerSvc, svc.runtime, log)
496496
497497 // Create admin handler for admin API
498498 svc.adminHandler = admin.NewHandler(svc.configSvc, svc.authSvc, svc.containerSvc, healthSvc, secretSvc, logSvc, svc.registrySvc, svc.eventBus, log, svc.backupSvc, svc.imageSvc)
@@ -1254,6 +1254,9 @@ func syncAndAutoStart(ctx context.Context, svc *services, log zerowrap.Logger) {
12541254 log.Warn().Err(err).Msg("failed to auto-start containers")
12551255 }
12561256 }
1257+
1258+ // Start background monitor to restart crashed containers.
1259+ svc.containerSvc.StartMonitor(ctx)
12571260 }
12581261
12591262 // createHTTPHandlers creates HTTP handlers with middleware.
@@ -1662,6 +1665,8 @@ func gracefulShutdown(registrySrv, proxySrv *http.Server, containerSvc *containe
16621665 log.Warn().Err(err).Msg("proxy server shutdown error")
16631666 }
16641667
1668+ containerSvc.StopMonitor()
1669+
16651670 if err := containerSvc.Shutdown(shutdownCtx); err != nil {
16661671 log.Warn().Err(err).Msg("error during container shutdown")
16671672 }
@@ -492,7 +492,7 @@ func createServices(ctx context.Context, v *viper.Viper, cfg Config, log zerowra
492 healthSvc := health.NewService(svc.configSvc, svc.containerSvc, prober, log)492 healthSvc := health.NewService(svc.configSvc, svc.containerSvc, prober, log)
493 493
494 // Create log service for accessing logs via admin API494 // Create log service for accessing logs via admin API
495- logSvc := logs.NewService(resolveLogFilePath(cfg), svc.containerSvc, svc.runtime, log)495+ logSvc := logs.NewService(resolveLogFilePath(cfg), cfg.Logging.File.Enabled, svc.containerSvc, svc.runtime, log)
496 496
497 // Create admin handler for admin API497 // Create admin handler for admin API
498 svc.adminHandler = admin.NewHandler(svc.configSvc, svc.authSvc, svc.containerSvc, healthSvc, secretSvc, logSvc, svc.registrySvc, svc.eventBus, log, svc.backupSvc, svc.imageSvc)498 svc.adminHandler = admin.NewHandler(svc.configSvc, svc.authSvc, svc.containerSvc, healthSvc, secretSvc, logSvc, svc.registrySvc, svc.eventBus, log, svc.backupSvc, svc.imageSvc)
@@ -1254,6 +1254,9 @@ func syncAndAutoStart(ctx context.Context, svc *services, log zerowrap.Logger) {
1254 log.Warn().Err(err).Msg("failed to auto-start containers")1254 log.Warn().Err(err).Msg("failed to auto-start containers")
1255 }1255 }
1256 }1256 }
1257+
1258+ // Start background monitor to restart crashed containers.
1259+ svc.containerSvc.StartMonitor(ctx)
1257 }1260 }
1258 1261
1259 // createHTTPHandlers creates HTTP handlers with middleware.1262 // createHTTPHandlers creates HTTP handlers with middleware.
@@ -1662,6 +1665,8 @@ func gracefulShutdown(registrySrv, proxySrv *http.Server, containerSvc *containe
1662 log.Warn().Err(err).Msg("proxy server shutdown error")1665 log.Warn().Err(err).Msg("proxy server shutdown error")
1663 }1666 }
1664 1667
1668+ containerSvc.StopMonitor()
1669+
1665 if err := containerSvc.Shutdown(shutdownCtx); err != nil {1670 if err := containerSvc.Shutdown(shutdownCtx); err != nil {
1666 log.Warn().Err(err).Msg("error during container shutdown")1671 log.Warn().Err(err).Msg("error during container shutdown")
1667 }1672 }
M internal/domain/container.go +7 -6
@@ -4,12 +4,13 @@ package domain
44
55 // Container represents a running container in the system.
66 type Container struct {
7- ID string
8- Image string
9- Name string
10- Status string
11- Ports []int
12- Labels map[string]string
7+ ID string
8+ Image string
9+ Name string
10+ Status string
11+ ExitCode int
12+ Ports []int
13+ Labels map[string]string
1314 }
1415
1516 // NetworkInfo represents network configuration and state.
@@ -4,12 +4,13 @@ package domain
4 4
5 // Container represents a running container in the system.5 // Container represents a running container in the system.
6 type Container struct {6 type Container struct {
7- ID string7+ ID string
8- Image string8+ Image string
9- Name string9+ Name string
10- Status string10+ Status string
11- Ports []int11+ ExitCode int
12- Labels map[string]string12+ Ports []int
13+ Labels map[string]string
13 }14 }
14 15
15 // NetworkInfo represents network configuration and state.16 // NetworkInfo represents network configuration and state.
M internal/domain/event.go +17 -0
@@ -81,3 +81,20 @@ func IsInternalDeploy(ctx context.Context) bool {
8181 func WithInternalDeploy(ctx context.Context) context.Context {
8282 return context.WithValue(ctx, ContextKeyInternalDeploy, true)
8383 }
84+
85+const (
86+ // ContextKeySkipReadiness indicates that readiness checks should be skipped
87+ // (e.g., during AutoStart where the background monitor handles crash recovery).
88+ ContextKeySkipReadiness contextKey = "skip_readiness"
89+)
90+
91+// WithSkipReadiness returns a context that skips readiness checks on deploy.
92+func WithSkipReadiness(ctx context.Context) context.Context {
93+ return context.WithValue(ctx, ContextKeySkipReadiness, true)
94+}
95+
96+// IsSkipReadiness checks if the context indicates readiness should be skipped.
97+func IsSkipReadiness(ctx context.Context) bool {
98+ v, ok := ctx.Value(ContextKeySkipReadiness).(bool)
99+ return ok && v
100+}
@@ -81,3 +81,20 @@ func IsInternalDeploy(ctx context.Context) bool {
81 func WithInternalDeploy(ctx context.Context) context.Context {81 func WithInternalDeploy(ctx context.Context) context.Context {
82 return context.WithValue(ctx, ContextKeyInternalDeploy, true)82 return context.WithValue(ctx, ContextKeyInternalDeploy, true)
83 }83 }
84+
85+const (
86+ // ContextKeySkipReadiness indicates that readiness checks should be skipped
87+ // (e.g., during AutoStart where the background monitor handles crash recovery).
88+ ContextKeySkipReadiness contextKey = "skip_readiness"
89+)
90+
91+// WithSkipReadiness returns a context that skips readiness checks on deploy.
92+func WithSkipReadiness(ctx context.Context) context.Context {
93+ return context.WithValue(ctx, ContextKeySkipReadiness, true)
94+}
95+
96+// IsSkipReadiness checks if the context indicates readiness should be skipped.
97+func IsSkipReadiness(ctx context.Context) bool {
98+ v, ok := ctx.Value(ContextKeySkipReadiness).(bool)
99+ return ok && v
100+}
M internal/usecase/backup/detector.go +49 -27
@@ -1,49 +1,71 @@
11 package backup
22
33 import (
4+ "slices"
45 "strings"
56
67 "github.com/bnema/gordon/internal/domain"
78 )
89
10+// postgresAliases contains substrings that identify a PostgreSQL image or service.
11+var postgresAliases = []string{"postgres", "postgresql", "pgsql", "postgis"}
12+
913 // detectDatabaseFromAttachment maps a container attachment to DB metadata.
1014 func detectDatabaseFromAttachment(domainName string, a domain.Attachment) (domain.DBInfo, bool) {
1115 image := strings.ToLower(a.Image)
1216 name := strings.ToLower(a.Name)
1317
1418 switch {
15- case strings.Contains(image, "postgres"), strings.Contains(name, "postgres"):
16- port := 5432
17- hasPostgresPort := false
18- firstPositive := 0
19- for _, p := range a.Ports {
20- if p == 5432 {
21- port = 5432
22- hasPostgresPort = true
23- break
24- }
25- if p > 0 && firstPositive == 0 {
26- firstPositive = p
27- }
28- }
29- if !hasPostgresPort && firstPositive > 0 {
30- port = firstPositive
31- }
32- return domain.DBInfo{
33- Type: domain.DBTypePostgreSQL,
34- Version: postgresVersionFromImage(a.Image),
35- Domain: domainName,
36- Name: a.Name,
37- Host: a.Name,
38- Port: port,
39- ContainerID: a.ContainerID,
40- ImageName: a.Image,
41- }, true
19+ case looksLikePostgres(image) || looksLikePostgres(name):
20+ return buildPostgresInfo(domainName, a), true
21+ case hasPort(a.Ports, 5432):
22+ // Port 5432 fallback: likely PostgreSQL even with non-standard naming.
23+ return buildPostgresInfo(domainName, a), true
4224 default:
4325 return domain.DBInfo{}, false
4426 }
4527 }
4628
29+// looksLikePostgres returns true if s contains any known PostgreSQL alias.
30+func looksLikePostgres(s string) bool {
31+ for _, alias := range postgresAliases {
32+ if strings.Contains(s, alias) {
33+ return true
34+ }
35+ }
36+ return false
37+}
38+
39+// hasPort returns true if the port list contains p.
40+func hasPort(ports []int, p int) bool {
41+ return slices.Contains(ports, p)
42+}
43+
44+// buildPostgresInfo constructs a DBInfo for a PostgreSQL attachment.
45+func buildPostgresInfo(domainName string, a domain.Attachment) domain.DBInfo {
46+ port := 5432
47+ for _, p := range a.Ports {
48+ if p == 5432 {
49+ port = 5432
50+ break
51+ }
52+ if p > 0 && port == 5432 {
53+ port = p
54+ }
55+ }
56+
57+ return domain.DBInfo{
58+ Type: domain.DBTypePostgreSQL,
59+ Version: postgresVersionFromImage(a.Image),
60+ Domain: domainName,
61+ Name: a.Name,
62+ Host: a.Name,
63+ Port: port,
64+ ContainerID: a.ContainerID,
65+ ImageName: a.Image,
66+ }
67+}
68+
4769 func postgresVersionFromImage(image string) string {
4870 lastColon := strings.LastIndex(image, ":")
4971 if lastColon == -1 {
@@ -1,49 +1,71 @@
1 package backup1 package backup
2 2
3 import (3 import (
4+ "slices"
4 "strings"5 "strings"
5 6
6 "github.com/bnema/gordon/internal/domain"7 "github.com/bnema/gordon/internal/domain"
7 )8 )
8 9
10+// postgresAliases contains substrings that identify a PostgreSQL image or service.
11+var postgresAliases = []string{"postgres", "postgresql", "pgsql", "postgis"}
12+
9 // detectDatabaseFromAttachment maps a container attachment to DB metadata.13 // detectDatabaseFromAttachment maps a container attachment to DB metadata.
10 func detectDatabaseFromAttachment(domainName string, a domain.Attachment) (domain.DBInfo, bool) {14 func detectDatabaseFromAttachment(domainName string, a domain.Attachment) (domain.DBInfo, bool) {
11 image := strings.ToLower(a.Image)15 image := strings.ToLower(a.Image)
12 name := strings.ToLower(a.Name)16 name := strings.ToLower(a.Name)
13 17
14 switch {18 switch {
15- case strings.Contains(image, "postgres"), strings.Contains(name, "postgres"):19+ case looksLikePostgres(image) || looksLikePostgres(name):
16- port := 543220+ return buildPostgresInfo(domainName, a), true
17- hasPostgresPort := false21+ case hasPort(a.Ports, 5432):
18- firstPositive := 022+ // Port 5432 fallback: likely PostgreSQL even with non-standard naming.
19- for _, p := range a.Ports {23+ return buildPostgresInfo(domainName, a), true
20- if p == 5432 {
21- port = 5432
22- hasPostgresPort = true
23- break
24- }
25- if p > 0 && firstPositive == 0 {
26- firstPositive = p
27- }
28- }
29- if !hasPostgresPort && firstPositive > 0 {
30- port = firstPositive
31- }
32- return domain.DBInfo{
33- Type: domain.DBTypePostgreSQL,
34- Version: postgresVersionFromImage(a.Image),
35- Domain: domainName,
36- Name: a.Name,
37- Host: a.Name,
38- Port: port,
39- ContainerID: a.ContainerID,
40- ImageName: a.Image,
41- }, true
42 default:24 default:
43 return domain.DBInfo{}, false25 return domain.DBInfo{}, false
44 }26 }
45 }27 }
46 28
29+// looksLikePostgres returns true if s contains any known PostgreSQL alias.
30+func looksLikePostgres(s string) bool {
31+ for _, alias := range postgresAliases {
32+ if strings.Contains(s, alias) {
33+ return true
34+ }
35+ }
36+ return false
37+}
38+
39+// hasPort returns true if the port list contains p.
40+func hasPort(ports []int, p int) bool {
41+ return slices.Contains(ports, p)
42+}
43+
44+// buildPostgresInfo constructs a DBInfo for a PostgreSQL attachment.
45+func buildPostgresInfo(domainName string, a domain.Attachment) domain.DBInfo {
46+ port := 5432
47+ for _, p := range a.Ports {
48+ if p == 5432 {
49+ port = 5432
50+ break
51+ }
52+ if p > 0 && port == 5432 {
53+ port = p
54+ }
55+ }
56+
57+ return domain.DBInfo{
58+ Type: domain.DBTypePostgreSQL,
59+ Version: postgresVersionFromImage(a.Image),
60+ Domain: domainName,
61+ Name: a.Name,
62+ Host: a.Name,
63+ Port: port,
64+ ContainerID: a.ContainerID,
65+ ImageName: a.Image,
66+ }
67+}
68+
47 func postgresVersionFromImage(image string) string {69 func postgresVersionFromImage(image string) string {
48 lastColon := strings.LastIndex(image, ":")70 lastColon := strings.LastIndex(image, ":")
49 if lastColon == -1 {71 if lastColon == -1 {
M internal/usecase/backup/detector_test.go +102 -0
@@ -76,3 +76,105 @@ func TestDetectDatabaseFromAttachment_NonPostgres(t *testing.T) {
7676 })
7777 assert.False(t, ok)
7878 }
79+
80+func TestDetectDatabaseFromAttachment_PgsqlImage(t *testing.T) {
81+ db, ok := detectDatabaseFromAttachment("app.example.com", domain.Attachment{
82+ Name: "pgsql",
83+ Image: "pgsql:16",
84+ ContainerID: "pgsql-1",
85+ Ports: []int{5432},
86+ })
87+
88+ require.True(t, ok)
89+ assert.Equal(t, domain.DBTypePostgreSQL, db.Type)
90+ assert.Equal(t, "16", db.Version)
91+ assert.Equal(t, 5432, db.Port)
92+}
93+
94+func TestDetectDatabaseFromAttachment_PgsqlInCompositeName(t *testing.T) {
95+ db, ok := detectDatabaseFromAttachment("mon-fauteuil.fr", domain.Attachment{
96+ Name: "mon-fauteuil-prod-pgsql",
97+ Image: "mon-fauteuil-prod-pgsql:v18",
98+ ContainerID: "pgsql-2",
99+ Ports: []int{5432},
100+ })
101+
102+ require.True(t, ok)
103+ assert.Equal(t, domain.DBTypePostgreSQL, db.Type)
104+ assert.Equal(t, "", db.Version) // v18 tag starts with non-digit, no version extracted
105+}
106+
107+func TestDetectDatabaseFromAttachment_PostgresqlFullName(t *testing.T) {
108+ db, ok := detectDatabaseFromAttachment("app.example.com", domain.Attachment{
109+ Name: "postgresql",
110+ Image: "postgresql:15",
111+ ContainerID: "pg-full-1",
112+ Ports: []int{5432},
113+ })
114+
115+ require.True(t, ok)
116+ assert.Equal(t, domain.DBTypePostgreSQL, db.Type)
117+ assert.Equal(t, "15", db.Version)
118+}
119+
120+func TestDetectDatabaseFromAttachment_PostgisImage(t *testing.T) {
121+ db, ok := detectDatabaseFromAttachment("app.example.com", domain.Attachment{
122+ Name: "postgis",
123+ Image: "postgis/postgis:16-3.4",
124+ ContainerID: "postgis-1",
125+ Ports: []int{5432},
126+ })
127+
128+ require.True(t, ok)
129+ assert.Equal(t, domain.DBTypePostgreSQL, db.Type)
130+ assert.Equal(t, "16", db.Version)
131+}
132+
133+func TestDetectDatabaseFromAttachment_Port5432Fallback(t *testing.T) {
134+ db, ok := detectDatabaseFromAttachment("app.example.com", domain.Attachment{
135+ Name: "custom-db",
136+ Image: "custom-db:latest",
137+ ContainerID: "fallback-1",
138+ Ports: []int{5432},
139+ })
140+
141+ require.True(t, ok, "port 5432 should trigger PostgreSQL fallback detection")
142+ assert.Equal(t, domain.DBTypePostgreSQL, db.Type)
143+ assert.Equal(t, 5432, db.Port)
144+}
145+
146+func TestDetectDatabaseFromAttachment_UnknownImageNoPostgresPort(t *testing.T) {
147+ _, ok := detectDatabaseFromAttachment("app.example.com", domain.Attachment{
148+ Name: "custom-db",
149+ Image: "custom-db:latest",
150+ ContainerID: "unknown-1",
151+ Ports: []int{3306},
152+ })
153+ assert.False(t, ok, "non-postgres port with unknown image should not match")
154+}
155+
156+func TestLooksLikePostgres(t *testing.T) {
157+ tests := []struct {
158+ name string
159+ image string
160+ want bool
161+ }{
162+ {name: "postgres base image", image: "postgres", want: true},
163+ {name: "postgres with version tag", image: "postgres:16", want: true},
164+ {name: "name contains postgres", image: "my-postgres-db", want: true},
165+ {name: "postgresql alias", image: "postgresql", want: true},
166+ {name: "pgsql alias", image: "pgsql", want: true},
167+ {name: "project specific pgsql alias", image: "mon-fauteuil-prod-pgsql", want: true},
168+ {name: "postgis alias", image: "postgis/postgis", want: true},
169+ {name: "mysql negative", image: "mysql", want: false},
170+ {name: "redis negative", image: "redis", want: false},
171+ {name: "custom negative", image: "custom-db", want: false},
172+ {name: "empty negative", image: "", want: false},
173+ }
174+
175+ for _, tc := range tests {
176+ t.Run(tc.name, func(t *testing.T) {
177+ assert.Equal(t, tc.want, looksLikePostgres(tc.image))
178+ })
179+ }
180+}
@@ -76,3 +76,105 @@ func TestDetectDatabaseFromAttachment_NonPostgres(t *testing.T) {
76 })76 })
77 assert.False(t, ok)77 assert.False(t, ok)
78 }78 }
79+
80+func TestDetectDatabaseFromAttachment_PgsqlImage(t *testing.T) {
81+ db, ok := detectDatabaseFromAttachment("app.example.com", domain.Attachment{
82+ Name: "pgsql",
83+ Image: "pgsql:16",
84+ ContainerID: "pgsql-1",
85+ Ports: []int{5432},
86+ })
87+
88+ require.True(t, ok)
89+ assert.Equal(t, domain.DBTypePostgreSQL, db.Type)
90+ assert.Equal(t, "16", db.Version)
91+ assert.Equal(t, 5432, db.Port)
92+}
93+
94+func TestDetectDatabaseFromAttachment_PgsqlInCompositeName(t *testing.T) {
95+ db, ok := detectDatabaseFromAttachment("mon-fauteuil.fr", domain.Attachment{
96+ Name: "mon-fauteuil-prod-pgsql",
97+ Image: "mon-fauteuil-prod-pgsql:v18",
98+ ContainerID: "pgsql-2",
99+ Ports: []int{5432},
100+ })
101+
102+ require.True(t, ok)
103+ assert.Equal(t, domain.DBTypePostgreSQL, db.Type)
104+ assert.Equal(t, "", db.Version) // v18 tag starts with non-digit, no version extracted
105+}
106+
107+func TestDetectDatabaseFromAttachment_PostgresqlFullName(t *testing.T) {
108+ db, ok := detectDatabaseFromAttachment("app.example.com", domain.Attachment{
109+ Name: "postgresql",
110+ Image: "postgresql:15",
111+ ContainerID: "pg-full-1",
112+ Ports: []int{5432},
113+ })
114+
115+ require.True(t, ok)
116+ assert.Equal(t, domain.DBTypePostgreSQL, db.Type)
117+ assert.Equal(t, "15", db.Version)
118+}
119+
120+func TestDetectDatabaseFromAttachment_PostgisImage(t *testing.T) {
121+ db, ok := detectDatabaseFromAttachment("app.example.com", domain.Attachment{
122+ Name: "postgis",
123+ Image: "postgis/postgis:16-3.4",
124+ ContainerID: "postgis-1",
125+ Ports: []int{5432},
126+ })
127+
128+ require.True(t, ok)
129+ assert.Equal(t, domain.DBTypePostgreSQL, db.Type)
130+ assert.Equal(t, "16", db.Version)
131+}
132+
133+func TestDetectDatabaseFromAttachment_Port5432Fallback(t *testing.T) {
134+ db, ok := detectDatabaseFromAttachment("app.example.com", domain.Attachment{
135+ Name: "custom-db",
136+ Image: "custom-db:latest",
137+ ContainerID: "fallback-1",
138+ Ports: []int{5432},
139+ })
140+
141+ require.True(t, ok, "port 5432 should trigger PostgreSQL fallback detection")
142+ assert.Equal(t, domain.DBTypePostgreSQL, db.Type)
143+ assert.Equal(t, 5432, db.Port)
144+}
145+
146+func TestDetectDatabaseFromAttachment_UnknownImageNoPostgresPort(t *testing.T) {
147+ _, ok := detectDatabaseFromAttachment("app.example.com", domain.Attachment{
148+ Name: "custom-db",
149+ Image: "custom-db:latest",
150+ ContainerID: "unknown-1",
151+ Ports: []int{3306},
152+ })
153+ assert.False(t, ok, "non-postgres port with unknown image should not match")
154+}
155+
156+func TestLooksLikePostgres(t *testing.T) {
157+ tests := []struct {
158+ name string
159+ image string
160+ want bool
161+ }{
162+ {name: "postgres base image", image: "postgres", want: true},
163+ {name: "postgres with version tag", image: "postgres:16", want: true},
164+ {name: "name contains postgres", image: "my-postgres-db", want: true},
165+ {name: "postgresql alias", image: "postgresql", want: true},
166+ {name: "pgsql alias", image: "pgsql", want: true},
167+ {name: "project specific pgsql alias", image: "mon-fauteuil-prod-pgsql", want: true},
168+ {name: "postgis alias", image: "postgis/postgis", want: true},
169+ {name: "mysql negative", image: "mysql", want: false},
170+ {name: "redis negative", image: "redis", want: false},
171+ {name: "custom negative", image: "custom-db", want: false},
172+ {name: "empty negative", image: "", want: false},
173+ }
174+
175+ for _, tc := range tests {
176+ t.Run(tc.name, func(t *testing.T) {
177+ assert.Equal(t, tc.want, looksLikePostgres(tc.image))
178+ })
179+ }
180+}
M internal/usecase/backup/service.go +2 -6
@@ -291,18 +291,14 @@ func selectDatabase(dbs []domain.DBInfo, requested string) (domain.DBInfo, error
291291 return domain.DBInfo{}, fmt.Errorf("database %q not found for domain", requested)
292292 }
293293
294-func postgresDumpCommand(dbName string) string {
295- return fmt.Sprintf("PGDATABASE=%s pg_dump -Fc --dbname=\"$PGDATABASE\" --username=\"${POSTGRES_USER:-postgres}\"", shellQuote(dbName))
294+func postgresDumpCommand(_ string) string {
295+ return "pg_dump -Fc --dbname=\"${POSTGRES_DB:-postgres}\" --username=\"${POSTGRES_USER:-postgres}\""
296296 }
297297
298298 func pgDumpToPathCommand(path, dbName string) string {
299299 return fmt.Sprintf("%s > %q", postgresDumpCommand(dbName), path)
300300 }
301301
302-func shellQuote(value string) string {
303- return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
304-}
305-
306302 func (s *Service) cleanupDumpFile(containerID, dumpPath string) {
307303 cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
308304 defer cancel()
@@ -291,18 +291,14 @@ func selectDatabase(dbs []domain.DBInfo, requested string) (domain.DBInfo, error
291 return domain.DBInfo{}, fmt.Errorf("database %q not found for domain", requested)291 return domain.DBInfo{}, fmt.Errorf("database %q not found for domain", requested)
292 }292 }
293 293
294-func postgresDumpCommand(dbName string) string {294+func postgresDumpCommand(_ string) string {
295- return fmt.Sprintf("PGDATABASE=%s pg_dump -Fc --dbname=\"$PGDATABASE\" --username=\"${POSTGRES_USER:-postgres}\"", shellQuote(dbName))295+ return "pg_dump -Fc --dbname=\"${POSTGRES_DB:-postgres}\" --username=\"${POSTGRES_USER:-postgres}\""
296 }296 }
297 297
298 func pgDumpToPathCommand(path, dbName string) string {298 func pgDumpToPathCommand(path, dbName string) string {
299 return fmt.Sprintf("%s > %q", postgresDumpCommand(dbName), path)299 return fmt.Sprintf("%s > %q", postgresDumpCommand(dbName), path)
300 }300 }
301 301
302-func shellQuote(value string) string {
303- return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
304-}
305-
306 func (s *Service) cleanupDumpFile(containerID, dumpPath string) {302 func (s *Service) cleanupDumpFile(containerID, dumpPath string) {
307 cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)303 cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
308 defer cancel()304 defer cancel()
M internal/usecase/backup/service_test.go +5 -4
@@ -50,7 +50,7 @@ func TestService_RunBackup_Postgres(t *testing.T) {
5050 return false
5151 }
5252 return bytes.Contains([]byte(cmd[2]), []byte("pg_dump -Fc")) &&
53- bytes.Contains([]byte(cmd[2]), []byte("PGDATABASE='postgres'")) &&
53+ bytes.Contains([]byte(cmd[2]), []byte("${POSTGRES_DB:-postgres}")) &&
5454 bytes.Contains([]byte(cmd[2]), []byte(" > "))
5555 })).Return(&outiface.ExecResult{ExitCode: 0, Stdout: []byte("backup-data")}, nil)
5656
@@ -125,9 +125,10 @@ func TestSelectDatabaseAutoSelectsOnlyDatabaseWhenUnspecified(t *testing.T) {
125125 assert.Equal(t, "postgres", db.Name)
126126 }
127127
128-func TestPostgresDumpCommandQuotesDatabaseName(t *testing.T) {
129- cmd := postgresDumpCommand("customer data")
130- assert.Contains(t, cmd, "PGDATABASE='customer data'")
128+func TestPostgresDumpCommandUsesEnvVar(t *testing.T) {
129+ cmd := postgresDumpCommand("ignored")
130+ assert.Contains(t, cmd, "${POSTGRES_DB:-postgres}")
131+ assert.Contains(t, cmd, "${POSTGRES_USER:-postgres}")
131132 }
132133
133134 func TestServiceStatusReturnsWhenContextCancelledDuringSemaphoreAcquire(t *testing.T) {
@@ -50,7 +50,7 @@ func TestService_RunBackup_Postgres(t *testing.T) {
50 return false50 return false
51 }51 }
52 return bytes.Contains([]byte(cmd[2]), []byte("pg_dump -Fc")) &&52 return bytes.Contains([]byte(cmd[2]), []byte("pg_dump -Fc")) &&
53- bytes.Contains([]byte(cmd[2]), []byte("PGDATABASE='postgres'")) &&53+ bytes.Contains([]byte(cmd[2]), []byte("${POSTGRES_DB:-postgres}")) &&
54 bytes.Contains([]byte(cmd[2]), []byte(" > "))54 bytes.Contains([]byte(cmd[2]), []byte(" > "))
55 })).Return(&outiface.ExecResult{ExitCode: 0, Stdout: []byte("backup-data")}, nil)55 })).Return(&outiface.ExecResult{ExitCode: 0, Stdout: []byte("backup-data")}, nil)
56 56
@@ -125,9 +125,10 @@ func TestSelectDatabaseAutoSelectsOnlyDatabaseWhenUnspecified(t *testing.T) {
125 assert.Equal(t, "postgres", db.Name)125 assert.Equal(t, "postgres", db.Name)
126 }126 }
127 127
128-func TestPostgresDumpCommandQuotesDatabaseName(t *testing.T) {128+func TestPostgresDumpCommandUsesEnvVar(t *testing.T) {
129- cmd := postgresDumpCommand("customer data")129+ cmd := postgresDumpCommand("ignored")
130- assert.Contains(t, cmd, "PGDATABASE='customer data'")130+ assert.Contains(t, cmd, "${POSTGRES_DB:-postgres}")
131+ assert.Contains(t, cmd, "${POSTGRES_USER:-postgres}")
131 }132 }
132 133
133 func TestServiceStatusReturnsWhenContextCancelledDuringSemaphoreAcquire(t *testing.T) {134 func TestServiceStatusReturnsWhenContextCancelledDuringSemaphoreAcquire(t *testing.T) {
A internal/usecase/container/monitor.go +239 -0
@@ -0,0 +1,239 @@
1+package container
2+
3+import (
4+ "context"
5+ "sync"
6+ "time"
7+
8+ "github.com/bnema/zerowrap"
9+
10+ "github.com/bnema/gordon/internal/domain"
11+)
12+
13+const (
14+ monitorDefaultInterval = 15 * time.Second
15+ crashLoopThreshold = 3
16+ crashLoopWindow = 5 * time.Minute
17+ backoffCap = 15 * time.Minute
18+ stableRunningDuration = 5 * time.Minute
19+)
20+
21+// restartRecord tracks restart attempts for crash loop detection.
22+type restartRecord struct {
23+ attempts []time.Time
24+ backoffEnd time.Time
25+ consecutive int
26+ lastSeen time.Time // last time the container was seen running
27+}
28+
29+// Monitor watches tracked containers and restarts crashed ones.
30+type Monitor struct {
31+ service *Service
32+ stopCh chan struct{}
33+ stopped chan struct{}
34+ interval time.Duration
35+ mu sync.Mutex
36+ history map[string]*restartRecord // keyed by domain
37+}
38+
39+// newMonitor creates a new container monitor.
40+func newMonitor(service *Service) *Monitor {
41+ return &Monitor{
42+ service: service,
43+ stopCh: make(chan struct{}),
44+ stopped: make(chan struct{}),
45+ interval: monitorDefaultInterval,
46+ history: make(map[string]*restartRecord),
47+ }
48+}
49+
50+// Start begins the background monitoring loop.
51+func (m *Monitor) Start(ctx context.Context) {
52+ log := zerowrap.FromCtx(ctx)
53+ log.Info().Dur("interval", m.interval).Msg("container monitor started")
54+
55+ go m.run(ctx)
56+}
57+
58+// Stop signals the monitor to stop and waits for it to finish.
59+func (m *Monitor) Stop() {
60+ close(m.stopCh)
61+ <-m.stopped
62+}
63+
64+func (m *Monitor) run(ctx context.Context) {
65+ defer close(m.stopped)
66+
67+ ticker := time.NewTicker(m.interval)
68+ defer ticker.Stop()
69+
70+ for {
71+ select {
72+ case <-m.stopCh:
73+ return
74+ case <-ctx.Done():
75+ return
76+ case <-ticker.C:
77+ m.check(ctx)
78+ }
79+ }
80+}
81+
82+func (m *Monitor) check(ctx context.Context) {
83+ log := zerowrap.FromCtx(ctx)
84+
85+ // Snapshot tracked containers under lock.
86+ m.service.mu.RLock()
87+ snapshot := make(map[string]*domain.Container, len(m.service.containers))
88+ for d, c := range m.service.containers {
89+ snapshot[d] = c
90+ }
91+ m.service.mu.RUnlock()
92+
93+ now := time.Now()
94+
95+ for domainName, tracked := range snapshot {
96+ m.checkContainer(ctx, log, domainName, tracked, now)
97+ }
98+}
99+
100+func (m *Monitor) checkContainer(ctx context.Context, log zerowrap.Logger, domainName string, tracked *domain.Container, now time.Time) {
101+ if tracked == nil {
102+ return
103+ }
104+
105+ inspected, err := m.service.runtime.InspectContainer(ctx, tracked.ID)
106+ if err != nil {
107+ log.Debug().Err(err).Str("domain", domainName).Str("container_id", tracked.ID).
108+ Msg("monitor: failed to inspect container, may have been replaced by deploy")
109+ return
110+ }
111+
112+ switch {
113+ case inspected.Status == string(domain.ContainerStatusRunning):
114+ m.handleRunning(ctx, log, domainName, tracked.ID, now)
115+
116+ case inspected.Status == string(domain.ContainerStatusExited):
117+ if inspected.ExitCode == 0 {
118+ // Graceful stop (exit 0): respect user intent, don't restart.
119+ log.Debug().Str("domain", domainName).Msg("monitor: container exited gracefully (code 0), skipping")
120+ return
121+ }
122+ // Crashed (non-zero exit): restart if not in backoff.
123+ m.handleCrash(ctx, log, domainName, tracked.ID, inspected.ExitCode, now)
124+ }
125+}
126+
127+func (m *Monitor) handleRunning(ctx context.Context, log zerowrap.Logger, domainName, containerID string, now time.Time) {
128+ // Check Docker health status for running containers.
129+ healthStatus, hasHealthcheck, err := m.service.runtime.GetContainerHealthStatus(ctx, containerID)
130+ if err != nil {
131+ return
132+ }
133+
134+ if hasHealthcheck && healthStatus == "unhealthy" {
135+ log.Warn().Str("domain", domainName).Msg("monitor: container is unhealthy, restarting")
136+ if err := m.service.runtime.RestartContainer(ctx, containerID); err != nil {
137+ log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart unhealthy container")
138+ }
139+ return
140+ }
141+
142+ // Container is running and healthy — clear backoff if stable long enough.
143+ m.mu.Lock()
144+ rec, exists := m.history[domainName]
145+ if exists {
146+ if rec.lastSeen.IsZero() {
147+ rec.lastSeen = now
148+ }
149+ if now.Sub(rec.lastSeen) >= stableRunningDuration {
150+ // Container has been running for 5+ min — clear crash history.
151+ delete(m.history, domainName)
152+ m.mu.Unlock()
153+ log.Info().Str("domain", domainName).Msg("monitor: container stable, cleared crash history")
154+ return
155+ }
156+ }
157+ m.mu.Unlock()
158+}
159+
160+func (m *Monitor) handleCrash(ctx context.Context, log zerowrap.Logger, domainName, containerID string, exitCode int, now time.Time) {
161+ m.mu.Lock()
162+ rec := m.history[domainName]
163+ if rec == nil {
164+ rec = &restartRecord{}
165+ m.history[domainName] = rec
166+ }
167+
168+ // Check backoff.
169+ if now.Before(rec.backoffEnd) {
170+ m.mu.Unlock()
171+ log.Debug().Str("domain", domainName).
172+ Time("backoff_until", rec.backoffEnd).
173+ Msg("monitor: in crash loop backoff, skipping restart")
174+ return
175+ }
176+
177+ // Record this crash attempt.
178+ rec.attempts = append(rec.attempts, now)
179+ rec.consecutive++
180+ rec.lastSeen = time.Time{} // reset stable timer
181+
182+ // Prune old attempts outside the crash loop window.
183+ cutoff := now.Add(-crashLoopWindow)
184+ pruned := rec.attempts[:0]
185+ for _, t := range rec.attempts {
186+ if t.After(cutoff) {
187+ pruned = append(pruned, t)
188+ }
189+ }
190+ rec.attempts = pruned
191+
192+ // Check for crash loop.
193+ if len(rec.attempts) >= crashLoopThreshold {
194+ backoff := m.computeBackoff(rec.consecutive)
195+ rec.backoffEnd = now.Add(backoff)
196+ m.mu.Unlock()
197+
198+ log.Error().Str("domain", domainName).
199+ Int("consecutive", rec.consecutive).
200+ Dur("backoff", backoff).
201+ Msg("monitor: crash loop detected, backing off")
202+ return
203+ }
204+ m.mu.Unlock()
205+
206+ // Verify container ID still matches tracked state (deploy may have replaced it).
207+ m.service.mu.RLock()
208+ current := m.service.containers[domainName]
209+ m.service.mu.RUnlock()
210+ if current == nil || current.ID != containerID {
211+ log.Debug().Str("domain", domainName).Msg("monitor: container replaced by deploy, skipping restart")
212+ return
213+ }
214+
215+ log.Warn().Str("domain", domainName).Str("container_id", containerID).
216+ Int("exit_code", exitCode).
217+ Msg("monitor: restarting crashed container")
218+
219+ if err := m.service.runtime.StartContainer(ctx, containerID); err != nil {
220+ log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart container")
221+ }
222+}
223+
224+func (m *Monitor) computeBackoff(consecutive int) time.Duration {
225+ // Exponential backoff: 1min, 2min, 4min, 8min, cap 15min.
226+ base := time.Minute
227+ shift := consecutive - crashLoopThreshold
228+ if shift < 0 {
229+ shift = 0
230+ }
231+ if shift > 4 {
232+ shift = 4 // cap shift to avoid overflow (max 16min, then capped to backoffCap)
233+ }
234+ backoff := base << uint(shift) // #nosec G115 -- shift is bounded [0,4]
235+ if backoff > backoffCap {
236+ backoff = backoffCap
237+ }
238+ return backoff
239+}
@@ -0,0 +1,239 @@
1+package container
2+
3+import (
4+ "context"
5+ "sync"
6+ "time"
7+
8+ "github.com/bnema/zerowrap"
9+
10+ "github.com/bnema/gordon/internal/domain"
11+)
12+
13+const (
14+ monitorDefaultInterval = 15 * time.Second
15+ crashLoopThreshold = 3
16+ crashLoopWindow = 5 * time.Minute
17+ backoffCap = 15 * time.Minute
18+ stableRunningDuration = 5 * time.Minute
19+)
20+
21+// restartRecord tracks restart attempts for crash loop detection.
22+type restartRecord struct {
23+ attempts []time.Time
24+ backoffEnd time.Time
25+ consecutive int
26+ lastSeen time.Time // last time the container was seen running
27+}
28+
29+// Monitor watches tracked containers and restarts crashed ones.
30+type Monitor struct {
31+ service *Service
32+ stopCh chan struct{}
33+ stopped chan struct{}
34+ interval time.Duration
35+ mu sync.Mutex
36+ history map[string]*restartRecord // keyed by domain
37+}
38+
39+// newMonitor creates a new container monitor.
40+func newMonitor(service *Service) *Monitor {
41+ return &Monitor{
42+ service: service,
43+ stopCh: make(chan struct{}),
44+ stopped: make(chan struct{}),
45+ interval: monitorDefaultInterval,
46+ history: make(map[string]*restartRecord),
47+ }
48+}
49+
50+// Start begins the background monitoring loop.
51+func (m *Monitor) Start(ctx context.Context) {
52+ log := zerowrap.FromCtx(ctx)
53+ log.Info().Dur("interval", m.interval).Msg("container monitor started")
54+
55+ go m.run(ctx)
56+}
57+
58+// Stop signals the monitor to stop and waits for it to finish.
59+func (m *Monitor) Stop() {
60+ close(m.stopCh)
61+ <-m.stopped
62+}
63+
64+func (m *Monitor) run(ctx context.Context) {
65+ defer close(m.stopped)
66+
67+ ticker := time.NewTicker(m.interval)
68+ defer ticker.Stop()
69+
70+ for {
71+ select {
72+ case <-m.stopCh:
73+ return
74+ case <-ctx.Done():
75+ return
76+ case <-ticker.C:
77+ m.check(ctx)
78+ }
79+ }
80+}
81+
82+func (m *Monitor) check(ctx context.Context) {
83+ log := zerowrap.FromCtx(ctx)
84+
85+ // Snapshot tracked containers under lock.
86+ m.service.mu.RLock()
87+ snapshot := make(map[string]*domain.Container, len(m.service.containers))
88+ for d, c := range m.service.containers {
89+ snapshot[d] = c
90+ }
91+ m.service.mu.RUnlock()
92+
93+ now := time.Now()
94+
95+ for domainName, tracked := range snapshot {
96+ m.checkContainer(ctx, log, domainName, tracked, now)
97+ }
98+}
99+
100+func (m *Monitor) checkContainer(ctx context.Context, log zerowrap.Logger, domainName string, tracked *domain.Container, now time.Time) {
101+ if tracked == nil {
102+ return
103+ }
104+
105+ inspected, err := m.service.runtime.InspectContainer(ctx, tracked.ID)
106+ if err != nil {
107+ log.Debug().Err(err).Str("domain", domainName).Str("container_id", tracked.ID).
108+ Msg("monitor: failed to inspect container, may have been replaced by deploy")
109+ return
110+ }
111+
112+ switch {
113+ case inspected.Status == string(domain.ContainerStatusRunning):
114+ m.handleRunning(ctx, log, domainName, tracked.ID, now)
115+
116+ case inspected.Status == string(domain.ContainerStatusExited):
117+ if inspected.ExitCode == 0 {
118+ // Graceful stop (exit 0): respect user intent, don't restart.
119+ log.Debug().Str("domain", domainName).Msg("monitor: container exited gracefully (code 0), skipping")
120+ return
121+ }
122+ // Crashed (non-zero exit): restart if not in backoff.
123+ m.handleCrash(ctx, log, domainName, tracked.ID, inspected.ExitCode, now)
124+ }
125+}
126+
127+func (m *Monitor) handleRunning(ctx context.Context, log zerowrap.Logger, domainName, containerID string, now time.Time) {
128+ // Check Docker health status for running containers.
129+ healthStatus, hasHealthcheck, err := m.service.runtime.GetContainerHealthStatus(ctx, containerID)
130+ if err != nil {
131+ return
132+ }
133+
134+ if hasHealthcheck && healthStatus == "unhealthy" {
135+ log.Warn().Str("domain", domainName).Msg("monitor: container is unhealthy, restarting")
136+ if err := m.service.runtime.RestartContainer(ctx, containerID); err != nil {
137+ log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart unhealthy container")
138+ }
139+ return
140+ }
141+
142+ // Container is running and healthy — clear backoff if stable long enough.
143+ m.mu.Lock()
144+ rec, exists := m.history[domainName]
145+ if exists {
146+ if rec.lastSeen.IsZero() {
147+ rec.lastSeen = now
148+ }
149+ if now.Sub(rec.lastSeen) >= stableRunningDuration {
150+ // Container has been running for 5+ min — clear crash history.
151+ delete(m.history, domainName)
152+ m.mu.Unlock()
153+ log.Info().Str("domain", domainName).Msg("monitor: container stable, cleared crash history")
154+ return
155+ }
156+ }
157+ m.mu.Unlock()
158+}
159+
160+func (m *Monitor) handleCrash(ctx context.Context, log zerowrap.Logger, domainName, containerID string, exitCode int, now time.Time) {
161+ m.mu.Lock()
162+ rec := m.history[domainName]
163+ if rec == nil {
164+ rec = &restartRecord{}
165+ m.history[domainName] = rec
166+ }
167+
168+ // Check backoff.
169+ if now.Before(rec.backoffEnd) {
170+ m.mu.Unlock()
171+ log.Debug().Str("domain", domainName).
172+ Time("backoff_until", rec.backoffEnd).
173+ Msg("monitor: in crash loop backoff, skipping restart")
174+ return
175+ }
176+
177+ // Record this crash attempt.
178+ rec.attempts = append(rec.attempts, now)
179+ rec.consecutive++
180+ rec.lastSeen = time.Time{} // reset stable timer
181+
182+ // Prune old attempts outside the crash loop window.
183+ cutoff := now.Add(-crashLoopWindow)
184+ pruned := rec.attempts[:0]
185+ for _, t := range rec.attempts {
186+ if t.After(cutoff) {
187+ pruned = append(pruned, t)
188+ }
189+ }
190+ rec.attempts = pruned
191+
192+ // Check for crash loop.
193+ if len(rec.attempts) >= crashLoopThreshold {
194+ backoff := m.computeBackoff(rec.consecutive)
195+ rec.backoffEnd = now.Add(backoff)
196+ m.mu.Unlock()
197+
198+ log.Error().Str("domain", domainName).
199+ Int("consecutive", rec.consecutive).
200+ Dur("backoff", backoff).
201+ Msg("monitor: crash loop detected, backing off")
202+ return
203+ }
204+ m.mu.Unlock()
205+
206+ // Verify container ID still matches tracked state (deploy may have replaced it).
207+ m.service.mu.RLock()
208+ current := m.service.containers[domainName]
209+ m.service.mu.RUnlock()
210+ if current == nil || current.ID != containerID {
211+ log.Debug().Str("domain", domainName).Msg("monitor: container replaced by deploy, skipping restart")
212+ return
213+ }
214+
215+ log.Warn().Str("domain", domainName).Str("container_id", containerID).
216+ Int("exit_code", exitCode).
217+ Msg("monitor: restarting crashed container")
218+
219+ if err := m.service.runtime.StartContainer(ctx, containerID); err != nil {
220+ log.Warn().Err(err).Str("domain", domainName).Msg("monitor: failed to restart container")
221+ }
222+}
223+
224+func (m *Monitor) computeBackoff(consecutive int) time.Duration {
225+ // Exponential backoff: 1min, 2min, 4min, 8min, cap 15min.
226+ base := time.Minute
227+ shift := consecutive - crashLoopThreshold
228+ if shift < 0 {
229+ shift = 0
230+ }
231+ if shift > 4 {
232+ shift = 4 // cap shift to avoid overflow (max 16min, then capped to backoffCap)
233+ }
234+ backoff := base << uint(shift) // #nosec G115 -- shift is bounded [0,4]
235+ if backoff > backoffCap {
236+ backoff = backoffCap
237+ }
238+ return backoff
239+}
A internal/usecase/container/monitor_test.go +324 -0
@@ -0,0 +1,324 @@
1+package container
2+
3+import (
4+ "context"
5+ "testing"
6+ "time"
7+
8+ "github.com/bnema/zerowrap"
9+ "github.com/stretchr/testify/assert"
10+ "github.com/stretchr/testify/mock"
11+
12+ "github.com/bnema/gordon/internal/boundaries/out/mocks"
13+ "github.com/bnema/gordon/internal/domain"
14+)
15+
16+func newTestService(runtime *mocks.MockContainerRuntime) *Service {
17+ return &Service{
18+ runtime: runtime,
19+ config: Config{},
20+ containers: make(map[string]*domain.Container),
21+ attachments: make(map[string][]string),
22+ }
23+}
24+
25+func monitorTestContext() context.Context {
26+ return zerowrap.WithCtx(context.Background(), zerowrap.Default())
27+}
28+
29+func TestMonitor_RestartsCrashedContainer(t *testing.T) {
30+ runtime := mocks.NewMockContainerRuntime(t)
31+ svc := newTestService(runtime)
32+
33+ // Track a container.
34+ svc.containers["app.example.com"] = &domain.Container{
35+ ID: "ctr-1",
36+ Image: "myapp:latest",
37+ }
38+
39+ // Inspect returns exited with non-zero exit code.
40+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
41+ ID: "ctr-1",
42+ Status: string(domain.ContainerStatusExited),
43+ ExitCode: 1,
44+ }, nil)
45+
46+ // Monitor should restart it.
47+ runtime.EXPECT().StartContainer(mock.Anything, "ctr-1").Return(nil)
48+
49+ m := newMonitor(svc)
50+ m.check(monitorTestContext())
51+
52+ runtime.AssertExpectations(t)
53+}
54+
55+func TestMonitor_SkipsGracefulExit(t *testing.T) {
56+ runtime := mocks.NewMockContainerRuntime(t)
57+ svc := newTestService(runtime)
58+
59+ svc.containers["app.example.com"] = &domain.Container{
60+ ID: "ctr-1",
61+ Image: "myapp:latest",
62+ }
63+
64+ // Exited with code 0 — graceful stop.
65+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
66+ ID: "ctr-1",
67+ Status: string(domain.ContainerStatusExited),
68+ ExitCode: 0,
69+ }, nil)
70+
71+ // Should NOT call StartContainer.
72+
73+ m := newMonitor(svc)
74+ m.check(monitorTestContext())
75+
76+ runtime.AssertExpectations(t)
77+ runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
78+}
79+
80+func TestMonitor_SkipsRunningContainer(t *testing.T) {
81+ runtime := mocks.NewMockContainerRuntime(t)
82+ svc := newTestService(runtime)
83+
84+ svc.containers["app.example.com"] = &domain.Container{
85+ ID: "ctr-1",
86+ Image: "myapp:latest",
87+ }
88+
89+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
90+ ID: "ctr-1",
91+ Status: string(domain.ContainerStatusRunning),
92+ }, nil)
93+
94+ // Running container — check health status.
95+ runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("healthy", true, nil)
96+
97+ m := newMonitor(svc)
98+ m.check(monitorTestContext())
99+
100+ runtime.AssertExpectations(t)
101+ runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
102+}
103+
104+func TestMonitor_RestartsUnhealthyContainer(t *testing.T) {
105+ runtime := mocks.NewMockContainerRuntime(t)
106+ svc := newTestService(runtime)
107+
108+ svc.containers["app.example.com"] = &domain.Container{
109+ ID: "ctr-1",
110+ Image: "myapp:latest",
111+ }
112+
113+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
114+ ID: "ctr-1",
115+ Status: string(domain.ContainerStatusRunning),
116+ }, nil)
117+
118+ // Running but Docker health says unhealthy.
119+ runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("unhealthy", true, nil)
120+
121+ // Should restart the unhealthy container.
122+ runtime.EXPECT().RestartContainer(mock.Anything, "ctr-1").Return(nil)
123+
124+ m := newMonitor(svc)
125+ m.check(monitorTestContext())
126+
127+ runtime.AssertExpectations(t)
128+}
129+
130+func TestMonitor_CrashLoopBackoff(t *testing.T) {
131+ runtime := mocks.NewMockContainerRuntime(t)
132+ svc := newTestService(runtime)
133+
134+ svc.containers["app.example.com"] = &domain.Container{
135+ ID: "ctr-1",
136+ Image: "myapp:latest",
137+ }
138+
139+ m := newMonitor(svc)
140+ ctx := monitorTestContext()
141+ now := time.Now()
142+
143+ // Simulate 3 crashes within the crash loop window.
144+ m.mu.Lock()
145+ m.history["app.example.com"] = &restartRecord{
146+ attempts: []time.Time{now.Add(-2 * time.Minute), now.Add(-1 * time.Minute)},
147+ consecutive: 2,
148+ }
149+ m.mu.Unlock()
150+
151+ // Third crash — this should trigger backoff.
152+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
153+ ID: "ctr-1",
154+ Status: string(domain.ContainerStatusExited),
155+ ExitCode: 137,
156+ }, nil)
157+
158+ // Should NOT call StartContainer because crash loop is detected.
159+ m.check(ctx)
160+
161+ runtime.AssertExpectations(t)
162+ runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
163+
164+ // Verify backoff was set.
165+ m.mu.Lock()
166+ rec := m.history["app.example.com"]
167+ assert.False(t, rec.backoffEnd.IsZero(), "backoff should be set after crash loop detection")
168+ assert.True(t, rec.backoffEnd.After(time.Now()), "backoff end should be in the future")
169+ m.mu.Unlock()
170+}
171+
172+func TestMonitor_BackoffPreventsRestart(t *testing.T) {
173+ runtime := mocks.NewMockContainerRuntime(t)
174+ svc := newTestService(runtime)
175+
176+ svc.containers["app.example.com"] = &domain.Container{
177+ ID: "ctr-1",
178+ Image: "myapp:latest",
179+ }
180+
181+ m := newMonitor(svc)
182+
183+ // Set backoff into the future.
184+ m.mu.Lock()
185+ m.history["app.example.com"] = &restartRecord{
186+ backoffEnd: time.Now().Add(5 * time.Minute),
187+ consecutive: 3,
188+ }
189+ m.mu.Unlock()
190+
191+ // Container is crashed.
192+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
193+ ID: "ctr-1",
194+ Status: string(domain.ContainerStatusExited),
195+ ExitCode: 1,
196+ }, nil)
197+
198+ // Should NOT restart — in backoff.
199+ m.check(monitorTestContext())
200+
201+ runtime.AssertExpectations(t)
202+ runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
203+}
204+
205+func TestMonitor_ClearsCrashHistoryAfterStableRunning(t *testing.T) {
206+ runtime := mocks.NewMockContainerRuntime(t)
207+ svc := newTestService(runtime)
208+
209+ svc.containers["app.example.com"] = &domain.Container{
210+ ID: "ctr-1",
211+ Image: "myapp:latest",
212+ }
213+
214+ m := newMonitor(svc)
215+
216+ // Container has been running for longer than stableRunningDuration.
217+ m.mu.Lock()
218+ m.history["app.example.com"] = &restartRecord{
219+ attempts: []time.Time{time.Now().Add(-10 * time.Minute)},
220+ consecutive: 2,
221+ lastSeen: time.Now().Add(-6 * time.Minute), // 6 min ago > stableRunningDuration (5 min)
222+ }
223+ m.mu.Unlock()
224+
225+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
226+ ID: "ctr-1",
227+ Status: string(domain.ContainerStatusRunning),
228+ }, nil)
229+
230+ runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("healthy", true, nil)
231+
232+ m.check(monitorTestContext())
233+
234+ // History should be cleared.
235+ m.mu.Lock()
236+ _, exists := m.history["app.example.com"]
237+ m.mu.Unlock()
238+ assert.False(t, exists, "crash history should be cleared after stable running")
239+
240+ runtime.AssertExpectations(t)
241+}
242+
243+func TestMonitor_SkipsReplacedContainer(t *testing.T) {
244+ runtime := mocks.NewMockContainerRuntime(t)
245+ svc := newTestService(runtime)
246+
247+ // Track the OLD container ID.
248+ svc.containers["app.example.com"] = &domain.Container{
249+ ID: "ctr-NEW",
250+ Image: "myapp:latest",
251+ }
252+
253+ m := newMonitor(svc)
254+
255+ // Monitor snapshot captured "ctr-OLD" but a deploy replaced it with "ctr-NEW".
256+ // We simulate this by directly calling checkContainer with the old ID.
257+ ctx := monitorTestContext()
258+ log := zerowrap.FromCtx(ctx)
259+
260+ // Inspect the old container — it's exited.
261+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-OLD").Return(&domain.Container{
262+ ID: "ctr-OLD",
263+ Status: string(domain.ContainerStatusExited),
264+ ExitCode: 1,
265+ }, nil)
266+
267+ // Should NOT restart — tracked container has been replaced.
268+ oldTracked := &domain.Container{ID: "ctr-OLD", Image: "myapp:latest"}
269+ m.checkContainer(ctx, log, "app.example.com", oldTracked, time.Now())
270+
271+ runtime.AssertExpectations(t)
272+ runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
273+}
274+
275+func TestMonitor_ComputeBackoff(t *testing.T) {
276+ m := &Monitor{}
277+
278+ tests := []struct {
279+ consecutive int
280+ expected time.Duration
281+ }{
282+ {3, 1 * time.Minute}, // first backoff
283+ {4, 2 * time.Minute}, // second
284+ {5, 4 * time.Minute}, // third
285+ {6, 8 * time.Minute}, // fourth
286+ {7, 15 * time.Minute}, // capped
287+ {10, 15 * time.Minute}, // still capped
288+ }
289+
290+ for _, tt := range tests {
291+ got := m.computeBackoff(tt.consecutive)
292+ assert.Equal(t, tt.expected, got, "consecutive=%d", tt.consecutive)
293+ }
294+}
295+
296+func TestMonitor_StartStop(t *testing.T) {
297+ runtime := mocks.NewMockContainerRuntime(t)
298+ svc := newTestService(runtime)
299+
300+ m := newMonitor(svc)
301+ m.interval = 50 * time.Millisecond // fast ticking for test
302+
303+ ctx, cancel := context.WithCancel(monitorTestContext())
304+ defer cancel()
305+
306+ m.Start(ctx)
307+
308+ // Let it tick a couple times.
309+ time.Sleep(150 * time.Millisecond)
310+
311+ // Stop should return promptly.
312+ done := make(chan struct{})
313+ go func() {
314+ m.Stop()
315+ close(done)
316+ }()
317+
318+ select {
319+ case <-done:
320+ // OK
321+ case <-time.After(2 * time.Second):
322+ t.Fatal("monitor.Stop() did not return within 2 seconds")
323+ }
324+}
@@ -0,0 +1,324 @@
1+package container
2+
3+import (
4+ "context"
5+ "testing"
6+ "time"
7+
8+ "github.com/bnema/zerowrap"
9+ "github.com/stretchr/testify/assert"
10+ "github.com/stretchr/testify/mock"
11+
12+ "github.com/bnema/gordon/internal/boundaries/out/mocks"
13+ "github.com/bnema/gordon/internal/domain"
14+)
15+
16+func newTestService(runtime *mocks.MockContainerRuntime) *Service {
17+ return &Service{
18+ runtime: runtime,
19+ config: Config{},
20+ containers: make(map[string]*domain.Container),
21+ attachments: make(map[string][]string),
22+ }
23+}
24+
25+func monitorTestContext() context.Context {
26+ return zerowrap.WithCtx(context.Background(), zerowrap.Default())
27+}
28+
29+func TestMonitor_RestartsCrashedContainer(t *testing.T) {
30+ runtime := mocks.NewMockContainerRuntime(t)
31+ svc := newTestService(runtime)
32+
33+ // Track a container.
34+ svc.containers["app.example.com"] = &domain.Container{
35+ ID: "ctr-1",
36+ Image: "myapp:latest",
37+ }
38+
39+ // Inspect returns exited with non-zero exit code.
40+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
41+ ID: "ctr-1",
42+ Status: string(domain.ContainerStatusExited),
43+ ExitCode: 1,
44+ }, nil)
45+
46+ // Monitor should restart it.
47+ runtime.EXPECT().StartContainer(mock.Anything, "ctr-1").Return(nil)
48+
49+ m := newMonitor(svc)
50+ m.check(monitorTestContext())
51+
52+ runtime.AssertExpectations(t)
53+}
54+
55+func TestMonitor_SkipsGracefulExit(t *testing.T) {
56+ runtime := mocks.NewMockContainerRuntime(t)
57+ svc := newTestService(runtime)
58+
59+ svc.containers["app.example.com"] = &domain.Container{
60+ ID: "ctr-1",
61+ Image: "myapp:latest",
62+ }
63+
64+ // Exited with code 0 — graceful stop.
65+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
66+ ID: "ctr-1",
67+ Status: string(domain.ContainerStatusExited),
68+ ExitCode: 0,
69+ }, nil)
70+
71+ // Should NOT call StartContainer.
72+
73+ m := newMonitor(svc)
74+ m.check(monitorTestContext())
75+
76+ runtime.AssertExpectations(t)
77+ runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
78+}
79+
80+func TestMonitor_SkipsRunningContainer(t *testing.T) {
81+ runtime := mocks.NewMockContainerRuntime(t)
82+ svc := newTestService(runtime)
83+
84+ svc.containers["app.example.com"] = &domain.Container{
85+ ID: "ctr-1",
86+ Image: "myapp:latest",
87+ }
88+
89+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
90+ ID: "ctr-1",
91+ Status: string(domain.ContainerStatusRunning),
92+ }, nil)
93+
94+ // Running container — check health status.
95+ runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("healthy", true, nil)
96+
97+ m := newMonitor(svc)
98+ m.check(monitorTestContext())
99+
100+ runtime.AssertExpectations(t)
101+ runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
102+}
103+
104+func TestMonitor_RestartsUnhealthyContainer(t *testing.T) {
105+ runtime := mocks.NewMockContainerRuntime(t)
106+ svc := newTestService(runtime)
107+
108+ svc.containers["app.example.com"] = &domain.Container{
109+ ID: "ctr-1",
110+ Image: "myapp:latest",
111+ }
112+
113+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
114+ ID: "ctr-1",
115+ Status: string(domain.ContainerStatusRunning),
116+ }, nil)
117+
118+ // Running but Docker health says unhealthy.
119+ runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("unhealthy", true, nil)
120+
121+ // Should restart the unhealthy container.
122+ runtime.EXPECT().RestartContainer(mock.Anything, "ctr-1").Return(nil)
123+
124+ m := newMonitor(svc)
125+ m.check(monitorTestContext())
126+
127+ runtime.AssertExpectations(t)
128+}
129+
130+func TestMonitor_CrashLoopBackoff(t *testing.T) {
131+ runtime := mocks.NewMockContainerRuntime(t)
132+ svc := newTestService(runtime)
133+
134+ svc.containers["app.example.com"] = &domain.Container{
135+ ID: "ctr-1",
136+ Image: "myapp:latest",
137+ }
138+
139+ m := newMonitor(svc)
140+ ctx := monitorTestContext()
141+ now := time.Now()
142+
143+ // Simulate 3 crashes within the crash loop window.
144+ m.mu.Lock()
145+ m.history["app.example.com"] = &restartRecord{
146+ attempts: []time.Time{now.Add(-2 * time.Minute), now.Add(-1 * time.Minute)},
147+ consecutive: 2,
148+ }
149+ m.mu.Unlock()
150+
151+ // Third crash — this should trigger backoff.
152+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
153+ ID: "ctr-1",
154+ Status: string(domain.ContainerStatusExited),
155+ ExitCode: 137,
156+ }, nil)
157+
158+ // Should NOT call StartContainer because crash loop is detected.
159+ m.check(ctx)
160+
161+ runtime.AssertExpectations(t)
162+ runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
163+
164+ // Verify backoff was set.
165+ m.mu.Lock()
166+ rec := m.history["app.example.com"]
167+ assert.False(t, rec.backoffEnd.IsZero(), "backoff should be set after crash loop detection")
168+ assert.True(t, rec.backoffEnd.After(time.Now()), "backoff end should be in the future")
169+ m.mu.Unlock()
170+}
171+
172+func TestMonitor_BackoffPreventsRestart(t *testing.T) {
173+ runtime := mocks.NewMockContainerRuntime(t)
174+ svc := newTestService(runtime)
175+
176+ svc.containers["app.example.com"] = &domain.Container{
177+ ID: "ctr-1",
178+ Image: "myapp:latest",
179+ }
180+
181+ m := newMonitor(svc)
182+
183+ // Set backoff into the future.
184+ m.mu.Lock()
185+ m.history["app.example.com"] = &restartRecord{
186+ backoffEnd: time.Now().Add(5 * time.Minute),
187+ consecutive: 3,
188+ }
189+ m.mu.Unlock()
190+
191+ // Container is crashed.
192+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
193+ ID: "ctr-1",
194+ Status: string(domain.ContainerStatusExited),
195+ ExitCode: 1,
196+ }, nil)
197+
198+ // Should NOT restart — in backoff.
199+ m.check(monitorTestContext())
200+
201+ runtime.AssertExpectations(t)
202+ runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
203+}
204+
205+func TestMonitor_ClearsCrashHistoryAfterStableRunning(t *testing.T) {
206+ runtime := mocks.NewMockContainerRuntime(t)
207+ svc := newTestService(runtime)
208+
209+ svc.containers["app.example.com"] = &domain.Container{
210+ ID: "ctr-1",
211+ Image: "myapp:latest",
212+ }
213+
214+ m := newMonitor(svc)
215+
216+ // Container has been running for longer than stableRunningDuration.
217+ m.mu.Lock()
218+ m.history["app.example.com"] = &restartRecord{
219+ attempts: []time.Time{time.Now().Add(-10 * time.Minute)},
220+ consecutive: 2,
221+ lastSeen: time.Now().Add(-6 * time.Minute), // 6 min ago > stableRunningDuration (5 min)
222+ }
223+ m.mu.Unlock()
224+
225+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-1").Return(&domain.Container{
226+ ID: "ctr-1",
227+ Status: string(domain.ContainerStatusRunning),
228+ }, nil)
229+
230+ runtime.EXPECT().GetContainerHealthStatus(mock.Anything, "ctr-1").Return("healthy", true, nil)
231+
232+ m.check(monitorTestContext())
233+
234+ // History should be cleared.
235+ m.mu.Lock()
236+ _, exists := m.history["app.example.com"]
237+ m.mu.Unlock()
238+ assert.False(t, exists, "crash history should be cleared after stable running")
239+
240+ runtime.AssertExpectations(t)
241+}
242+
243+func TestMonitor_SkipsReplacedContainer(t *testing.T) {
244+ runtime := mocks.NewMockContainerRuntime(t)
245+ svc := newTestService(runtime)
246+
247+ // Track the OLD container ID.
248+ svc.containers["app.example.com"] = &domain.Container{
249+ ID: "ctr-NEW",
250+ Image: "myapp:latest",
251+ }
252+
253+ m := newMonitor(svc)
254+
255+ // Monitor snapshot captured "ctr-OLD" but a deploy replaced it with "ctr-NEW".
256+ // We simulate this by directly calling checkContainer with the old ID.
257+ ctx := monitorTestContext()
258+ log := zerowrap.FromCtx(ctx)
259+
260+ // Inspect the old container — it's exited.
261+ runtime.EXPECT().InspectContainer(mock.Anything, "ctr-OLD").Return(&domain.Container{
262+ ID: "ctr-OLD",
263+ Status: string(domain.ContainerStatusExited),
264+ ExitCode: 1,
265+ }, nil)
266+
267+ // Should NOT restart — tracked container has been replaced.
268+ oldTracked := &domain.Container{ID: "ctr-OLD", Image: "myapp:latest"}
269+ m.checkContainer(ctx, log, "app.example.com", oldTracked, time.Now())
270+
271+ runtime.AssertExpectations(t)
272+ runtime.AssertNotCalled(t, "StartContainer", mock.Anything, mock.Anything)
273+}
274+
275+func TestMonitor_ComputeBackoff(t *testing.T) {
276+ m := &Monitor{}
277+
278+ tests := []struct {
279+ consecutive int
280+ expected time.Duration
281+ }{
282+ {3, 1 * time.Minute}, // first backoff
283+ {4, 2 * time.Minute}, // second
284+ {5, 4 * time.Minute}, // third
285+ {6, 8 * time.Minute}, // fourth
286+ {7, 15 * time.Minute}, // capped
287+ {10, 15 * time.Minute}, // still capped
288+ }
289+
290+ for _, tt := range tests {
291+ got := m.computeBackoff(tt.consecutive)
292+ assert.Equal(t, tt.expected, got, "consecutive=%d", tt.consecutive)
293+ }
294+}
295+
296+func TestMonitor_StartStop(t *testing.T) {
297+ runtime := mocks.NewMockContainerRuntime(t)
298+ svc := newTestService(runtime)
299+
300+ m := newMonitor(svc)
301+ m.interval = 50 * time.Millisecond // fast ticking for test
302+
303+ ctx, cancel := context.WithCancel(monitorTestContext())
304+ defer cancel()
305+
306+ m.Start(ctx)
307+
308+ // Let it tick a couple times.
309+ time.Sleep(150 * time.Millisecond)
310+
311+ // Stop should return promptly.
312+ done := make(chan struct{})
313+ go func() {
314+ m.Stop()
315+ close(done)
316+ }()
317+
318+ select {
319+ case <-done:
320+ // OK
321+ case <-time.After(2 * time.Second):
322+ t.Fatal("monitor.Stop() did not return within 2 seconds")
323+ }
324+}
M internal/usecase/container/service.go +90 -66
@@ -69,6 +69,7 @@ type Service struct {
6969 attachments map[string][]string // ownerDomain → []containerIDs
7070 mu sync.RWMutex
7171 deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)
72+ monitor *Monitor
7273 }
7374
7475 // domainDeployLock is a context-aware mutex using a buffered channel.
@@ -347,9 +348,11 @@ func (s *Service) createStartedContainer(ctx context.Context, route domain.Route
347348 s.runtime.RemoveContainer(ctx, newContainer.ID, true)
348349 return nil, log.WrapErr(err, "failed to start container")
349350 }
350- if err := s.waitForReady(ctx, newContainer.ID); err != nil {
351- s.cleanupFailedContainer(ctx, newContainer.ID)
352- return nil, log.WrapErr(err, "container failed readiness check")
351+ if !domain.IsSkipReadiness(ctx) {
352+ if err := s.waitForReady(ctx, newContainer.ID); err != nil {
353+ s.cleanupFailedContainer(ctx, newContainer.ID)
354+ return nil, log.WrapErr(err, "container failed readiness check")
355+ }
353356 }
354357
355358 inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
@@ -805,6 +808,8 @@ func (s *Service) SyncContainers(ctx context.Context) error {
805808 }
806809
807810 // AutoStart starts containers for the provided routes that aren't running.
811+// It skips readiness checks to avoid blocking boot; the background monitor
812+// handles crash recovery.
808813 func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
809814 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
810815 zerowrap.FieldLayer: "usecase",
@@ -814,37 +819,67 @@ func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
814819
815820 log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
816821
817- var started, skipped, errors int
822+ // Filter out routes that already have a running container.
823+ var pending []domain.Route
824+ skipped := 0
818825 for _, route := range routes {
819- // Check if container already exists and is running
820826 if _, exists := s.Get(ctx, route.Domain); exists {
821827 log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
822828 skipped++
823- continue
829+ } else {
830+ pending = append(pending, route)
824831 }
832+ }
825833
826- log.Info().
827- Str("domain", route.Domain).
828- Str("image", route.Image).
829- Msg("auto-starting container for route")
834+ if len(pending) == 0 {
835+ log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
836+ return nil
837+ }
830838
831- if _, err := s.Deploy(ctx, route); err != nil {
832- log.Warn().Err(err).Str("domain", route.Domain).Msg("failed to auto-start container")
833- errors++
834- continue
835- }
839+ // Deploy all pending routes concurrently with readiness checks skipped.
840+ deployCtx := domain.WithSkipReadiness(ctx)
841+
842+ type result struct {
843+ route domain.Route
844+ err error
845+ }
846+ results := make(chan result, len(pending))
847+ sem := make(chan struct{}, 4) // limit concurrency
836848
837- started++
849+ for _, route := range pending {
850+ sem <- struct{}{}
851+ go func(r domain.Route) {
852+ defer func() { <-sem }()
853+
854+ log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
855+
856+ if _, err := s.Deploy(deployCtx, r); err != nil {
857+ results <- result{route: r, err: err}
858+ return
859+ }
860+ results <- result{route: r}
861+ }(route)
862+ }
863+
864+ var started, errCount int
865+ for i := 0; i < len(pending); i++ {
866+ res := <-results
867+ if res.err != nil {
868+ log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
869+ errCount++
870+ } else {
871+ started++
872+ }
838873 }
839874
840875 log.Info().
841876 Int("started", started).
842877 Int("skipped", skipped).
843- Int("errors", errors).
878+ Int("errors", errCount).
844879 Msg("auto-start completed")
845880
846- if errors > 0 {
847- return fmt.Errorf("auto-start completed with %d errors", errors)
881+ if errCount > 0 {
882+ return fmt.Errorf("auto-start completed with %d errors", errCount)
848883 }
849884 return nil
850885 }
@@ -858,9 +893,8 @@ func (s *Service) Shutdown(ctx context.Context) error {
858893 log := zerowrap.FromCtx(ctx)
859894 log.Info().Msg("shutting down container manager...")
860895
861- if err := s.stopAllManagedContainers(ctx); err != nil {
862- log.WrapErr(err, "failed to stop all containers during shutdown")
863- }
896+ // Containers are left running across Gordon restarts.
897+ // SyncContainers + AutoStart will pick them back up on next boot.
864898
865899 // Close log writer to stop all log collection
866900 if s.logWriter != nil {
@@ -873,6 +907,33 @@ func (s *Service) Shutdown(ctx context.Context) error {
873907 return nil
874908 }
875909
910+// StartMonitor begins background monitoring of tracked containers.
911+// The monitor restarts crashed containers and detects crash loops.
912+// Safe to call multiple times; subsequent calls are no-ops.
913+func (s *Service) StartMonitor(ctx context.Context) {
914+ s.mu.Lock()
915+ if s.monitor != nil {
916+ s.mu.Unlock()
917+ return
918+ }
919+ s.monitor = newMonitor(s)
920+ m := s.monitor
921+ s.mu.Unlock()
922+ m.Start(ctx)
923+}
924+
925+// StopMonitor stops the background container monitor.
926+// Safe to call multiple times; subsequent calls are no-ops.
927+func (s *Service) StopMonitor() {
928+ s.mu.Lock()
929+ m := s.monitor
930+ s.monitor = nil
931+ s.mu.Unlock()
932+ if m != nil {
933+ m.Stop()
934+ }
935+}
936+
876937 // UpdateConfig updates the service configuration.
877938 func (s *Service) UpdateConfig(config Config) {
878939 s.mu.Lock()
@@ -1720,46 +1781,6 @@ func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, servic
17201781 return nil
17211782 }
17221783
1723-func (s *Service) stopAllManagedContainers(ctx context.Context) error {
1724- log := zerowrap.FromCtx(ctx)
1725-
1726- s.mu.RLock()
1727- containers := make(map[string]*domain.Container, len(s.containers))
1728- maps.Copy(containers, s.containers)
1729- s.mu.RUnlock()
1730-
1731- if len(containers) == 0 {
1732- log.Info().Msg("no managed containers to stop")
1733- return nil
1734- }
1735-
1736- log.Info().Int(zerowrap.FieldCount, len(containers)).Msg("stopping all managed containers...")
1737-
1738- errorCount := 0
1739- for d, c := range containers {
1740- log.Info().Str("domain", d).Str(zerowrap.FieldEntityID, c.ID).Msg("stopping managed container")
1741-
1742- if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
1743- log.WrapErrWithFields(err, "failed to stop container", map[string]any{"domain": d, zerowrap.FieldEntityID: c.ID})
1744- errorCount++
1745- continue
1746- }
1747-
1748- s.mu.Lock()
1749- delete(s.containers, d)
1750- s.mu.Unlock()
1751-
1752- log.Info().Str("domain", d).Str(zerowrap.FieldEntityID, c.ID).Msg("container stopped successfully")
1753- }
1754-
1755- if errorCount > 0 {
1756- return fmt.Errorf("failed to stop %d containers", errorCount)
1757- }
1758-
1759- log.Info().Msg("all managed containers stopped successfully")
1760- return nil
1761-}
1762-
17631784 // Utility functions
17641785
17651786 func normalizeImageRef(image string) string {
@@ -1969,7 +1990,7 @@ func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeou
19691990 waitCtx, cancel := context.WithTimeout(ctx, timeout)
19701991 defer cancel()
19711992
1972- lastStatus := "starting"
1993+ var lastStatus string
19731994 for {
19741995 status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
19751996 if err != nil {
@@ -1978,12 +1999,15 @@ func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeou
19781999 if !hasHealthcheck {
19792000 return errors.New("no healthcheck detected")
19802001 }
2002+ // Treat empty health status as transitional startup. Some runtimes can
2003+ // temporarily report an empty status before first probe results.
2004+ if status == "" {
2005+ status = "starting"
2006+ }
19812007 if status == "healthy" {
19822008 return nil
19832009 }
1984- if status != "" {
1985- lastStatus = status
1986- }
2010+ lastStatus = status
19872011
19882012 select {
19892013 case <-time.After(time.Second):
@@ -69,6 +69,7 @@ type Service struct {
69 attachments map[string][]string // ownerDomain → []containerIDs69 attachments map[string][]string // ownerDomain → []containerIDs
70 mu sync.RWMutex70 mu sync.RWMutex
71 deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)71 deployMu sync.Map // per-domain deploy locks (domain → *domainDeployLock)
72+ monitor *Monitor
72 }73 }
73 74
74 // domainDeployLock is a context-aware mutex using a buffered channel.75 // domainDeployLock is a context-aware mutex using a buffered channel.
@@ -347,9 +348,11 @@ func (s *Service) createStartedContainer(ctx context.Context, route domain.Route
347 s.runtime.RemoveContainer(ctx, newContainer.ID, true)348 s.runtime.RemoveContainer(ctx, newContainer.ID, true)
348 return nil, log.WrapErr(err, "failed to start container")349 return nil, log.WrapErr(err, "failed to start container")
349 }350 }
350- if err := s.waitForReady(ctx, newContainer.ID); err != nil {351+ if !domain.IsSkipReadiness(ctx) {
351- s.cleanupFailedContainer(ctx, newContainer.ID)352+ if err := s.waitForReady(ctx, newContainer.ID); err != nil {
352- return nil, log.WrapErr(err, "container failed readiness check")353+ s.cleanupFailedContainer(ctx, newContainer.ID)
354+ return nil, log.WrapErr(err, "container failed readiness check")
355+ }
353 }356 }
354 357
355 inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)358 inspected, err := s.runtime.InspectContainer(ctx, newContainer.ID)
@@ -805,6 +808,8 @@ func (s *Service) SyncContainers(ctx context.Context) error {
805 }808 }
806 809
807 // AutoStart starts containers for the provided routes that aren't running.810 // AutoStart starts containers for the provided routes that aren't running.
811+// It skips readiness checks to avoid blocking boot; the background monitor
812+// handles crash recovery.
808 func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {813 func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
809 ctx = zerowrap.CtxWithFields(ctx, map[string]any{814 ctx = zerowrap.CtxWithFields(ctx, map[string]any{
810 zerowrap.FieldLayer: "usecase",815 zerowrap.FieldLayer: "usecase",
@@ -814,37 +819,67 @@ func (s *Service) AutoStart(ctx context.Context, routes []domain.Route) error {
814 819
815 log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")820 log.Info().Int("route_count", len(routes)).Msg("auto-starting containers for configured routes")
816 821
817- var started, skipped, errors int822+ // Filter out routes that already have a running container.
823+ var pending []domain.Route
824+ skipped := 0
818 for _, route := range routes {825 for _, route := range routes {
819- // Check if container already exists and is running
820 if _, exists := s.Get(ctx, route.Domain); exists {826 if _, exists := s.Get(ctx, route.Domain); exists {
821 log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")827 log.Debug().Str("domain", route.Domain).Msg("container already running, skipping")
822 skipped++828 skipped++
823- continue829+ } else {
830+ pending = append(pending, route)
824 }831 }
832+ }
825 833
826- log.Info().834+ if len(pending) == 0 {
827- Str("domain", route.Domain).835+ log.Info().Int("skipped", skipped).Msg("auto-start completed, all containers already running")
828- Str("image", route.Image).836+ return nil
829- Msg("auto-starting container for route")837+ }
830 838
831- if _, err := s.Deploy(ctx, route); err != nil {839+ // Deploy all pending routes concurrently with readiness checks skipped.
832- log.Warn().Err(err).Str("domain", route.Domain).Msg("failed to auto-start container")840+ deployCtx := domain.WithSkipReadiness(ctx)
833- errors++841+
834- continue842+ type result struct {
835- }843+ route domain.Route
844+ err error
845+ }
846+ results := make(chan result, len(pending))
847+ sem := make(chan struct{}, 4) // limit concurrency
836 848
837- started++849+ for _, route := range pending {
850+ sem <- struct{}{}
851+ go func(r domain.Route) {
852+ defer func() { <-sem }()
853+
854+ log.Info().Str("domain", r.Domain).Str("image", r.Image).Msg("auto-starting container for route")
855+
856+ if _, err := s.Deploy(deployCtx, r); err != nil {
857+ results <- result{route: r, err: err}
858+ return
859+ }
860+ results <- result{route: r}
861+ }(route)
862+ }
863+
864+ var started, errCount int
865+ for i := 0; i < len(pending); i++ {
866+ res := <-results
867+ if res.err != nil {
868+ log.Warn().Err(res.err).Str("domain", res.route.Domain).Msg("failed to auto-start container")
869+ errCount++
870+ } else {
871+ started++
872+ }
838 }873 }
839 874
840 log.Info().875 log.Info().
841 Int("started", started).876 Int("started", started).
842 Int("skipped", skipped).877 Int("skipped", skipped).
843- Int("errors", errors).878+ Int("errors", errCount).
844 Msg("auto-start completed")879 Msg("auto-start completed")
845 880
846- if errors > 0 {881+ if errCount > 0 {
847- return fmt.Errorf("auto-start completed with %d errors", errors)882+ return fmt.Errorf("auto-start completed with %d errors", errCount)
848 }883 }
849 return nil884 return nil
850 }885 }
@@ -858,9 +893,8 @@ func (s *Service) Shutdown(ctx context.Context) error {
858 log := zerowrap.FromCtx(ctx)893 log := zerowrap.FromCtx(ctx)
859 log.Info().Msg("shutting down container manager...")894 log.Info().Msg("shutting down container manager...")
860 895
861- if err := s.stopAllManagedContainers(ctx); err != nil {896+ // Containers are left running across Gordon restarts.
862- log.WrapErr(err, "failed to stop all containers during shutdown")897+ // SyncContainers + AutoStart will pick them back up on next boot.
863- }
864 898
865 // Close log writer to stop all log collection899 // Close log writer to stop all log collection
866 if s.logWriter != nil {900 if s.logWriter != nil {
@@ -873,6 +907,33 @@ func (s *Service) Shutdown(ctx context.Context) error {
873 return nil907 return nil
874 }908 }
875 909
910+// StartMonitor begins background monitoring of tracked containers.
911+// The monitor restarts crashed containers and detects crash loops.
912+// Safe to call multiple times; subsequent calls are no-ops.
913+func (s *Service) StartMonitor(ctx context.Context) {
914+ s.mu.Lock()
915+ if s.monitor != nil {
916+ s.mu.Unlock()
917+ return
918+ }
919+ s.monitor = newMonitor(s)
920+ m := s.monitor
921+ s.mu.Unlock()
922+ m.Start(ctx)
923+}
924+
925+// StopMonitor stops the background container monitor.
926+// Safe to call multiple times; subsequent calls are no-ops.
927+func (s *Service) StopMonitor() {
928+ s.mu.Lock()
929+ m := s.monitor
930+ s.monitor = nil
931+ s.mu.Unlock()
932+ if m != nil {
933+ m.Stop()
934+ }
935+}
936+
876 // UpdateConfig updates the service configuration.937 // UpdateConfig updates the service configuration.
877 func (s *Service) UpdateConfig(config Config) {938 func (s *Service) UpdateConfig(config Config) {
878 s.mu.Lock()939 s.mu.Lock()
@@ -1720,46 +1781,6 @@ func (s *Service) deployAttachedService(ctx context.Context, ownerDomain, servic
1720 return nil1781 return nil
1721 }1782 }
1722 1783
1723-func (s *Service) stopAllManagedContainers(ctx context.Context) error {
1724- log := zerowrap.FromCtx(ctx)
1725-
1726- s.mu.RLock()
1727- containers := make(map[string]*domain.Container, len(s.containers))
1728- maps.Copy(containers, s.containers)
1729- s.mu.RUnlock()
1730-
1731- if len(containers) == 0 {
1732- log.Info().Msg("no managed containers to stop")
1733- return nil
1734- }
1735-
1736- log.Info().Int(zerowrap.FieldCount, len(containers)).Msg("stopping all managed containers...")
1737-
1738- errorCount := 0
1739- for d, c := range containers {
1740- log.Info().Str("domain", d).Str(zerowrap.FieldEntityID, c.ID).Msg("stopping managed container")
1741-
1742- if err := s.runtime.StopContainer(ctx, c.ID); err != nil {
1743- log.WrapErrWithFields(err, "failed to stop container", map[string]any{"domain": d, zerowrap.FieldEntityID: c.ID})
1744- errorCount++
1745- continue
1746- }
1747-
1748- s.mu.Lock()
1749- delete(s.containers, d)
1750- s.mu.Unlock()
1751-
1752- log.Info().Str("domain", d).Str(zerowrap.FieldEntityID, c.ID).Msg("container stopped successfully")
1753- }
1754-
1755- if errorCount > 0 {
1756- return fmt.Errorf("failed to stop %d containers", errorCount)
1757- }
1758-
1759- log.Info().Msg("all managed containers stopped successfully")
1760- return nil
1761-}
1762-
1763 // Utility functions1784 // Utility functions
1764 1785
1765 func normalizeImageRef(image string) string {1786 func normalizeImageRef(image string) string {
@@ -1969,7 +1990,7 @@ func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeou
1969 waitCtx, cancel := context.WithTimeout(ctx, timeout)1990 waitCtx, cancel := context.WithTimeout(ctx, timeout)
1970 defer cancel()1991 defer cancel()
1971 1992
1972- lastStatus := "starting"1993+ var lastStatus string
1973 for {1994 for {
1974 status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)1995 status, hasHealthcheck, err := s.runtime.GetContainerHealthStatus(waitCtx, containerID)
1975 if err != nil {1996 if err != nil {
@@ -1978,12 +1999,15 @@ func (s *Service) waitForHealthy(ctx context.Context, containerID string, timeou
1978 if !hasHealthcheck {1999 if !hasHealthcheck {
1979 return errors.New("no healthcheck detected")2000 return errors.New("no healthcheck detected")
1980 }2001 }
2002+ // Treat empty health status as transitional startup. Some runtimes can
2003+ // temporarily report an empty status before first probe results.
2004+ if status == "" {
2005+ status = "starting"
2006+ }
1981 if status == "healthy" {2007 if status == "healthy" {
1982 return nil2008 return nil
1983 }2009 }
1984- if status != "" {2010+ lastStatus = status
1985- lastStatus = status
1986- }
1987 2011
1988 select {2012 select {
1989 case <-time.After(time.Second):2013 case <-time.After(time.Second):
A internal/usecase/container/service_readiness_test.go +47 -0
@@ -0,0 +1,47 @@
1+package container
2+
3+import (
4+ "strings"
5+ "testing"
6+ "time"
7+
8+ "github.com/stretchr/testify/assert"
9+ "github.com/stretchr/testify/mock"
10+
11+ "github.com/bnema/gordon/internal/boundaries/out/mocks"
12+)
13+
14+func TestService_WaitForReady_AutoFallsBackToDelayWhenNoHealthcheck(t *testing.T) {
15+ runtime := mocks.NewMockContainerRuntime(t)
16+ svc := NewService(runtime, nil, nil, nil, Config{
17+ ReadinessMode: "auto",
18+ ReadinessDelay: time.Millisecond,
19+ })
20+
21+ ctx := testContext()
22+ containerID := "container-1"
23+
24+ // Initial running poll
25+ runtime.EXPECT().IsContainerRunning(mock.Anything, containerID).Return(true, nil).Once()
26+ // Auto mode probes for healthcheck support first
27+ runtime.EXPECT().GetContainerHealthStatus(mock.Anything, containerID).Return("", false, nil).Once()
28+ // Delay-mode fallback verification
29+ runtime.EXPECT().IsContainerRunning(mock.Anything, containerID).Return(true, nil).Once()
30+
31+ err := svc.waitForReady(ctx, containerID)
32+ assert.NoError(t, err)
33+}
34+
35+func TestService_WaitForHealthy_EmptyStatusIsReportedAsStarting(t *testing.T) {
36+ runtime := mocks.NewMockContainerRuntime(t)
37+ svc := NewService(runtime, nil, nil, nil, Config{})
38+
39+ ctx := testContext()
40+ containerID := "container-1"
41+
42+ runtime.EXPECT().GetContainerHealthStatus(mock.Anything, containerID).Return("", true, nil).Once()
43+
44+ err := svc.waitForHealthy(ctx, containerID, 10*time.Millisecond)
45+ assert.Error(t, err)
46+ assert.True(t, strings.Contains(err.Error(), "last status: starting"))
47+}
@@ -0,0 +1,47 @@
1+package container
2+
3+import (
4+ "strings"
5+ "testing"
6+ "time"
7+
8+ "github.com/stretchr/testify/assert"
9+ "github.com/stretchr/testify/mock"
10+
11+ "github.com/bnema/gordon/internal/boundaries/out/mocks"
12+)
13+
14+func TestService_WaitForReady_AutoFallsBackToDelayWhenNoHealthcheck(t *testing.T) {
15+ runtime := mocks.NewMockContainerRuntime(t)
16+ svc := NewService(runtime, nil, nil, nil, Config{
17+ ReadinessMode: "auto",
18+ ReadinessDelay: time.Millisecond,
19+ })
20+
21+ ctx := testContext()
22+ containerID := "container-1"
23+
24+ // Initial running poll
25+ runtime.EXPECT().IsContainerRunning(mock.Anything, containerID).Return(true, nil).Once()
26+ // Auto mode probes for healthcheck support first
27+ runtime.EXPECT().GetContainerHealthStatus(mock.Anything, containerID).Return("", false, nil).Once()
28+ // Delay-mode fallback verification
29+ runtime.EXPECT().IsContainerRunning(mock.Anything, containerID).Return(true, nil).Once()
30+
31+ err := svc.waitForReady(ctx, containerID)
32+ assert.NoError(t, err)
33+}
34+
35+func TestService_WaitForHealthy_EmptyStatusIsReportedAsStarting(t *testing.T) {
36+ runtime := mocks.NewMockContainerRuntime(t)
37+ svc := NewService(runtime, nil, nil, nil, Config{})
38+
39+ ctx := testContext()
40+ containerID := "container-1"
41+
42+ runtime.EXPECT().GetContainerHealthStatus(mock.Anything, containerID).Return("", true, nil).Once()
43+
44+ err := svc.waitForHealthy(ctx, containerID, 10*time.Millisecond)
45+ assert.Error(t, err)
46+ assert.True(t, strings.Contains(err.Error(), "last status: starting"))
47+}
M internal/usecase/container/service_test.go +9 -33
@@ -630,35 +630,13 @@ func TestService_Shutdown(t *testing.T) {
630630 svc.containers["app1.example.com"] = &domain.Container{ID: "container-1"}
631631 svc.containers["app2.example.com"] = &domain.Container{ID: "container-2"}
632632
633- runtime.EXPECT().StopContainer(mock.Anything, "container-1").Return(nil)
634- runtime.EXPECT().StopContainer(mock.Anything, "container-2").Return(nil)
635-
633+ // Shutdown no longer stops containers — they are left running for
634+ // SyncContainers + AutoStart to pick back up on next boot.
636635 err := svc.Shutdown(ctx)
637636
638637 assert.NoError(t, err)
639- assert.Empty(t, svc.containers)
640-}
641-
642-func TestService_Shutdown_PartialFailure(t *testing.T) {
643- runtime := mocks.NewMockContainerRuntime(t)
644- envLoader := mocks.NewMockEnvLoader(t)
645- eventBus := mocks.NewMockEventPublisher(t)
646-
647- svc := NewService(runtime, envLoader, eventBus, nil, Config{})
648- ctx := testContext()
649-
650- svc.containers["app1.example.com"] = &domain.Container{ID: "container-1"}
651- svc.containers["app2.example.com"] = &domain.Container{ID: "container-2"}
652-
653- runtime.EXPECT().StopContainer(mock.Anything, "container-1").Return(nil)
654- runtime.EXPECT().StopContainer(mock.Anything, "container-2").Return(errors.New("stop failed"))
655-
656- err := svc.Shutdown(ctx)
657-
658- // Shutdown logs errors but always returns nil for graceful degradation
659- assert.NoError(t, err)
660- // One container should still be tracked (the one that failed to stop)
661- assert.Len(t, svc.containers, 1)
638+ // Containers remain tracked (not stopped).
639+ assert.Len(t, svc.containers, 2)
662640 }
663641
664642 func TestService_Restart_NotFound(t *testing.T) {
@@ -1196,14 +1174,14 @@ func TestService_AutoStart_StartsNewContainers(t *testing.T) {
11961174 {Domain: "app2.example.com", Image: "myapp2:latest"},
11971175 }
11981176
1199- // Setup mocks for first route deployment
1177+ // Setup mocks for route deployments
12001178 runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Times(2)
12011179 runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp1:latest", "myapp2:latest"}, nil).Times(2)
12021180 runtime.EXPECT().GetImageExposedPorts(mock.Anything, mock.AnythingOfType("string")).Return([]int{8080}, nil).Times(2)
12031181 envLoader.EXPECT().LoadEnv(mock.Anything, mock.AnythingOfType("string")).Return([]string{}, nil).Times(2)
12041182 runtime.EXPECT().InspectImageEnv(mock.Anything, mock.AnythingOfType("string")).Return([]string{}, nil).Times(2)
12051183
1206- // Create and start containers
1184+ // Create and start containers — readiness is skipped so no IsContainerRunning calls.
12071185 runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(&domain.Container{
12081186 ID: "container-1", Name: "gordon-app1.example.com", Status: "created",
12091187 }, nil).Once()
@@ -1211,7 +1189,6 @@ func TestService_AutoStart_StartsNewContainers(t *testing.T) {
12111189 ID: "container-2", Name: "gordon-app2.example.com", Status: "created",
12121190 }, nil).Once()
12131191 runtime.EXPECT().StartContainer(mock.Anything, mock.AnythingOfType("string")).Return(nil).Times(2)
1214- runtime.EXPECT().IsContainerRunning(mock.Anything, mock.AnythingOfType("string")).Return(true, nil).Times(4)
12151192 runtime.EXPECT().InspectContainer(mock.Anything, mock.AnythingOfType("string")).Return(&domain.Container{
12161193 Status: "running",
12171194 }, nil).Times(2)
@@ -1245,7 +1222,7 @@ func TestService_AutoStart_SkipsExistingContainers(t *testing.T) {
12451222 {Domain: "app2.example.com", Image: "myapp2:latest"}, // New route
12461223 }
12471224
1248- // Only deploy for app2 (app1 is skipped)
1225+ // Only deploy for app2 (app1 is skipped). Readiness is skipped — no IsContainerRunning.
12491226 runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Once()
12501227 runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp2:latest"}, nil).Once()
12511228 runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp2:latest").Return([]int{8080}, nil).Once()
@@ -1255,7 +1232,6 @@ func TestService_AutoStart_SkipsExistingContainers(t *testing.T) {
12551232 ID: "container-2", Status: "created",
12561233 }, nil).Once()
12571234 runtime.EXPECT().StartContainer(mock.Anything, "container-2").Return(nil).Once()
1258- runtime.EXPECT().IsContainerRunning(mock.Anything, "container-2").Return(true, nil).Times(2)
12591235 runtime.EXPECT().InspectContainer(mock.Anything, "container-2").Return(&domain.Container{
12601236 Status: "running",
12611237 }, nil).Once()
@@ -1313,7 +1289,8 @@ func TestService_AutoStart_UsesInternalDeployContext(t *testing.T) {
13131289 }
13141290
13151291 // Key assertion: PullImage should be called with localhost:5000 rewrite,
1316- // NOT the original reg.example.com/myapp:latest
1292+ // NOT the original reg.example.com/myapp:latest.
1293+ // Readiness is skipped — no IsContainerRunning calls.
13171294 runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil)
13181295 runtime.EXPECT().PullImage(mock.Anything, "localhost:5000/myapp:latest").Return(nil)
13191296 runtime.EXPECT().TagImage(mock.Anything, "localhost:5000/myapp:latest", "reg.example.com/myapp:latest").Return(nil)
@@ -1325,7 +1302,6 @@ func TestService_AutoStart_UsesInternalDeployContext(t *testing.T) {
13251302 ID: "container-1", Name: "gordon-app1.example.com", Status: "created",
13261303 }, nil)
13271304 runtime.EXPECT().StartContainer(mock.Anything, "container-1").Return(nil)
1328- runtime.EXPECT().IsContainerRunning(mock.Anything, "container-1").Return(true, nil).Times(2)
13291305 runtime.EXPECT().InspectContainer(mock.Anything, "container-1").Return(&domain.Container{
13301306 Status: "running",
13311307 }, nil)
@@ -630,35 +630,13 @@ func TestService_Shutdown(t *testing.T) {
630 svc.containers["app1.example.com"] = &domain.Container{ID: "container-1"}630 svc.containers["app1.example.com"] = &domain.Container{ID: "container-1"}
631 svc.containers["app2.example.com"] = &domain.Container{ID: "container-2"}631 svc.containers["app2.example.com"] = &domain.Container{ID: "container-2"}
632 632
633- runtime.EXPECT().StopContainer(mock.Anything, "container-1").Return(nil)633+ // Shutdown no longer stops containers — they are left running for
634- runtime.EXPECT().StopContainer(mock.Anything, "container-2").Return(nil)634+ // SyncContainers + AutoStart to pick back up on next boot.
635-
636 err := svc.Shutdown(ctx)635 err := svc.Shutdown(ctx)
637 636
638 assert.NoError(t, err)637 assert.NoError(t, err)
639- assert.Empty(t, svc.containers)638+ // Containers remain tracked (not stopped).
640-}639+ assert.Len(t, svc.containers, 2)
641-
642-func TestService_Shutdown_PartialFailure(t *testing.T) {
643- runtime := mocks.NewMockContainerRuntime(t)
644- envLoader := mocks.NewMockEnvLoader(t)
645- eventBus := mocks.NewMockEventPublisher(t)
646-
647- svc := NewService(runtime, envLoader, eventBus, nil, Config{})
648- ctx := testContext()
649-
650- svc.containers["app1.example.com"] = &domain.Container{ID: "container-1"}
651- svc.containers["app2.example.com"] = &domain.Container{ID: "container-2"}
652-
653- runtime.EXPECT().StopContainer(mock.Anything, "container-1").Return(nil)
654- runtime.EXPECT().StopContainer(mock.Anything, "container-2").Return(errors.New("stop failed"))
655-
656- err := svc.Shutdown(ctx)
657-
658- // Shutdown logs errors but always returns nil for graceful degradation
659- assert.NoError(t, err)
660- // One container should still be tracked (the one that failed to stop)
661- assert.Len(t, svc.containers, 1)
662 }640 }
663 641
664 func TestService_Restart_NotFound(t *testing.T) {642 func TestService_Restart_NotFound(t *testing.T) {
@@ -1196,14 +1174,14 @@ func TestService_AutoStart_StartsNewContainers(t *testing.T) {
1196 {Domain: "app2.example.com", Image: "myapp2:latest"},1174 {Domain: "app2.example.com", Image: "myapp2:latest"},
1197 }1175 }
1198 1176
1199- // Setup mocks for first route deployment1177+ // Setup mocks for route deployments
1200 runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Times(2)1178 runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Times(2)
1201 runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp1:latest", "myapp2:latest"}, nil).Times(2)1179 runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp1:latest", "myapp2:latest"}, nil).Times(2)
1202 runtime.EXPECT().GetImageExposedPorts(mock.Anything, mock.AnythingOfType("string")).Return([]int{8080}, nil).Times(2)1180 runtime.EXPECT().GetImageExposedPorts(mock.Anything, mock.AnythingOfType("string")).Return([]int{8080}, nil).Times(2)
1203 envLoader.EXPECT().LoadEnv(mock.Anything, mock.AnythingOfType("string")).Return([]string{}, nil).Times(2)1181 envLoader.EXPECT().LoadEnv(mock.Anything, mock.AnythingOfType("string")).Return([]string{}, nil).Times(2)
1204 runtime.EXPECT().InspectImageEnv(mock.Anything, mock.AnythingOfType("string")).Return([]string{}, nil).Times(2)1182 runtime.EXPECT().InspectImageEnv(mock.Anything, mock.AnythingOfType("string")).Return([]string{}, nil).Times(2)
1205 1183
1206- // Create and start containers1184+ // Create and start containers — readiness is skipped so no IsContainerRunning calls.
1207 runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(&domain.Container{1185 runtime.EXPECT().CreateContainer(mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Return(&domain.Container{
1208 ID: "container-1", Name: "gordon-app1.example.com", Status: "created",1186 ID: "container-1", Name: "gordon-app1.example.com", Status: "created",
1209 }, nil).Once()1187 }, nil).Once()
@@ -1211,7 +1189,6 @@ func TestService_AutoStart_StartsNewContainers(t *testing.T) {
1211 ID: "container-2", Name: "gordon-app2.example.com", Status: "created",1189 ID: "container-2", Name: "gordon-app2.example.com", Status: "created",
1212 }, nil).Once()1190 }, nil).Once()
1213 runtime.EXPECT().StartContainer(mock.Anything, mock.AnythingOfType("string")).Return(nil).Times(2)1191 runtime.EXPECT().StartContainer(mock.Anything, mock.AnythingOfType("string")).Return(nil).Times(2)
1214- runtime.EXPECT().IsContainerRunning(mock.Anything, mock.AnythingOfType("string")).Return(true, nil).Times(4)
1215 runtime.EXPECT().InspectContainer(mock.Anything, mock.AnythingOfType("string")).Return(&domain.Container{1192 runtime.EXPECT().InspectContainer(mock.Anything, mock.AnythingOfType("string")).Return(&domain.Container{
1216 Status: "running",1193 Status: "running",
1217 }, nil).Times(2)1194 }, nil).Times(2)
@@ -1245,7 +1222,7 @@ func TestService_AutoStart_SkipsExistingContainers(t *testing.T) {
1245 {Domain: "app2.example.com", Image: "myapp2:latest"}, // New route1222 {Domain: "app2.example.com", Image: "myapp2:latest"}, // New route
1246 }1223 }
1247 1224
1248- // Only deploy for app2 (app1 is skipped)1225+ // Only deploy for app2 (app1 is skipped). Readiness is skipped — no IsContainerRunning.
1249 runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Once()1226 runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil).Once()
1250 runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp2:latest"}, nil).Once()1227 runtime.EXPECT().ListImages(mock.Anything).Return([]string{"myapp2:latest"}, nil).Once()
1251 runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp2:latest").Return([]int{8080}, nil).Once()1228 runtime.EXPECT().GetImageExposedPorts(mock.Anything, "myapp2:latest").Return([]int{8080}, nil).Once()
@@ -1255,7 +1232,6 @@ func TestService_AutoStart_SkipsExistingContainers(t *testing.T) {
1255 ID: "container-2", Status: "created",1232 ID: "container-2", Status: "created",
1256 }, nil).Once()1233 }, nil).Once()
1257 runtime.EXPECT().StartContainer(mock.Anything, "container-2").Return(nil).Once()1234 runtime.EXPECT().StartContainer(mock.Anything, "container-2").Return(nil).Once()
1258- runtime.EXPECT().IsContainerRunning(mock.Anything, "container-2").Return(true, nil).Times(2)
1259 runtime.EXPECT().InspectContainer(mock.Anything, "container-2").Return(&domain.Container{1235 runtime.EXPECT().InspectContainer(mock.Anything, "container-2").Return(&domain.Container{
1260 Status: "running",1236 Status: "running",
1261 }, nil).Once()1237 }, nil).Once()
@@ -1313,7 +1289,8 @@ func TestService_AutoStart_UsesInternalDeployContext(t *testing.T) {
1313 }1289 }
1314 1290
1315 // Key assertion: PullImage should be called with localhost:5000 rewrite,1291 // Key assertion: PullImage should be called with localhost:5000 rewrite,
1316- // NOT the original reg.example.com/myapp:latest1292+ // NOT the original reg.example.com/myapp:latest.
1293+ // Readiness is skipped — no IsContainerRunning calls.
1317 runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil)1294 runtime.EXPECT().ListContainers(mock.Anything, true).Return([]*domain.Container{}, nil)
1318 runtime.EXPECT().PullImage(mock.Anything, "localhost:5000/myapp:latest").Return(nil)1295 runtime.EXPECT().PullImage(mock.Anything, "localhost:5000/myapp:latest").Return(nil)
1319 runtime.EXPECT().TagImage(mock.Anything, "localhost:5000/myapp:latest", "reg.example.com/myapp:latest").Return(nil)1296 runtime.EXPECT().TagImage(mock.Anything, "localhost:5000/myapp:latest", "reg.example.com/myapp:latest").Return(nil)
@@ -1325,7 +1302,6 @@ func TestService_AutoStart_UsesInternalDeployContext(t *testing.T) {
1325 ID: "container-1", Name: "gordon-app1.example.com", Status: "created",1302 ID: "container-1", Name: "gordon-app1.example.com", Status: "created",
1326 }, nil)1303 }, nil)
1327 runtime.EXPECT().StartContainer(mock.Anything, "container-1").Return(nil)1304 runtime.EXPECT().StartContainer(mock.Anything, "container-1").Return(nil)
1328- runtime.EXPECT().IsContainerRunning(mock.Anything, "container-1").Return(true, nil).Times(2)
1329 runtime.EXPECT().InspectContainer(mock.Anything, "container-1").Return(&domain.Container{1305 runtime.EXPECT().InspectContainer(mock.Anything, "container-1").Return(&domain.Container{
1330 Status: "running",1306 Status: "running",
1331 }, nil)1307 }, nil)
M internal/usecase/logs/service.go +243 -57
@@ -3,11 +3,14 @@ package logs
33
44 import (
55 "bufio"
6+ "bytes"
67 "context"
78 "encoding/binary"
89 "fmt"
910 "io"
1011 "os"
12+ "os/exec"
13+ "strconv"
1114 "strings"
1215 "time"
1316
@@ -19,24 +22,29 @@ import (
1922
2023 // Service implements the LogService interface.
2124 type Service struct {
22- logFilePath string
23- containerSvc in.ContainerService
24- runtime out.ContainerRuntime
25- log zerowrap.Logger
25+ logFilePath string
26+ fileLoggingEnabled bool
27+ containerSvc in.ContainerService
28+ runtime out.ContainerRuntime
29+ log zerowrap.Logger
2630 }
2731
32+var execCommandContext = exec.CommandContext
33+
2834 // NewService creates a new log service.
2935 func NewService(
3036 logFilePath string,
37+ fileLoggingEnabled bool,
3138 containerSvc in.ContainerService,
3239 runtime out.ContainerRuntime,
3340 log zerowrap.Logger,
3441 ) *Service {
3542 return &Service{
36- logFilePath: logFilePath,
37- containerSvc: containerSvc,
38- runtime: runtime,
39- log: log,
43+ logFilePath: logFilePath,
44+ fileLoggingEnabled: fileLoggingEnabled,
45+ containerSvc: containerSvc,
46+ runtime: runtime,
47+ log: log,
4048 }
4149 }
4250
@@ -49,20 +57,26 @@ func (s *Service) GetProcessLogs(ctx context.Context, lines int) ([]string, erro
4957 })
5058 log := zerowrap.FromCtx(ctx)
5159
52- if s.logFilePath == "" {
53- return nil, fmt.Errorf("log file path not configured")
54- }
60+ if s.useFileProcessLogs() {
61+ if s.logFilePath == "" {
62+ return nil, fmt.Errorf("log file path not configured")
63+ }
5564
56- file, err := os.Open(s.logFilePath)
57- if err != nil {
58- if os.IsNotExist(err) {
59- return []string{}, nil
65+ file, err := os.Open(s.logFilePath)
66+ if err != nil {
67+ if os.IsNotExist(err) {
68+ log.Warn().Str("log_path", s.logFilePath).
69+ Msg("process log file missing, falling back to journalctl")
70+ return s.getProcessLogsFromJournal(ctx, lines)
71+ }
72+ return nil, log.WrapErr(err, "failed to open log file")
6073 }
61- return nil, log.WrapErr(err, "failed to open log file")
74+ defer file.Close()
75+
76+ return tailLines(file, lines)
6277 }
63- defer file.Close()
6478
65- return tailLines(file, lines)
79+ return s.getProcessLogsFromJournal(ctx, lines)
6680 }
6781
6882 // FollowProcessLogs returns a channel that streams Gordon process log lines.
@@ -72,67 +86,198 @@ func (s *Service) FollowProcessLogs(ctx context.Context, initialLines int) (<-ch
7286 zerowrap.FieldUseCase: "FollowProcessLogs",
7387 "initial_lines": initialLines,
7488 })
89+ if !s.useFileProcessLogs() {
90+ return s.followProcessLogsFromJournal(ctx, initialLines)
91+ }
92+
93+ return s.followProcessLogsFromFileOrJournal(ctx, initialLines)
94+}
95+
96+func (s *Service) useFileProcessLogs() bool {
97+ return s.fileLoggingEnabled
98+}
99+
100+func (s *Service) followProcessLogsFromFileOrJournal(
101+ ctx context.Context,
102+ initialLines int,
103+) (<-chan string, error) {
75104 log := zerowrap.FromCtx(ctx)
76105
106+ file, err := s.openProcessLogFile()
107+ if err != nil {
108+ if os.IsNotExist(err) {
109+ log.Warn().Str("log_path", s.logFilePath).
110+ Msg("process log file missing, falling back to journalctl follow")
111+ return s.followProcessLogsFromJournal(ctx, initialLines)
112+ }
113+ return nil, log.WrapErr(err, "failed to open log file")
114+ }
115+
116+ return s.followProcessLogsFromFile(ctx, file, initialLines), nil
117+}
118+
119+func (s *Service) openProcessLogFile() (*os.File, error) {
77120 if s.logFilePath == "" {
78121 return nil, fmt.Errorf("log file path not configured")
79122 }
80123
81- file, err := os.Open(s.logFilePath)
82- if err != nil {
83- return nil, log.WrapErr(err, "failed to open log file")
84- }
124+ return os.Open(s.logFilePath)
125+}
85126
127+func (s *Service) followProcessLogsFromFile(
128+ ctx context.Context,
129+ file *os.File,
130+ initialLines int,
131+) <-chan string {
132+ log := zerowrap.FromCtx(ctx)
86133 ch := make(chan string, 100)
87134
88135 go func() {
89136 defer close(ch)
90137 defer file.Close()
91138
92- // First, get initial lines
93- if initialLines > 0 {
94- lines, err := tailLines(file, initialLines)
95- if err != nil {
96- log.Warn().Err(err).Msg("failed to read initial lines")
97- } else {
98- for _, line := range lines {
99- select {
100- case ch <- line:
101- case <-ctx.Done():
102- return
103- }
104- }
139+ if !s.emitInitialFileLogs(ctx, file, initialLines, ch) {
140+ return
141+ }
142+ if err := s.seekToEndForFileFollow(file); err != nil {
143+ log.Warn().Err(err).Msg("failed to seek to end")
144+ return
145+ }
146+
147+ s.streamFileFollowLogs(ctx, file, ch)
148+ }()
149+
150+ return ch
151+}
152+
153+func (s *Service) emitInitialFileLogs(
154+ ctx context.Context,
155+ file *os.File,
156+ initialLines int,
157+ ch chan<- string,
158+) bool {
159+ if initialLines <= 0 {
160+ return true
161+ }
162+
163+ log := zerowrap.FromCtx(ctx)
164+ lines, err := tailLines(file, initialLines)
165+ if err != nil {
166+ log.Warn().Err(err).Msg("failed to read initial lines")
167+ return true
168+ }
169+
170+ for _, line := range lines {
171+ if !sendLogLineWithChannel(ctx, ch, line) {
172+ return false
173+ }
174+ }
175+
176+ return true
177+}
178+
179+func (s *Service) seekToEndForFileFollow(file *os.File) error {
180+ _, err := file.Seek(0, io.SeekEnd)
181+ return err
182+}
183+
184+func (s *Service) streamFileFollowLogs(ctx context.Context, file *os.File, ch chan<- string) {
185+ log := zerowrap.FromCtx(ctx)
186+ reader := bufio.NewReader(file)
187+
188+ for {
189+ if ctx.Err() != nil {
190+ return
191+ }
192+
193+ line, err := reader.ReadString('\n')
194+ if err != nil {
195+ if err == io.EOF {
196+ time.Sleep(100 * time.Millisecond)
197+ continue
105198 }
199+ log.Warn().Err(err).Msg("error reading log file")
200+ return
106201 }
107202
108- // Seek to end for following
109- if _, err := file.Seek(0, io.SeekEnd); err != nil {
110- log.Warn().Err(err).Msg("failed to seek to end")
203+ line = strings.TrimRight(line, "\n\r")
204+ if !sendLogLineWithChannel(ctx, ch, line) {
111205 return
112206 }
207+ }
208+}
209+
210+func sendLogLineWithChannel(ctx context.Context, ch chan<- string, line string) bool {
211+ select {
212+ case ch <- line:
213+ return true
214+ case <-ctx.Done():
215+ return false
216+ }
217+}
218+
219+func (s *Service) getProcessLogsFromJournal(ctx context.Context, lines int) ([]string, error) {
220+ userScope, err := resolveJournalctlScope(ctx)
221+ if err != nil {
222+ return nil, err
223+ }
224+
225+ args := []string{"-u", "gordon", "-n", strconv.Itoa(lines), "--no-pager", "-o", "cat"}
226+ if userScope {
227+ args = append([]string{"--user"}, args...)
228+ }
113229
114- // Follow new lines
115- reader := bufio.NewReader(file)
116- for {
230+ out, err := execCommandContext(ctx, "journalctl", args...).Output()
231+ if err != nil {
232+ return nil, fmt.Errorf("failed to read journalctl logs: %w", err)
233+ }
234+
235+ return outputLines(out), nil
236+}
237+
238+func (s *Service) followProcessLogsFromJournal(ctx context.Context, initialLines int) (<-chan string, error) {
239+ userScope, err := resolveJournalctlScope(ctx)
240+ if err != nil {
241+ return nil, err
242+ }
243+
244+ args := []string{
245+ "-u", "gordon",
246+ "-n", strconv.Itoa(initialLines),
247+ "--no-pager",
248+ "-o", "cat",
249+ "-f",
250+ }
251+ if userScope {
252+ args = append([]string{"--user"}, args...)
253+ }
254+
255+ cmd := execCommandContext(ctx, "journalctl", args...) // #nosec G204 -- internal args only
256+ stdout, err := cmd.StdoutPipe()
257+ if err != nil {
258+ return nil, fmt.Errorf("failed to open journalctl stdout: %w", err)
259+ }
260+ cmd.Stderr = io.Discard
261+
262+ if err := cmd.Start(); err != nil {
263+ return nil, fmt.Errorf("failed to start journalctl follow: %w", err)
264+ }
265+
266+ ch := make(chan string, 100)
267+
268+ go func() {
269+ defer close(ch)
270+ defer stdout.Close()
271+ defer cmd.Wait() //nolint:errcheck // best-effort cleanup
272+
273+ scanner := bufio.NewScanner(stdout)
274+ scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
275+
276+ for scanner.Scan() {
117277 select {
278+ case ch <- scanner.Text():
118279 case <-ctx.Done():
119280 return
120- default:
121- line, err := reader.ReadString('\n')
122- if err != nil {
123- if err == io.EOF {
124- time.Sleep(100 * time.Millisecond)
125- continue
126- }
127- log.Warn().Err(err).Msg("error reading log file")
128- return
129- }
130- line = strings.TrimRight(line, "\n\r")
131- select {
132- case ch <- line:
133- case <-ctx.Done():
134- return
135- }
136281 }
137282 }
138283 }()
@@ -140,6 +285,47 @@ func (s *Service) FollowProcessLogs(ctx context.Context, initialLines int) (<-ch
140285 return ch, nil
141286 }
142287
288+func resolveJournalctlScope(ctx context.Context) (bool, error) {
289+ userLogs, userErr := hasJournalctlLogs(ctx, true)
290+ if userErr == nil && userLogs {
291+ return true, nil
292+ }
293+
294+ systemLogs, systemErr := hasJournalctlLogs(ctx, false)
295+ if systemErr == nil && systemLogs {
296+ return false, nil
297+ }
298+
299+ if userErr != nil && systemErr != nil {
300+ return false, fmt.Errorf("journalctl unavailable for user and system services")
301+ }
302+
303+ // Default to user scope when logs are currently empty; this matches rootless setups.
304+ return true, nil
305+}
306+
307+func hasJournalctlLogs(ctx context.Context, userService bool) (bool, error) {
308+ args := []string{"-u", "gordon", "-n", "1", "--no-pager", "-q", "-o", "cat"}
309+ if userService {
310+ args = append([]string{"--user"}, args...)
311+ }
312+
313+ out, err := execCommandContext(ctx, "journalctl", args...).Output()
314+ if err != nil {
315+ return false, err
316+ }
317+
318+ return len(bytes.TrimSpace(out)) > 0, nil
319+}
320+
321+func outputLines(out []byte) []string {
322+ trimmed := strings.TrimRight(string(out), "\n\r")
323+ if trimmed == "" {
324+ return []string{}
325+ }
326+ return strings.Split(trimmed, "\n")
327+}
328+
143329 // validateDomain checks that domain is safe for use in logs and error messages.
144330 func validateDomain(domain string) error {
145331 if domain == "" {
@@ -3,11 +3,14 @@ package logs
3 3
4 import (4 import (
5 "bufio"5 "bufio"
6+ "bytes"
6 "context"7 "context"
7 "encoding/binary"8 "encoding/binary"
8 "fmt"9 "fmt"
9 "io"10 "io"
10 "os"11 "os"
12+ "os/exec"
13+ "strconv"
11 "strings"14 "strings"
12 "time"15 "time"
13 16
@@ -19,24 +22,29 @@ import (
19 22
20 // Service implements the LogService interface.23 // Service implements the LogService interface.
21 type Service struct {24 type Service struct {
22- logFilePath string25+ logFilePath string
23- containerSvc in.ContainerService26+ fileLoggingEnabled bool
24- runtime out.ContainerRuntime27+ containerSvc in.ContainerService
25- log zerowrap.Logger28+ runtime out.ContainerRuntime
29+ log zerowrap.Logger
26 }30 }
27 31
32+var execCommandContext = exec.CommandContext
33+
28 // NewService creates a new log service.34 // NewService creates a new log service.
29 func NewService(35 func NewService(
30 logFilePath string,36 logFilePath string,
37+ fileLoggingEnabled bool,
31 containerSvc in.ContainerService,38 containerSvc in.ContainerService,
32 runtime out.ContainerRuntime,39 runtime out.ContainerRuntime,
33 log zerowrap.Logger,40 log zerowrap.Logger,
34 ) *Service {41 ) *Service {
35 return &Service{42 return &Service{
36- logFilePath: logFilePath,43+ logFilePath: logFilePath,
37- containerSvc: containerSvc,44+ fileLoggingEnabled: fileLoggingEnabled,
38- runtime: runtime,45+ containerSvc: containerSvc,
39- log: log,46+ runtime: runtime,
47+ log: log,
40 }48 }
41 }49 }
42 50
@@ -49,20 +57,26 @@ func (s *Service) GetProcessLogs(ctx context.Context, lines int) ([]string, erro
49 })57 })
50 log := zerowrap.FromCtx(ctx)58 log := zerowrap.FromCtx(ctx)
51 59
52- if s.logFilePath == "" {60+ if s.useFileProcessLogs() {
53- return nil, fmt.Errorf("log file path not configured")61+ if s.logFilePath == "" {
54- }62+ return nil, fmt.Errorf("log file path not configured")
63+ }
55 64
56- file, err := os.Open(s.logFilePath)65+ file, err := os.Open(s.logFilePath)
57- if err != nil {66+ if err != nil {
58- if os.IsNotExist(err) {67+ if os.IsNotExist(err) {
59- return []string{}, nil68+ log.Warn().Str("log_path", s.logFilePath).
69+ Msg("process log file missing, falling back to journalctl")
70+ return s.getProcessLogsFromJournal(ctx, lines)
71+ }
72+ return nil, log.WrapErr(err, "failed to open log file")
60 }73 }
61- return nil, log.WrapErr(err, "failed to open log file")74+ defer file.Close()
75+
76+ return tailLines(file, lines)
62 }77 }
63- defer file.Close()
64 78
65- return tailLines(file, lines)79+ return s.getProcessLogsFromJournal(ctx, lines)
66 }80 }
67 81
68 // FollowProcessLogs returns a channel that streams Gordon process log lines.82 // FollowProcessLogs returns a channel that streams Gordon process log lines.
@@ -72,67 +86,198 @@ func (s *Service) FollowProcessLogs(ctx context.Context, initialLines int) (<-ch
72 zerowrap.FieldUseCase: "FollowProcessLogs",86 zerowrap.FieldUseCase: "FollowProcessLogs",
73 "initial_lines": initialLines,87 "initial_lines": initialLines,
74 })88 })
89+ if !s.useFileProcessLogs() {
90+ return s.followProcessLogsFromJournal(ctx, initialLines)
91+ }
92+
93+ return s.followProcessLogsFromFileOrJournal(ctx, initialLines)
94+}
95+
96+func (s *Service) useFileProcessLogs() bool {
97+ return s.fileLoggingEnabled
98+}
99+
100+func (s *Service) followProcessLogsFromFileOrJournal(
101+ ctx context.Context,
102+ initialLines int,
103+) (<-chan string, error) {
75 log := zerowrap.FromCtx(ctx)104 log := zerowrap.FromCtx(ctx)
76 105
106+ file, err := s.openProcessLogFile()
107+ if err != nil {
108+ if os.IsNotExist(err) {
109+ log.Warn().Str("log_path", s.logFilePath).
110+ Msg("process log file missing, falling back to journalctl follow")
111+ return s.followProcessLogsFromJournal(ctx, initialLines)
112+ }
113+ return nil, log.WrapErr(err, "failed to open log file")
114+ }
115+
116+ return s.followProcessLogsFromFile(ctx, file, initialLines), nil
117+}
118+
119+func (s *Service) openProcessLogFile() (*os.File, error) {
77 if s.logFilePath == "" {120 if s.logFilePath == "" {
78 return nil, fmt.Errorf("log file path not configured")121 return nil, fmt.Errorf("log file path not configured")
79 }122 }
80 123
81- file, err := os.Open(s.logFilePath)124+ return os.Open(s.logFilePath)
82- if err != nil {125+}
83- return nil, log.WrapErr(err, "failed to open log file")
84- }
85 126
127+func (s *Service) followProcessLogsFromFile(
128+ ctx context.Context,
129+ file *os.File,
130+ initialLines int,
131+) <-chan string {
132+ log := zerowrap.FromCtx(ctx)
86 ch := make(chan string, 100)133 ch := make(chan string, 100)
87 134
88 go func() {135 go func() {
89 defer close(ch)136 defer close(ch)
90 defer file.Close()137 defer file.Close()
91 138
92- // First, get initial lines139+ if !s.emitInitialFileLogs(ctx, file, initialLines, ch) {
93- if initialLines > 0 {140+ return
94- lines, err := tailLines(file, initialLines)141+ }
95- if err != nil {142+ if err := s.seekToEndForFileFollow(file); err != nil {
96- log.Warn().Err(err).Msg("failed to read initial lines")143+ log.Warn().Err(err).Msg("failed to seek to end")
97- } else {144+ return
98- for _, line := range lines {145+ }
99- select {146+
100- case ch <- line:147+ s.streamFileFollowLogs(ctx, file, ch)
101- case <-ctx.Done():148+ }()
102- return149+
103- }150+ return ch
104- }151+}
152+
153+func (s *Service) emitInitialFileLogs(
154+ ctx context.Context,
155+ file *os.File,
156+ initialLines int,
157+ ch chan<- string,
158+) bool {
159+ if initialLines <= 0 {
160+ return true
161+ }
162+
163+ log := zerowrap.FromCtx(ctx)
164+ lines, err := tailLines(file, initialLines)
165+ if err != nil {
166+ log.Warn().Err(err).Msg("failed to read initial lines")
167+ return true
168+ }
169+
170+ for _, line := range lines {
171+ if !sendLogLineWithChannel(ctx, ch, line) {
172+ return false
173+ }
174+ }
175+
176+ return true
177+}
178+
179+func (s *Service) seekToEndForFileFollow(file *os.File) error {
180+ _, err := file.Seek(0, io.SeekEnd)
181+ return err
182+}
183+
184+func (s *Service) streamFileFollowLogs(ctx context.Context, file *os.File, ch chan<- string) {
185+ log := zerowrap.FromCtx(ctx)
186+ reader := bufio.NewReader(file)
187+
188+ for {
189+ if ctx.Err() != nil {
190+ return
191+ }
192+
193+ line, err := reader.ReadString('\n')
194+ if err != nil {
195+ if err == io.EOF {
196+ time.Sleep(100 * time.Millisecond)
197+ continue
105 }198 }
199+ log.Warn().Err(err).Msg("error reading log file")
200+ return
106 }201 }
107 202
108- // Seek to end for following203+ line = strings.TrimRight(line, "\n\r")
109- if _, err := file.Seek(0, io.SeekEnd); err != nil {204+ if !sendLogLineWithChannel(ctx, ch, line) {
110- log.Warn().Err(err).Msg("failed to seek to end")
111 return205 return
112 }206 }
207+ }
208+}
209+
210+func sendLogLineWithChannel(ctx context.Context, ch chan<- string, line string) bool {
211+ select {
212+ case ch <- line:
213+ return true
214+ case <-ctx.Done():
215+ return false
216+ }
217+}
218+
219+func (s *Service) getProcessLogsFromJournal(ctx context.Context, lines int) ([]string, error) {
220+ userScope, err := resolveJournalctlScope(ctx)
221+ if err != nil {
222+ return nil, err
223+ }
224+
225+ args := []string{"-u", "gordon", "-n", strconv.Itoa(lines), "--no-pager", "-o", "cat"}
226+ if userScope {
227+ args = append([]string{"--user"}, args...)
228+ }
113 229
114- // Follow new lines230+ out, err := execCommandContext(ctx, "journalctl", args...).Output()
115- reader := bufio.NewReader(file)231+ if err != nil {
116- for {232+ return nil, fmt.Errorf("failed to read journalctl logs: %w", err)
233+ }
234+
235+ return outputLines(out), nil
236+}
237+
238+func (s *Service) followProcessLogsFromJournal(ctx context.Context, initialLines int) (<-chan string, error) {
239+ userScope, err := resolveJournalctlScope(ctx)
240+ if err != nil {
241+ return nil, err
242+ }
243+
244+ args := []string{
245+ "-u", "gordon",
246+ "-n", strconv.Itoa(initialLines),
247+ "--no-pager",
248+ "-o", "cat",
249+ "-f",
250+ }
251+ if userScope {
252+ args = append([]string{"--user"}, args...)
253+ }
254+
255+ cmd := execCommandContext(ctx, "journalctl", args...) // #nosec G204 -- internal args only
256+ stdout, err := cmd.StdoutPipe()
257+ if err != nil {
258+ return nil, fmt.Errorf("failed to open journalctl stdout: %w", err)
259+ }
260+ cmd.Stderr = io.Discard
261+
262+ if err := cmd.Start(); err != nil {
263+ return nil, fmt.Errorf("failed to start journalctl follow: %w", err)
264+ }
265+
266+ ch := make(chan string, 100)
267+
268+ go func() {
269+ defer close(ch)
270+ defer stdout.Close()
271+ defer cmd.Wait() //nolint:errcheck // best-effort cleanup
272+
273+ scanner := bufio.NewScanner(stdout)
274+ scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
275+
276+ for scanner.Scan() {
117 select {277 select {
278+ case ch <- scanner.Text():
118 case <-ctx.Done():279 case <-ctx.Done():
119 return280 return
120- default:
121- line, err := reader.ReadString('\n')
122- if err != nil {
123- if err == io.EOF {
124- time.Sleep(100 * time.Millisecond)
125- continue
126- }
127- log.Warn().Err(err).Msg("error reading log file")
128- return
129- }
130- line = strings.TrimRight(line, "\n\r")
131- select {
132- case ch <- line:
133- case <-ctx.Done():
134- return
135- }
136 }281 }
137 }282 }
138 }()283 }()
@@ -140,6 +285,47 @@ func (s *Service) FollowProcessLogs(ctx context.Context, initialLines int) (<-ch
140 return ch, nil285 return ch, nil
141 }286 }
142 287
288+func resolveJournalctlScope(ctx context.Context) (bool, error) {
289+ userLogs, userErr := hasJournalctlLogs(ctx, true)
290+ if userErr == nil && userLogs {
291+ return true, nil
292+ }
293+
294+ systemLogs, systemErr := hasJournalctlLogs(ctx, false)
295+ if systemErr == nil && systemLogs {
296+ return false, nil
297+ }
298+
299+ if userErr != nil && systemErr != nil {
300+ return false, fmt.Errorf("journalctl unavailable for user and system services")
301+ }
302+
303+ // Default to user scope when logs are currently empty; this matches rootless setups.
304+ return true, nil
305+}
306+
307+func hasJournalctlLogs(ctx context.Context, userService bool) (bool, error) {
308+ args := []string{"-u", "gordon", "-n", "1", "--no-pager", "-q", "-o", "cat"}
309+ if userService {
310+ args = append([]string{"--user"}, args...)
311+ }
312+
313+ out, err := execCommandContext(ctx, "journalctl", args...).Output()
314+ if err != nil {
315+ return false, err
316+ }
317+
318+ return len(bytes.TrimSpace(out)) > 0, nil
319+}
320+
321+func outputLines(out []byte) []string {
322+ trimmed := strings.TrimRight(string(out), "\n\r")
323+ if trimmed == "" {
324+ return []string{}
325+ }
326+ return strings.Split(trimmed, "\n")
327+}
328+
143 // validateDomain checks that domain is safe for use in logs and error messages.329 // validateDomain checks that domain is safe for use in logs and error messages.
144 func validateDomain(domain string) error {330 func validateDomain(domain string) error {
145 if domain == "" {331 if domain == "" {
M internal/usecase/logs/service_test.go +28 -8
@@ -4,6 +4,7 @@ import (
44 "context"
55 "io"
66 "os"
7+ "os/exec"
78 "path/filepath"
89 "testing"
910 "time"
@@ -32,7 +33,7 @@ func TestService_GetProcessLogs(t *testing.T) {
3233 containerSvc := mocks.NewMockContainerService(t)
3334 runtime := outMocks.NewMockContainerRuntime(t)
3435
35- svc := NewService(logPath, containerSvc, runtime, log)
36+ svc := NewService(logPath, true, containerSvc, runtime, log)
3637
3738 lines, err := svc.GetProcessLogs(context.Background(), 3)
3839 require.NoError(t, err)
@@ -44,7 +45,7 @@ func TestService_GetProcessLogs(t *testing.T) {
4445 containerSvc := mocks.NewMockContainerService(t)
4546 runtime := outMocks.NewMockContainerRuntime(t)
4647
47- svc := NewService("/nonexistent/file.log", containerSvc, runtime, log)
48+ svc := NewService("/nonexistent/file.log", true, containerSvc, runtime, log)
4849
4950 lines, err := svc.GetProcessLogs(context.Background(), 10)
5051 require.NoError(t, err)
@@ -55,7 +56,7 @@ func TestService_GetProcessLogs(t *testing.T) {
5556 containerSvc := mocks.NewMockContainerService(t)
5657 runtime := outMocks.NewMockContainerRuntime(t)
5758
58- svc := NewService("", containerSvc, runtime, log)
59+ svc := NewService("", true, containerSvc, runtime, log)
5960
6061 _, err := svc.GetProcessLogs(context.Background(), 10)
6162 assert.Error(t, err)
@@ -72,12 +73,31 @@ func TestService_GetProcessLogs(t *testing.T) {
7273 containerSvc := mocks.NewMockContainerService(t)
7374 runtime := outMocks.NewMockContainerRuntime(t)
7475
75- svc := NewService(logPath, containerSvc, runtime, log)
76+ svc := NewService(logPath, true, containerSvc, runtime, log)
7677
7778 lines, err := svc.GetProcessLogs(context.Background(), 10)
7879 require.NoError(t, err)
7980 assert.Equal(t, []string{"line1", "line2"}, lines)
8081 })
82+
83+ t.Run("falls back to journalctl when file logging disabled", func(t *testing.T) {
84+ origExec := execCommandContext
85+ execCommandContext = func(ctx context.Context, name string, args ...string) *exec.Cmd {
86+ return exec.CommandContext(ctx, "bash", "-lc", "printf 'journal-line-1\\njournal-line-2\\n'") // #nosec G204
87+ }
88+ defer func() {
89+ execCommandContext = origExec
90+ }()
91+
92+ containerSvc := mocks.NewMockContainerService(t)
93+ runtime := outMocks.NewMockContainerRuntime(t)
94+
95+ svc := NewService("/tmp/unused.log", false, containerSvc, runtime, log)
96+
97+ lines, err := svc.GetProcessLogs(context.Background(), 10)
98+ require.NoError(t, err)
99+ assert.Equal(t, []string{"journal-line-1", "journal-line-2"}, lines)
100+ })
81101 }
82102
83103 func TestService_FollowProcessLogs(t *testing.T) {
@@ -93,7 +113,7 @@ func TestService_FollowProcessLogs(t *testing.T) {
93113 containerSvc := mocks.NewMockContainerService(t)
94114 runtime := outMocks.NewMockContainerRuntime(t)
95115
96- svc := NewService(logPath, containerSvc, runtime, log)
116+ svc := NewService(logPath, true, containerSvc, runtime, log)
97117
98118 ctx, cancel := context.WithCancel(context.Background())
99119 defer cancel()
@@ -135,7 +155,7 @@ func TestService_GetContainerLogs(t *testing.T) {
135155
136156 containerSvc.EXPECT().Get(mock.Anything, "unknown.local").Return(nil, false)
137157
138- svc := NewService("/tmp/test.log", containerSvc, runtime, log)
158+ svc := NewService("/tmp/test.log", true, containerSvc, runtime, log)
139159
140160 _, err := svc.GetContainerLogs(context.Background(), "unknown.local", 10)
141161 assert.Error(t, err)
@@ -156,7 +176,7 @@ func TestService_GetContainerLogs(t *testing.T) {
156176 // Create a mock reader that returns empty content
157177 runtime.EXPECT().GetContainerLogs(mock.Anything, "abc123", false).Return(&mockReader{}, nil)
158178
159- svc := NewService("/tmp/test.log", containerSvc, runtime, log)
179+ svc := NewService("/tmp/test.log", true, containerSvc, runtime, log)
160180
161181 lines, err := svc.GetContainerLogs(context.Background(), "app.local", 10)
162182 require.NoError(t, err)
@@ -173,7 +193,7 @@ func TestService_FollowContainerLogs(t *testing.T) {
173193
174194 containerSvc.EXPECT().Get(mock.Anything, "unknown.local").Return(nil, false)
175195
176- svc := NewService("/tmp/test.log", containerSvc, runtime, log)
196+ svc := NewService("/tmp/test.log", true, containerSvc, runtime, log)
177197
178198 _, err := svc.FollowContainerLogs(context.Background(), "unknown.local", 10)
179199 assert.Error(t, err)
@@ -4,6 +4,7 @@ import (
4 "context"4 "context"
5 "io"5 "io"
6 "os"6 "os"
7+ "os/exec"
7 "path/filepath"8 "path/filepath"
8 "testing"9 "testing"
9 "time"10 "time"
@@ -32,7 +33,7 @@ func TestService_GetProcessLogs(t *testing.T) {
32 containerSvc := mocks.NewMockContainerService(t)33 containerSvc := mocks.NewMockContainerService(t)
33 runtime := outMocks.NewMockContainerRuntime(t)34 runtime := outMocks.NewMockContainerRuntime(t)
34 35
35- svc := NewService(logPath, containerSvc, runtime, log)36+ svc := NewService(logPath, true, containerSvc, runtime, log)
36 37
37 lines, err := svc.GetProcessLogs(context.Background(), 3)38 lines, err := svc.GetProcessLogs(context.Background(), 3)
38 require.NoError(t, err)39 require.NoError(t, err)
@@ -44,7 +45,7 @@ func TestService_GetProcessLogs(t *testing.T) {
44 containerSvc := mocks.NewMockContainerService(t)45 containerSvc := mocks.NewMockContainerService(t)
45 runtime := outMocks.NewMockContainerRuntime(t)46 runtime := outMocks.NewMockContainerRuntime(t)
46 47
47- svc := NewService("/nonexistent/file.log", containerSvc, runtime, log)48+ svc := NewService("/nonexistent/file.log", true, containerSvc, runtime, log)
48 49
49 lines, err := svc.GetProcessLogs(context.Background(), 10)50 lines, err := svc.GetProcessLogs(context.Background(), 10)
50 require.NoError(t, err)51 require.NoError(t, err)
@@ -55,7 +56,7 @@ func TestService_GetProcessLogs(t *testing.T) {
55 containerSvc := mocks.NewMockContainerService(t)56 containerSvc := mocks.NewMockContainerService(t)
56 runtime := outMocks.NewMockContainerRuntime(t)57 runtime := outMocks.NewMockContainerRuntime(t)
57 58
58- svc := NewService("", containerSvc, runtime, log)59+ svc := NewService("", true, containerSvc, runtime, log)
59 60
60 _, err := svc.GetProcessLogs(context.Background(), 10)61 _, err := svc.GetProcessLogs(context.Background(), 10)
61 assert.Error(t, err)62 assert.Error(t, err)
@@ -72,12 +73,31 @@ func TestService_GetProcessLogs(t *testing.T) {
72 containerSvc := mocks.NewMockContainerService(t)73 containerSvc := mocks.NewMockContainerService(t)
73 runtime := outMocks.NewMockContainerRuntime(t)74 runtime := outMocks.NewMockContainerRuntime(t)
74 75
75- svc := NewService(logPath, containerSvc, runtime, log)76+ svc := NewService(logPath, true, containerSvc, runtime, log)
76 77
77 lines, err := svc.GetProcessLogs(context.Background(), 10)78 lines, err := svc.GetProcessLogs(context.Background(), 10)
78 require.NoError(t, err)79 require.NoError(t, err)
79 assert.Equal(t, []string{"line1", "line2"}, lines)80 assert.Equal(t, []string{"line1", "line2"}, lines)
80 })81 })
82+
83+ t.Run("falls back to journalctl when file logging disabled", func(t *testing.T) {
84+ origExec := execCommandContext
85+ execCommandContext = func(ctx context.Context, name string, args ...string) *exec.Cmd {
86+ return exec.CommandContext(ctx, "bash", "-lc", "printf 'journal-line-1\\njournal-line-2\\n'") // #nosec G204
87+ }
88+ defer func() {
89+ execCommandContext = origExec
90+ }()
91+
92+ containerSvc := mocks.NewMockContainerService(t)
93+ runtime := outMocks.NewMockContainerRuntime(t)
94+
95+ svc := NewService("/tmp/unused.log", false, containerSvc, runtime, log)
96+
97+ lines, err := svc.GetProcessLogs(context.Background(), 10)
98+ require.NoError(t, err)
99+ assert.Equal(t, []string{"journal-line-1", "journal-line-2"}, lines)
100+ })
81 }101 }
82 102
83 func TestService_FollowProcessLogs(t *testing.T) {103 func TestService_FollowProcessLogs(t *testing.T) {
@@ -93,7 +113,7 @@ func TestService_FollowProcessLogs(t *testing.T) {
93 containerSvc := mocks.NewMockContainerService(t)113 containerSvc := mocks.NewMockContainerService(t)
94 runtime := outMocks.NewMockContainerRuntime(t)114 runtime := outMocks.NewMockContainerRuntime(t)
95 115
96- svc := NewService(logPath, containerSvc, runtime, log)116+ svc := NewService(logPath, true, containerSvc, runtime, log)
97 117
98 ctx, cancel := context.WithCancel(context.Background())118 ctx, cancel := context.WithCancel(context.Background())
99 defer cancel()119 defer cancel()
@@ -135,7 +155,7 @@ func TestService_GetContainerLogs(t *testing.T) {
135 155
136 containerSvc.EXPECT().Get(mock.Anything, "unknown.local").Return(nil, false)156 containerSvc.EXPECT().Get(mock.Anything, "unknown.local").Return(nil, false)
137 157
138- svc := NewService("/tmp/test.log", containerSvc, runtime, log)158+ svc := NewService("/tmp/test.log", true, containerSvc, runtime, log)
139 159
140 _, err := svc.GetContainerLogs(context.Background(), "unknown.local", 10)160 _, err := svc.GetContainerLogs(context.Background(), "unknown.local", 10)
141 assert.Error(t, err)161 assert.Error(t, err)
@@ -156,7 +176,7 @@ func TestService_GetContainerLogs(t *testing.T) {
156 // Create a mock reader that returns empty content176 // Create a mock reader that returns empty content
157 runtime.EXPECT().GetContainerLogs(mock.Anything, "abc123", false).Return(&mockReader{}, nil)177 runtime.EXPECT().GetContainerLogs(mock.Anything, "abc123", false).Return(&mockReader{}, nil)
158 178
159- svc := NewService("/tmp/test.log", containerSvc, runtime, log)179+ svc := NewService("/tmp/test.log", true, containerSvc, runtime, log)
160 180
161 lines, err := svc.GetContainerLogs(context.Background(), "app.local", 10)181 lines, err := svc.GetContainerLogs(context.Background(), "app.local", 10)
162 require.NoError(t, err)182 require.NoError(t, err)
@@ -173,7 +193,7 @@ func TestService_FollowContainerLogs(t *testing.T) {
173 193
174 containerSvc.EXPECT().Get(mock.Anything, "unknown.local").Return(nil, false)194 containerSvc.EXPECT().Get(mock.Anything, "unknown.local").Return(nil, false)
175 195
176- svc := NewService("/tmp/test.log", containerSvc, runtime, log)196+ svc := NewService("/tmp/test.log", true, containerSvc, runtime, log)
177 197
178 _, err := svc.FollowContainerLogs(context.Background(), "unknown.local", 10)198 _, err := svc.FollowContainerLogs(context.Background(), "unknown.local", 10)
179 assert.Error(t, err)199 assert.Error(t, err)
M internal/usecase/proxy/service.go +27 -8
@@ -24,8 +24,8 @@ import (
2424 "github.com/bnema/gordon/internal/domain"
2525 )
2626
27-// proxyTransport is a shared HTTP transport with proper timeouts.
28-// This prevents resource exhaustion from slow backends or network issues.
27+// proxyTransport is a shared HTTP transport for proxying to application containers.
28+// ResponseHeaderTimeout is kept short to detect unresponsive backends quickly.
2929 var proxyTransport = &http.Transport{
3030 DialContext: (&net.Dialer{
3131 Timeout: 10 * time.Second,
@@ -38,6 +38,21 @@ var proxyTransport = &http.Transport{
3838 IdleConnTimeout: 90 * time.Second,
3939 }
4040
41+// registryTransport is used for the registry reverse proxy (localhost loopback).
42+// It needs a longer ResponseHeaderTimeout because admin endpoints like /admin/deploy
43+// perform blocking operations (image pull, container start, readiness checks) that
44+// can take well over 30 seconds.
45+var registryTransport = &http.Transport{
46+ DialContext: (&net.Dialer{
47+ Timeout: 10 * time.Second,
48+ KeepAlive: 30 * time.Second,
49+ }).DialContext,
50+ ResponseHeaderTimeout: 3 * time.Minute,
51+ MaxIdleConns: 20,
52+ MaxIdleConnsPerHost: 10,
53+ IdleConnTimeout: 90 * time.Second,
54+}
55+
4156 // Config holds configuration needed by the proxy service.
4257 type Config struct {
4358 RegistryDomain string
@@ -558,8 +573,14 @@ func (s *Service) proxyToRegistry(w http.ResponseWriter, r *http.Request) {
558573 return
559574 }
560575
561- proxy := newReverseProxy(targetURL, "",
562- func(w http.ResponseWriter, _ *http.Request, err error) {
576+ proxy := &httputil.ReverseProxy{
577+ Rewrite: func(pr *httputil.ProxyRequest) {
578+ pr.SetURL(targetURL)
579+ pr.SetXForwarded()
580+ pr.Out.Host = targetURL.Host
581+ },
582+ Transport: registryTransport,
583+ ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
563584 var maxBytesErr *http.MaxBytesError
564585 if errors.As(err, &maxBytesErr) {
565586 log.Warn().Err(err).Int("registry_port", cfg.RegistryPort).Msg("registry proxy error: request body too large")
@@ -569,13 +590,11 @@ func (s *Service) proxyToRegistry(w http.ResponseWriter, r *http.Request) {
569590 log.Error().Err(err).Int("registry_port", cfg.RegistryPort).Msg("registry proxy error")
570591 proxyError(w, "Registry Unavailable", http.StatusServiceUnavailable)
571592 },
572- func(resp *http.Response) error {
593+ ModifyResponse: func(resp *http.Response) error {
573594 resp.Header.Set("X-Proxied-By", "Gordon")
574595 resp.Header.Set("X-Registry-Backend", "gordon-registry")
575596
576597 // Remove security headers from registry response to prevent duplicates.
577- // The proxy middleware already adds these headers, so the registry's
578- // copies would create duplicates in the final response.
579598 resp.Header.Del("X-Content-Type-Options")
580599 resp.Header.Del("X-Frame-Options")
581600 resp.Header.Del("X-XSS-Protection")
@@ -586,7 +605,7 @@ func (s *Service) proxyToRegistry(w http.ResponseWriter, r *http.Request) {
586605
587606 return nil
588607 },
589- )
608+ }
590609
591610 log.Debug().Str("target", targetURL.String()).Msg("proxying request to registry")
592611
@@ -24,8 +24,8 @@ import (
24 "github.com/bnema/gordon/internal/domain"24 "github.com/bnema/gordon/internal/domain"
25 )25 )
26 26
27-// proxyTransport is a shared HTTP transport with proper timeouts.27+// proxyTransport is a shared HTTP transport for proxying to application containers.
28-// This prevents resource exhaustion from slow backends or network issues.28+// ResponseHeaderTimeout is kept short to detect unresponsive backends quickly.
29 var proxyTransport = &http.Transport{29 var proxyTransport = &http.Transport{
30 DialContext: (&net.Dialer{30 DialContext: (&net.Dialer{
31 Timeout: 10 * time.Second,31 Timeout: 10 * time.Second,
@@ -38,6 +38,21 @@ var proxyTransport = &http.Transport{
38 IdleConnTimeout: 90 * time.Second,38 IdleConnTimeout: 90 * time.Second,
39 }39 }
40 40
41+// registryTransport is used for the registry reverse proxy (localhost loopback).
42+// It needs a longer ResponseHeaderTimeout because admin endpoints like /admin/deploy
43+// perform blocking operations (image pull, container start, readiness checks) that
44+// can take well over 30 seconds.
45+var registryTransport = &http.Transport{
46+ DialContext: (&net.Dialer{
47+ Timeout: 10 * time.Second,
48+ KeepAlive: 30 * time.Second,
49+ }).DialContext,
50+ ResponseHeaderTimeout: 3 * time.Minute,
51+ MaxIdleConns: 20,
52+ MaxIdleConnsPerHost: 10,
53+ IdleConnTimeout: 90 * time.Second,
54+}
55+
41 // Config holds configuration needed by the proxy service.56 // Config holds configuration needed by the proxy service.
42 type Config struct {57 type Config struct {
43 RegistryDomain string58 RegistryDomain string
@@ -558,8 +573,14 @@ func (s *Service) proxyToRegistry(w http.ResponseWriter, r *http.Request) {
558 return573 return
559 }574 }
560 575
561- proxy := newReverseProxy(targetURL, "",576+ proxy := &httputil.ReverseProxy{
562- func(w http.ResponseWriter, _ *http.Request, err error) {577+ Rewrite: func(pr *httputil.ProxyRequest) {
578+ pr.SetURL(targetURL)
579+ pr.SetXForwarded()
580+ pr.Out.Host = targetURL.Host
581+ },
582+ Transport: registryTransport,
583+ ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
563 var maxBytesErr *http.MaxBytesError584 var maxBytesErr *http.MaxBytesError
564 if errors.As(err, &maxBytesErr) {585 if errors.As(err, &maxBytesErr) {
565 log.Warn().Err(err).Int("registry_port", cfg.RegistryPort).Msg("registry proxy error: request body too large")586 log.Warn().Err(err).Int("registry_port", cfg.RegistryPort).Msg("registry proxy error: request body too large")
@@ -569,13 +590,11 @@ func (s *Service) proxyToRegistry(w http.ResponseWriter, r *http.Request) {
569 log.Error().Err(err).Int("registry_port", cfg.RegistryPort).Msg("registry proxy error")590 log.Error().Err(err).Int("registry_port", cfg.RegistryPort).Msg("registry proxy error")
570 proxyError(w, "Registry Unavailable", http.StatusServiceUnavailable)591 proxyError(w, "Registry Unavailable", http.StatusServiceUnavailable)
571 },592 },
572- func(resp *http.Response) error {593+ ModifyResponse: func(resp *http.Response) error {
573 resp.Header.Set("X-Proxied-By", "Gordon")594 resp.Header.Set("X-Proxied-By", "Gordon")
574 resp.Header.Set("X-Registry-Backend", "gordon-registry")595 resp.Header.Set("X-Registry-Backend", "gordon-registry")
575 596
576 // Remove security headers from registry response to prevent duplicates.597 // Remove security headers from registry response to prevent duplicates.
577- // The proxy middleware already adds these headers, so the registry's
578- // copies would create duplicates in the final response.
579 resp.Header.Del("X-Content-Type-Options")598 resp.Header.Del("X-Content-Type-Options")
580 resp.Header.Del("X-Frame-Options")599 resp.Header.Del("X-Frame-Options")
581 resp.Header.Del("X-XSS-Protection")600 resp.Header.Del("X-XSS-Protection")
@@ -586,7 +605,7 @@ func (s *Service) proxyToRegistry(w http.ResponseWriter, r *http.Request) {
586 605
587 return nil606 return nil
588 },607 },
589- )608+ }
590 609
591 log.Debug().Str("target", targetURL.String()).Msg("proxying request to registry")610 log.Debug().Str("target", targetURL.String()).Msg("proxying request to registry")
592 611
M internal/usecase/registry/service.go +5 -2
@@ -6,6 +6,7 @@ import (
66 "crypto/sha256"
77 "fmt"
88 "io"
9+ "strings"
910
1011 "github.com/bnema/zerowrap"
1112
@@ -74,8 +75,10 @@ func (s *Service) PutManifest(ctx context.Context, manifest *domain.Manifest) (s
7475 return "", log.WrapErr(err, "failed to store manifest")
7576 }
7677
77- // Publish image pushed event
78- if s.eventBus != nil {
78+ // Publish image pushed event only for tag references (not digests).
79+ // A docker push sends manifests by both digest and tag; firing only on
80+ // tag prevents duplicate deploy triggers for the same push.
81+ if s.eventBus != nil && !strings.HasPrefix(manifest.Reference, "sha256:") {
7982 if err := s.eventBus.Publish(domain.EventImagePushed, domain.ImagePushedPayload{
8083 Name: manifest.Name,
8184 Reference: manifest.Reference,
@@ -6,6 +6,7 @@ import (
6 "crypto/sha256"6 "crypto/sha256"
7 "fmt"7 "fmt"
8 "io"8 "io"
9+ "strings"
9 10
10 "github.com/bnema/zerowrap"11 "github.com/bnema/zerowrap"
11 12
@@ -74,8 +75,10 @@ func (s *Service) PutManifest(ctx context.Context, manifest *domain.Manifest) (s
74 return "", log.WrapErr(err, "failed to store manifest")75 return "", log.WrapErr(err, "failed to store manifest")
75 }76 }
76 77
77- // Publish image pushed event78+ // Publish image pushed event only for tag references (not digests).
78- if s.eventBus != nil {79+ // A docker push sends manifests by both digest and tag; firing only on
80+ // tag prevents duplicate deploy triggers for the same push.
81+ if s.eventBus != nil && !strings.HasPrefix(manifest.Reference, "sha256:") {
79 if err := s.eventBus.Publish(domain.EventImagePushed, domain.ImagePushedPayload{82 if err := s.eventBus.Publish(domain.EventImagePushed, domain.ImagePushedPayload{
80 Name: manifest.Name,83 Name: manifest.Name,
81 Reference: manifest.Reference,84 Reference: manifest.Reference,
M wiki/guides/remote-cli.md +27 -1
@@ -18,7 +18,7 @@ Manage Gordon instances remotely using the CLI with the `--remote` flag or saved
1818
1919 ## Understanding the Remote URL
2020
21-The `--remote` URL must be the `gordon_domain` configured on the remote Gordon instance. This domain serves both the container registry and the Admin API.
21+Default: use the Gordon domain configured on the remote instance.
2222
2323 ```toml
2424 # On the remote Gordon server
@@ -28,6 +28,31 @@ gordon_domain = "gordon.example.com" # Use this as --remote URL
2828
2929 The CLI connects to `https://gordon.example.com/admin/*` endpoints.
3030
31+Private tailnet setup (recommended for hardened VPS): keep using the HTTPS domain, point that DNS record to the VPS tailnet IP, and set insecure TLS for self-signed certs.
32+
33+```bash
34+# Save remote once
35+gordon remotes add tailnet-reg https://gordon.example.com --token-env GORDON_TOKEN --insecure
36+gordon remotes use tailnet-reg
37+
38+# Use it for auth/admin commands
39+gordon auth login
40+gordon routes list
41+```
42+
43+Equivalent `remotes.toml` entry:
44+
45+```toml
46+active = "tailnet-reg"
47+
48+[remotes.tailnet-reg]
49+url = "https://gordon.example.com"
50+token_env = "GORDON_TOKEN"
51+insecure_tls = true
52+```
53+
54+Use this mode when registry/auth/admin access is intentionally restricted to tailnet CIDRs.
55+
3156 ## Quick Start
3257
3358 ### One-off Remote Command
@@ -65,6 +90,7 @@ These flags are available on all commands:
6590 |------|-------------|
6691 | `--remote <URL>` | Remote Gordon URL |
6792 | `--token <TOKEN>` | Authentication token |
93+| `--insecure` | Skip TLS verification (self-signed/private certs) |
6894
6995 ```bash
7096 # List routes on remote
@@ -18,7 +18,7 @@ Manage Gordon instances remotely using the CLI with the `--remote` flag or saved
18 18
19 ## Understanding the Remote URL19 ## Understanding the Remote URL
20 20
21-The `--remote` URL must be the `gordon_domain` configured on the remote Gordon instance. This domain serves both the container registry and the Admin API.21+Default: use the Gordon domain configured on the remote instance.
22 22
23 ```toml23 ```toml
24 # On the remote Gordon server24 # On the remote Gordon server
@@ -28,6 +28,31 @@ gordon_domain = "gordon.example.com" # Use this as --remote URL
28 28
29 The CLI connects to `https://gordon.example.com/admin/*` endpoints.29 The CLI connects to `https://gordon.example.com/admin/*` endpoints.
30 30
31+Private tailnet setup (recommended for hardened VPS): keep using the HTTPS domain, point that DNS record to the VPS tailnet IP, and set insecure TLS for self-signed certs.
32+
33+```bash
34+# Save remote once
35+gordon remotes add tailnet-reg https://gordon.example.com --token-env GORDON_TOKEN --insecure
36+gordon remotes use tailnet-reg
37+
38+# Use it for auth/admin commands
39+gordon auth login
40+gordon routes list
41+```
42+
43+Equivalent `remotes.toml` entry:
44+
45+```toml
46+active = "tailnet-reg"
47+
48+[remotes.tailnet-reg]
49+url = "https://gordon.example.com"
50+token_env = "GORDON_TOKEN"
51+insecure_tls = true
52+```
53+
54+Use this mode when registry/auth/admin access is intentionally restricted to tailnet CIDRs.
55+
31 ## Quick Start56 ## Quick Start
32 57
33 ### One-off Remote Command58 ### One-off Remote Command
@@ -65,6 +90,7 @@ These flags are available on all commands:
65 |------|-------------|90 |------|-------------|
66 | `--remote <URL>` | Remote Gordon URL |91 | `--remote <URL>` | Remote Gordon URL |
67 | `--token <TOKEN>` | Authentication token |92 | `--token <TOKEN>` | Authentication token |
93+| `--insecure` | Skip TLS verification (self-signed/private certs) |
68 94
69 ```bash95 ```bash
70 # List routes on remote96 # List routes on remote
M wiki/guides/secure-vps-setup.md +49 -0
@@ -142,6 +142,55 @@ Verify public SSH is blocked:
142142 ssh -o ConnectTimeout=5 root@<public-ip> # Should timeout
143143 ```
144144
145+## Step 6.5: Restrict Registry and Auth to Tailnet CIDR
146+
147+If you want `/auth/*` and registry (`/v2/*`) reachable only from your tailnet, set Gordon's registry allowlist to the Tailscale CGNAT range.
148+
149+Edit `~/.config/gordon/gordon.toml`:
150+
151+```toml
152+[server]
153+registry_allowed_ips = ["100.64.0.0/10"]
154+```
155+
156+Apply the change:
157+
158+```bash
159+systemctl --user restart gordon
160+```
161+
162+Behavior:
163+
164+- Requests from tailnet IPs (`100.64.0.0/10`) are allowed.
165+- Requests outside that CIDR get `403 Forbidden` on registry/auth endpoints.
166+- App routes still use normal host-based routing through your proxy ports.
167+
168+For private operations (auth, push/pull, admin API), keep using the HTTPS registry domain, but resolve that domain to the VPS tailnet IP (for example, via Cloudflare DNS to the tailnet address). With self-signed certs, configure the CLI remote with insecure TLS enabled.
169+
170+```bash
171+# Example: save remote once
172+gordon remotes add tailnet-reg https://gordon.example.com --token-env GORDON_TOKEN --insecure
173+gordon remotes use tailnet-reg
174+
175+# Then use commands normally (auth + admin API)
176+gordon auth login
177+gordon routes list
178+gordon backup status
179+```
180+
181+`~/.config/gordon/remotes.toml` should contain an entry like:
182+
183+```toml
184+active = "tailnet-reg"
185+
186+[remotes.tailnet-reg]
187+url = "https://gordon.example.com"
188+token_env = "GORDON_TOKEN"
189+insecure_tls = true
190+```
191+
192+This keeps transport on HTTPS while allowing self-signed certificates in tailnet-only deployments.
193+
145194 ---
146195
147196 ## Step 7: Install CrowdSec
@@ -142,6 +142,55 @@ Verify public SSH is blocked:
142 ssh -o ConnectTimeout=5 root@<public-ip> # Should timeout142 ssh -o ConnectTimeout=5 root@<public-ip> # Should timeout
143 ```143 ```
144 144
145+## Step 6.5: Restrict Registry and Auth to Tailnet CIDR
146+
147+If you want `/auth/*` and registry (`/v2/*`) reachable only from your tailnet, set Gordon's registry allowlist to the Tailscale CGNAT range.
148+
149+Edit `~/.config/gordon/gordon.toml`:
150+
151+```toml
152+[server]
153+registry_allowed_ips = ["100.64.0.0/10"]
154+```
155+
156+Apply the change:
157+
158+```bash
159+systemctl --user restart gordon
160+```
161+
162+Behavior:
163+
164+- Requests from tailnet IPs (`100.64.0.0/10`) are allowed.
165+- Requests outside that CIDR get `403 Forbidden` on registry/auth endpoints.
166+- App routes still use normal host-based routing through your proxy ports.
167+
168+For private operations (auth, push/pull, admin API), keep using the HTTPS registry domain, but resolve that domain to the VPS tailnet IP (for example, via Cloudflare DNS to the tailnet address). With self-signed certs, configure the CLI remote with insecure TLS enabled.
169+
170+```bash
171+# Example: save remote once
172+gordon remotes add tailnet-reg https://gordon.example.com --token-env GORDON_TOKEN --insecure
173+gordon remotes use tailnet-reg
174+
175+# Then use commands normally (auth + admin API)
176+gordon auth login
177+gordon routes list
178+gordon backup status
179+```
180+
181+`~/.config/gordon/remotes.toml` should contain an entry like:
182+
183+```toml
184+active = "tailnet-reg"
185+
186+[remotes.tailnet-reg]
187+url = "https://gordon.example.com"
188+token_env = "GORDON_TOKEN"
189+insecure_tls = true
190+```
191+
192+This keeps transport on HTTPS while allowing self-signed certificates in tailnet-only deployments.
193+
145 ---194 ---
146 195
147 ## Step 7: Install CrowdSec196 ## Step 7: Install CrowdSec

Keyboard shortcuts

Single-key shortcuts are on. ctrl k and esc work whatever this says.

ctrl k :
go to a section, branch, tag, file or repository
?
this list
j k
move the row cursor down / up
enter
open the row under the cursor; on Files, a directory opens or closes in place
↑ ↓ ← → home end
in the file tree (focus on a row): previous / next row, close / open a directory or go to its parent / first entry, first / last row. They never open a file
l h
on Files: open a directory or a file / close a directory or go up to its parent, then to the parent page. Elsewhere: into the side panel (on a merge request, its files) / back to the content; there j k move between its controls
g g G
first / last row
/
filter the list; on Files, find a file anywhere in the repository (esc clears, enter opens the best match)
b
switch branch or tag
g s g f g c g b g m g i g r g a
tabs: summary, files, commits, branches & tags, merge requests, issues, releases, actions (when available)
g h
home
w
wide mode: use the whole width of the screen, or not (remembered)
y
copy a permalink pinned to the commit
d u x
diff: split / unified / collapse or expand all
e E
diff: the whole file in view / every file, with the lines the hunks hide
n
merge request: next note in the diff
esc
close a dialog, leave a field

Branches 7

Tags 106

↑↓ moveenter openesc close