1package app
2
3import (
4 "context"
5 "fmt"
6 "net/http"
7 "os"
8 "time"
9
10 "github.com/bnema/zerowrap"
11
12 trafficadapter "github.com/bnema/gordon/internal/adapters/in/traffic"
13 "github.com/bnema/gordon/internal/boundaries/in"
14 "github.com/bnema/gordon/internal/boundaries/out"
15 "github.com/bnema/gordon/internal/usecase/container"
16 pkiusecase "github.com/bnema/gordon/internal/usecase/pki"
17 "github.com/bnema/gordon/internal/usecase/proxy"
18)
19
20// waitForShutdown blocks on the event loop, handling server errors and
21// Unix signals (reload, deploy, shutdown) until the context is cancelled.
22func waitForShutdown(ctx context.Context, errChan <-chan error, reloadChan <-chan os.Signal, reload reloadTrigger, eventBus out.EventBus, log zerowrap.Logger) {
23 for {
24 select {
25 case err := <-errChan:
26 log.Error().Err(err).Msg("server error")
27 return
28 case <-reloadChan:
29 log.Info().Msg("reload signal received (SIGUSR1)")
30 _ = reload.Trigger(ctx)
31 case <-ctx.Done():
32 log.Info().Msg("shutdown signal received")
33 return
34 }
35 }
36}
37
38// appAdministration is the daemon-owned application lifecycle torn down
39// during graceful shutdown: it refuses new background deploys, cancels
40// in-flight executions on the daemon context, and joins them.
41type appAdministration interface {
42 Shutdown(context.Context) error
43}
44
45// quiesceAppAdministration cancels and joins daemon-owned app work on the
46// bounded shutdown context. A non-nil error means the bound expired while an
47// execution was still unwinding: the caller must stop tearing down the state
48// and runtime that execution still uses.
49func quiesceAppAdministration(ctx context.Context, appAdmin appAdministration, log zerowrap.Logger) error {
50 if appAdmin == nil {
51 return nil
52 }
53 if err := appAdmin.Shutdown(ctx); err != nil {
54 log.Error().Err(err).Msg("app administration did not quiesce before the shutdown deadline")
55 return err
56 }
57 return nil
58}
59
60// gracefulShutdown stops HTTP servers with a 30s timeout, then shuts down
61// the container service and cleans up runtime files. It returns an error when
62// app administration could not quiesce: in that case the remaining teardown
63// is skipped and the error is propagated to the process entry point, which
64// exits non-zero instead of closing state or runtime under an in-flight
65// ExecuteDeploy.
66func gracefulShutdown(registrySrvs []*http.Server, proxySrv, tlsSrv *http.Server, containerSvc *container.Service, proxySvc *proxy.Service, pkiSvc *pkiusecase.Service, publicTLS in.PublicTLSService, trafficManager *trafficadapter.Manager, monitor *appMonitor, appAdmin appAdministration, appState out.AppState, log zerowrap.Logger) error {
67 log.Info().Msg("shutting down Gordon...")
68
69 // Phase 0: stop periodic app recovery first and wait for any in-flight
70 // bounded pass, so no reconcile can race runtime, traffic, or state
71 // teardown (and no goroutine is left behind).
72 monitor.Stop()
73
74 shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second)
75 defer shutdownCancel()
76
77 // Phase 0.5: quiesce daemon-owned app administration. New background
78 // deploys are refused and in-flight executions are cancelled on the
79 // daemon context and joined here — before traffic, runtime, or the app
80 // state store is torn down, so no execution can write to closed state.
81 // A timeout is fail-closed: the unfinished execution still owns the
82 // state and runtime it uses, so teardown stops here and the error makes
83 // the process exit non-zero rather than close resources underneath it.
84 if err := quiesceAppAdministration(shutdownCtx, appAdmin, log); err != nil {
85 return fmt.Errorf("app administration quiescence: %w", err)
86 }
87
88 // Phase 1: Stop ingress frontends (TLS, then proxy) — no new traffic accepted
89 shutdownHTTPServers(shutdownCtx, log, tlsSrv, proxySrv)
90
91 if trafficManager != nil {
92 if err := trafficManager.Shutdown(shutdownCtx); err != nil {
93 log.Warn().Err(err).Msg("traffic manager shutdown error")
94 }
95 }
96
97 // Stop PKI maintenance goroutines
98 if pkiSvc != nil {
99 pkiSvc.Stop()
100 }
101
102 // Stop public ACME TLS renewal loop
103 if publicTLS != nil {
104 if err := publicTLS.Stop(shutdownCtx); err != nil {
105 log.Warn().Err(err).Msg("public TLS stop error")
106 }
107 }
108
109 // Phase 2: Drain in-flight registry push sessions before stopping the backend
110 if proxySvc != nil {
111 log.Info().Msg("draining in-flight registry requests...")
112 if drained := proxySvc.DrainRegistryInFlight(25 * time.Second); !drained {
113 log.Warn().Int64("in_flight", proxySvc.RegistryInFlight()).Msg("registry drain timed out; some in-flight pushes may be interrupted")
114 }
115 }
116
117 // Phase 3: Stop the registry backends.
118 shutdownHTTPServers(shutdownCtx, log, registrySrvs...)
119
120 if err := containerSvc.Shutdown(shutdownCtx); err != nil {
121 log.Warn().Err(err).Msg("error during container shutdown")
122 }
123 closeAppState(appState, log)
124
125 cleanupInternalCredentials()
126 log.Info().Msg("Gordon stopped")
127 return nil
128}
129
130// shutdownHTTPServers gracefully stops each non-nil server on the shutdown
131// context, logging per-server failures so one listener cannot mask another.
132func shutdownHTTPServers(ctx context.Context, log zerowrap.Logger, servers ...*http.Server) {
133 for _, srv := range servers {
134 if srv == nil {
135 continue
136 }
137 if err := srv.Shutdown(ctx); err != nil {
138 log.Warn().Err(err).Str("addr", srv.Addr).Msg("server shutdown error")
139 }
140 }
141}
142
143func closeAppState(state out.AppState, log zerowrap.Logger) {
144 if state == nil {
145 return
146 }
147 if err := state.Close(); err != nil {
148 log.Warn().Err(err).Msg("app state close error")
149 }
150}
151
152// shutdownStartedServers gracefully shuts down listeners that bound before a
153// partial startup failure so an error return does not leak them.
154func shutdownStartedServers(servers []*http.Server, log zerowrap.Logger) {
155 shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
156 defer shutdownCancel()
157 for _, srv := range servers {
158 if srv != nil {
159 if err := srv.Shutdown(shutdownCtx); err != nil {
160 log.Error().Err(err).Msg("failed to shut down server during startup cleanup")
161 }
162 }
163 }
164}
165
166// cleanupStartupResources releases every resource a partial startup may have
167// acquired once the local admin socket is serving: the admin socket first (no
168// further mutations can begin), then the bound servers, then the traffic
169// manager that startProxyServers may already have populated.
170func cleanupStartupResources(localAdmin *localAdminServer, trafficManager *trafficadapter.Manager, log zerowrap.Logger, servers ...*http.Server) {
171 if localAdmin != nil {
172 localAdmin.Close()
173 }
174 shutdownStartedServers(servers, log)
175 shutdownTrafficManagerForStartupCleanup(trafficManager, log)
176}
177
178func shutdownTrafficManagerForStartupCleanup(manager *trafficadapter.Manager, log zerowrap.Logger) {
179 if manager == nil {
180 return
181 }
182 shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second)
183 defer shutdownCancel()
184 if err := manager.Shutdown(shutdownCtx); err != nil {
185 log.Error().Err(err).Msg("failed to shut down traffic manager during startup cleanup")
186 }
187}