1package app
2
3import (
4 "context"
5 "errors"
6 "net"
7 "net/http"
8 "os"
9 "path/filepath"
10 "strconv"
11 "strings"
12 "sync"
13 "testing"
14 "time"
15
16 "github.com/bnema/zerowrap"
17 "github.com/spf13/viper"
18 "github.com/stretchr/testify/assert"
19 "github.com/stretchr/testify/mock"
20 "github.com/stretchr/testify/require"
21
22 "github.com/bnema/gordon/internal/adapters/in/http/registry"
23 trafficadapter "github.com/bnema/gordon/internal/adapters/in/traffic"
24 outmocks "github.com/bnema/gordon/internal/boundaries/out/mocks"
25 "github.com/bnema/gordon/internal/domain"
26 cfgusecase "github.com/bnema/gordon/internal/usecase/config"
27 "github.com/bnema/gordon/internal/usecase/container"
28 "github.com/bnema/gordon/internal/usecase/proxy"
29 servicecfg "github.com/bnema/gordon/internal/usecase/services"
30 traffic "github.com/bnema/gordon/internal/usecase/traffic"
31)
32
33type watchRecorder struct {
34 called bool
35 onChange func()
36}
37
38func (w *watchRecorder) Watch(ctx context.Context, onChange func()) error {
39 w.called = true
40 w.onChange = onChange
41 return nil
42}
43
44type coordinatorRecorder struct {
45 mu sync.Mutex
46 triggerCalls int
47 applyCalls int
48 err error
49}
50
51func (c *coordinatorRecorder) Trigger(context.Context) error {
52 c.mu.Lock()
53 c.triggerCalls++
54 err := c.err
55 c.mu.Unlock()
56 return err
57}
58
59func (c *coordinatorRecorder) ApplyLoadedConfig(context.Context) error {
60 c.mu.Lock()
61 c.applyCalls++
62 err := c.err
63 c.mu.Unlock()
64 return err
65}
66
67func (c *coordinatorRecorder) TriggerCalls() int {
68 c.mu.Lock()
69 defer c.mu.Unlock()
70 return c.triggerCalls
71}
72
73func (c *coordinatorRecorder) ApplyCalls() int {
74 c.mu.Lock()
75 defer c.mu.Unlock()
76 return c.applyCalls
77}
78
79type reloadRecorder struct {
80 mu sync.Mutex
81 calls int
82 err error
83 onReload func()
84}
85
86func (r *reloadRecorder) Reload(context.Context) error {
87 r.mu.Lock()
88 r.calls++
89 err := r.err
90 onReload := r.onReload
91 r.mu.Unlock()
92
93 if onReload != nil {
94 onReload()
95 }
96
97 return err
98}
99
100func (r *reloadRecorder) Calls() int {
101 r.mu.Lock()
102 defer r.mu.Unlock()
103 return r.calls
104}
105
106type proxyRecorder struct {
107 calls int
108 config proxy.Config
109}
110
111func (p *proxyRecorder) UpdateConfig(config proxy.Config) {
112 p.calls++
113 p.config = config
114}
115
116type registryLimitsRecorder struct {
117 calls int
118 maxBlobChunkSize int64
119 maxBlobSize int64
120}
121
122func (r *registryLimitsRecorder) UpdateBlobLimits(maxBlobChunkSize, maxBlobSize int64) {
123 r.calls++
124 r.maxBlobChunkSize = maxBlobChunkSize
125 r.maxBlobSize = maxBlobSize
126}
127
128type containerConfigApplyRecorder struct {
129 calls int
130 ctx context.Context
131 cfg Config
132}
133
134func (r *containerConfigApplyRecorder) Apply(ctx context.Context, cfg Config) error {
135 r.calls++
136 r.ctx = ctx
137 r.cfg = cfg
138 return nil
139}
140
141type standaloneServiceRecorder struct {
142 mu sync.Mutex
143 calls int
144 services []domain.StandaloneService
145 err error
146 onReconcile func()
147}
148
149func (r *standaloneServiceRecorder) Reconcile(_ context.Context, services []domain.StandaloneService) error {
150 r.mu.Lock()
151 r.calls++
152 r.services = append([]domain.StandaloneService(nil), services...)
153 err := r.err
154 onReconcile := r.onReconcile
155 r.mu.Unlock()
156 if onReconcile != nil {
157 onReconcile()
158 }
159 return err
160}
161
162func (r *standaloneServiceRecorder) Status(context.Context) ([]domain.StandaloneServiceStatus, error) {
163 return nil, nil
164}
165
166func (r *standaloneServiceRecorder) Calls() int {
167 r.mu.Lock()
168 defer r.mu.Unlock()
169 return r.calls
170}
171
172type eventBusRecorder struct {
173 mu sync.Mutex
174 calls int
175 eventType domain.EventType
176 payload any
177 err error
178}
179
180func (e *eventBusRecorder) Publish(eventType domain.EventType, payload any) error {
181 e.mu.Lock()
182 e.calls++
183 e.eventType = eventType
184 e.payload = payload
185 err := e.err
186 e.mu.Unlock()
187 return err
188}
189
190func (e *eventBusRecorder) Calls() int {
191 e.mu.Lock()
192 defer e.mu.Unlock()
193 return e.calls
194}
195
196func TestSetupConfigHotReload_WatchCallbackInvokesCoordinator(t *testing.T) {
197 ctx := context.Background()
198 configSvc := &watchRecorder{}
199 coordinator := &coordinatorRecorder{}
200
201 err := setupConfigHotReload(ctx, configSvc, coordinator)
202 require.NoError(t, err)
203 require.True(t, configSvc.called)
204 require.NotNil(t, configSvc.onChange)
205
206 configSvc.onChange()
207 require.Equal(t, 1, coordinator.ApplyCalls())
208 require.Equal(t, 0, coordinator.TriggerCalls())
209}
210
211func TestBuildProxyConfig_ParsesMaxBlobSizeWithoutChangingChunkDefault(t *testing.T) {
212 cfg := Config{}
213 cfg.Server.MaxBlobSize = "2GB"
214
215 result, err := buildProxyConfig(cfg, zerowrap.Default())
216
217 require.NoError(t, err)
218 assert.Equal(t, int64(2<<30), result.maxBlobSize)
219 assert.Equal(t, int64(registry.DefaultMaxBlobChunkSize), result.maxBlobChunkSize)
220}
221
222func TestBuildProxyConfig_DefaultMaxBlobSize(t *testing.T) {
223 result, err := buildProxyConfig(Config{}, zerowrap.Default())
224
225 require.NoError(t, err)
226 assert.Equal(t, int64(registry.DefaultMaxBlobSize), result.maxBlobSize)
227}
228
229func TestReloadCoordinator_ApplyLoadedConfig_RebuildsProxyConfigAndPublishesEvent(t *testing.T) {
230 ctx := context.Background()
231 v := viper.New()
232 v.Set("server.gordon_domain", "reload.example.com")
233 v.Set("server.registry_port", 5000)
234 v.Set("server.max_proxy_body_size", "5MB")
235 v.Set("server.max_blob_chunk_size", "6MB")
236 v.Set("server.max_blob_size", "8MB")
237 v.Set("server.max_proxy_response_size", "7MB")
238 v.Set("server.max_concurrent_connections", 99)
239 v.Set("server.legacy_registry_domains", []string{"old.example.com"})
240 v.Set("images.allowed_registries", []string{"docker.io"})
241
242 reloadSvc := &reloadRecorder{}
243 proxySvc := &proxyRecorder{}
244 events := &eventBusRecorder{}
245 containerCfg := &containerConfigApplyRecorder{}
246
247 registryLimits := ®istryLimitsRecorder{}
248 coord := newReloadCoordinator(v, reloadSvc, proxySvc, registryLimits, events, nil, zerowrap.Default())
249 coord.SetContainerConfigApplier(containerCfg.Apply)
250 require.NoError(t, coord.ApplyLoadedConfig(ctx))
251
252 require.Equal(t, 0, reloadSvc.Calls())
253 require.Equal(t, 1, proxySvc.calls)
254 require.Equal(t, 1, events.Calls())
255 require.Equal(t, 1, registryLimits.calls)
256 require.Equal(t, 1, containerCfg.calls)
257 require.Equal(t, int64(6<<20), registryLimits.maxBlobChunkSize)
258 require.Equal(t, int64(8<<20), registryLimits.maxBlobSize)
259 require.Equal(t, domain.EventConfigReload, events.eventType)
260 require.Equal(t, "reload.example.com", containerCfg.cfg.Server.GordonDomain)
261 require.Equal(t, []string{"old.example.com"}, containerCfg.cfg.Server.LegacyRegistryDomains)
262 require.Equal(t, []string{"docker.io"}, containerCfg.cfg.Images.AllowedRegistries)
263 require.Equal(t, proxy.Config{
264 RegistryDomain: "reload.example.com",
265 RegistryPort: 5000,
266 MaxBodySize: 5 << 20,
267 MaxResponseSize: 7 << 20,
268 MaxConcurrentConns: 99,
269 }, proxySvc.config)
270}
271
272func TestServiceInit_ReloadRegistersCustomTLSMuxHTTPSFallback(t *testing.T) {
273 ctx := context.Background()
274 v := viper.New()
275 v.Set("server.gordon_domain", "reload.example.com")
276 v.Set("server.registry_port", 5000)
277
278 configSvc := cfgusecase.NewService(v, nil)
279 require.NoError(t, configSvc.Load(ctx))
280 manager := trafficadapter.NewManager()
281 defer func() { require.NoError(t, manager.Shutdown(ctx)) }()
282 httpsHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
283 _, _ = w.Write([]byte("reload secure"))
284 })
285 pkiSvc := newTestPKIService(t)
286
287 customAddress := freeTCPAddress(t)
288 si := &serviceInit{
289 ctx: ctx,
290 v: v,
291 cfg: Config{},
292 log: zerowrap.Default(),
293 svc: &services{
294 configSvc: configSvc,
295 containerSvc: container.NewService(nil, nil, nil, nil, container.Config{}, configSvc),
296 internalRegUser: "gordon",
297 internalRegPass: "secret",
298 reloadCoordinator: newReloadCoordinator(v, &reloadRecorder{}, &proxyRecorder{}, nil, nil, nil, zerowrap.Default()),
299 trafficManager: manager,
300 httpsProxyHandler: httpsHandler,
301 pkiSvc: pkiSvc,
302 },
303 }
304 si.registerReloadCoordinatorHooks()
305
306 reloadCfg := Config{}
307 reloadCfg.Server.GordonDomain = "reload.example.com"
308 reloadCfg.Server.RegistryPort = 5000
309 reloadCfg.Server.TLSPort = freeTCPPort(t)
310 reloadCfg.EntryPoints = map[string]traffic.EntryPointConfig{
311 "custom-secure": {Address: customAddress, Protocol: domain.EntryPointProtocolTLSMux},
312 }
313
314 require.NoError(t, si.svc.reloadCoordinator.applyContainerConfig(ctx, reloadCfg))
315 _, portValue, err := net.SplitHostPort(customAddress)
316 require.NoError(t, err)
317 port, err := strconv.Atoi(portValue)
318 require.NoError(t, err)
319 body := httpsGetForTest(t, port, "app.example.com")
320 require.Contains(t, body, "reload secure")
321}
322
323func TestServiceInit_RegisterReloadCoordinatorHooks_WiresContainerConfigApplier(t *testing.T) {
324 ctx := context.Background()
325 v := viper.New()
326 v.Set("server.gordon_domain", "reload.example.com")
327 v.Set("server.registry_port", 5000)
328
329 configSvc := cfgusecase.NewService(v, nil)
330 require.NoError(t, configSvc.Load(ctx))
331
332 si := &serviceInit{
333 ctx: ctx,
334 v: v,
335 cfg: Config{},
336 log: zerowrap.Default(),
337 svc: &services{
338 configSvc: configSvc,
339 containerSvc: container.NewService(nil, nil, nil, nil, container.Config{}, configSvc),
340 internalRegUser: "gordon",
341 internalRegPass: "secret",
342 reloadCoordinator: newReloadCoordinator(v, &reloadRecorder{}, &proxyRecorder{}, nil, nil, nil, zerowrap.Default()),
343 },
344 }
345
346 si.registerReloadCoordinatorHooks()
347 require.NotNil(t, si.svc.reloadCoordinator.applyContainerConfig)
348
349 reloadCfg := Config{}
350 reloadCfg.Server.GordonDomain = "reload.example.com"
351 reloadCfg.Server.RegistryPort = 5000
352 reloadCfg.Images.AllowedRegistries = []string{"docker.io"}
353
354 require.NoError(t, si.svc.reloadCoordinator.applyContainerConfig(ctx, reloadCfg))
355}
356func TestStandaloneServiceSecretProviderUnsafeBackendReconcilesServiceSecrets(t *testing.T) {
357 ctx := context.Background()
358 dataDir := t.TempDir()
359 secretPath := "service/game/rcon"
360 require.NoError(t, os.MkdirAll(filepath.Join(dataDir, "secrets", "service", "game"), 0700))
361 require.NoError(t, os.WriteFile(filepath.Join(dataDir, "secrets", secretPath), []byte("test-secret-value"), 0600))
362
363 rt := outmocks.NewMockContainerRuntime(t)
364 var sawSecretEnv bool
365 rt.On("ListContainers", mock.Anything, true).Return([]*domain.Container{}, nil).Once()
366 rt.On("InspectImageVolumes", mock.Anything, "game:latest").Return([]string(nil), nil).Once()
367 rt.On("CreateContainer", mock.Anything, mock.AnythingOfType("*domain.ContainerConfig")).Run(func(args mock.Arguments) {
368 config := args.Get(1).(*domain.ContainerConfig)
369 for _, entry := range config.Env {
370 if strings.HasPrefix(entry, "RCON_PASSWORD=") {
371 sawSecretEnv = true
372 }
373 }
374 }).Return(&domain.Container{ID: "created-1"}, nil).Once()
375 rt.On("StartContainer", mock.Anything, "created-1").Return(nil).Once()
376
377 provider := createStandaloneServiceSecretProvider(domain.SecretsBackendUnsafe, dataDir, zerowrap.Default())
378 reconciler := servicecfg.NewServiceWithSecretProvider(rt, provider)
379 err := reconciler.Reconcile(ctx, []domain.StandaloneService{{Name: "game", Image: "game:latest", Enabled: true, Secrets: []domain.StandaloneServiceSecretRef{{Name: "rcon", Key: "RCON_PASSWORD"}}}})
380
381 require.NoError(t, err)
382 assert.True(t, sawSecretEnv)
383}
384
385func TestServiceInit_RegisterReloadCoordinatorHooks_AppliesTrafficBeforeStandaloneServices(t *testing.T) {
386 ctx := context.Background()
387 v := viper.New()
388 v.Set("server.gordon_domain", "reload.example.com")
389 v.Set("server.registry_port", 5000)
390
391 configSvc := cfgusecase.NewService(v, nil)
392 require.NoError(t, configSvc.Load(ctx))
393 manager := trafficadapter.NewManager()
394 defer func() { require.NoError(t, manager.Shutdown(ctx)) }()
395
396 reconciler := &standaloneServiceRecorder{onReconcile: func() {
397 assert.NotEmpty(t, manager.Status().EntryPoints, "traffic graph should apply before standalone services reconcile")
398 }}
399 si := &serviceInit{
400 ctx: ctx,
401 v: v,
402 cfg: Config{},
403 log: zerowrap.Default(),
404 svc: &services{
405 configSvc: configSvc,
406 containerSvc: container.NewService(nil, nil, nil, nil, container.Config{}, configSvc),
407 internalRegUser: "gordon",
408 internalRegPass: "secret",
409 reloadCoordinator: newReloadCoordinator(v, &reloadRecorder{}, &proxyRecorder{}, nil, nil, nil, zerowrap.Default()),
410 trafficManager: manager,
411 standaloneServiceSvc: reconciler,
412 },
413 }
414 si.registerReloadCoordinatorHooks()
415
416 reloadCfg := Config{}
417 reloadCfg.Server.GordonDomain = "reload.example.com"
418 reloadCfg.Server.RegistryPort = 5000
419 reloadCfg.Services = []servicecfg.Config{{Name: "game", Image: "game:latest", Enabled: true}}
420 reloadCfg.EntryPoints = map[string]traffic.EntryPointConfig{"game": {Address: freeTCPAddress(t), Protocol: domain.EntryPointProtocolTCP}}
421 reloadCfg.NetworkServices = []traffic.NetworkServiceConfig{{Name: "game", Ports: []traffic.PortConfig{{Name: "game", Container: 28015, Protocol: domain.NetworkProtocolTCP}}}}
422 reloadCfg.Traffic.TCP.Routers = []traffic.RouterConfig{{Name: "game", EntryPoint: "game", Service: "network_service:game:game"}}
423
424 require.NoError(t, si.svc.reloadCoordinator.applyContainerConfig(ctx, reloadCfg))
425 require.Equal(t, 1, reconciler.Calls())
426 require.Len(t, reconciler.services, 1)
427 assert.Equal(t, "game", reconciler.services[0].Name)
428 assert.NotEmpty(t, manager.Status().EntryPoints)
429}
430
431func TestWaitForCoreProxyReadyAndApplyTraffic_AppliesTrafficBeforeStandaloneServices(t *testing.T) {
432 ctx := context.Background()
433 v := viper.New()
434 configSvc := cfgusecase.NewService(v, nil)
435 require.NoError(t, configSvc.Load(ctx))
436 manager := trafficadapter.NewManager()
437 defer func() { require.NoError(t, manager.Shutdown(ctx)) }()
438
439 reconciler := &standaloneServiceRecorder{onReconcile: func() {
440 assert.NotEmpty(t, manager.Status().EntryPoints, "traffic graph should apply before standalone services reconcile")
441 }}
442 cfg := Config{}
443 cfg.Services = []servicecfg.Config{{Name: "game", Image: "game:latest", Enabled: true}}
444 cfg.EntryPoints = map[string]traffic.EntryPointConfig{"game": {Address: freeTCPAddress(t), Protocol: domain.EntryPointProtocolTCP}}
445 cfg.NetworkServices = []traffic.NetworkServiceConfig{{Name: "game", Ports: []traffic.PortConfig{{Name: "game", Container: 28015, Protocol: domain.NetworkProtocolTCP}}}}
446 cfg.Traffic.TCP.Routers = []traffic.RouterConfig{{Name: "game", EntryPoint: "game", Service: "network_service:game:game"}}
447
448 ready := make(chan struct{})
449 close(ready)
450 svc := &services{configSvc: configSvc, trafficManager: manager, standaloneServiceSvc: reconciler}
451 require.NoError(t, waitForCoreProxyReadyAndApplyTraffic(ctx, cfg, svc, ready, ready, nil))
452 require.Equal(t, 1, reconciler.Calls())
453 assert.NotEmpty(t, manager.Status().EntryPoints)
454}
455
456func TestReconcileStandaloneServices_ConfigConversionErrorIsClear(t *testing.T) {
457 reconciler := &standaloneServiceRecorder{}
458 err := reconcileStandaloneServices(context.Background(), reconciler, Config{Services: []servicecfg.Config{{Name: "broken", Image: "", Enabled: true}}})
459 require.Error(t, err)
460 assert.ErrorContains(t, err, "convert standalone service config")
461 assert.Equal(t, 0, reconciler.Calls())
462}
463
464func TestReloadCoordinator_DebouncesRepeatedWatchCallbacks(t *testing.T) {
465 ctx := context.Background()
466 v := viper.New()
467 v.Set("server.gordon_domain", "new.example.com")
468 v.Set("server.registry_port", 5000)
469 v.Set("server.max_proxy_body_size", "5MB")
470 v.Set("server.max_blob_chunk_size", "6MB")
471 v.Set("server.max_blob_size", "8MB")
472 v.Set("server.max_proxy_response_size", "7MB")
473 v.Set("server.max_concurrent_connections", 99)
474
475 reloadSvc := &reloadRecorder{}
476 proxySvc := &proxyRecorder{}
477 events := &eventBusRecorder{}
478
479 registryLimits := ®istryLimitsRecorder{}
480 coord := newReloadCoordinator(v, reloadSvc, proxySvc, registryLimits, events, nil, zerowrap.Default())
481 require.NoError(t, coord.Trigger(ctx))
482 require.NoError(t, coord.Trigger(ctx))
483
484 require.Equal(t, 1, reloadSvc.Calls())
485 require.Equal(t, 1, proxySvc.calls)
486 require.Equal(t, 1, events.Calls())
487 require.Equal(t, 1, registryLimits.calls)
488 require.Equal(t, int64(6<<20), registryLimits.maxBlobChunkSize)
489 require.Equal(t, int64(8<<20), registryLimits.maxBlobSize)
490 require.Equal(t, domain.EventConfigReload, events.eventType)
491 require.Equal(t, proxy.Config{
492 RegistryDomain: "new.example.com",
493 RegistryPort: 5000,
494 MaxBodySize: 5 << 20,
495 MaxResponseSize: 7 << 20,
496 MaxConcurrentConns: 99,
497 }, proxySvc.config)
498}
499
500func TestReloadCoordinator_RetriesImmediatelyAfterFailedReload(t *testing.T) {
501 ctx := context.Background()
502 v := viper.New()
503 v.Set("server.gordon_domain", "retry.example.com")
504 v.Set("server.registry_port", 5000)
505 v.Set("server.max_proxy_body_size", "2MB")
506 v.Set("server.max_blob_chunk_size", "3MB")
507 v.Set("server.max_proxy_response_size", "4MB")
508 v.Set("server.max_concurrent_connections", 11)
509
510 reloadErr := errors.New("reload failed")
511 reloadSvc := &reloadRecorder{err: reloadErr}
512 proxySvc := &proxyRecorder{}
513 coord := newReloadCoordinator(v, reloadSvc, proxySvc, nil, nil, nil, zerowrap.Default())
514
515 err := coord.Trigger(ctx)
516 require.ErrorIs(t, err, reloadErr)
517
518 reloadSvc.mu.Lock()
519 reloadSvc.err = nil
520 reloadSvc.mu.Unlock()
521
522 require.NoError(t, coord.Trigger(ctx))
523 require.Equal(t, 2, reloadSvc.Calls())
524 assert.Equal(t, 1, proxySvc.calls)
525}
526
527func TestReloadCoordinator_PublishErrorDoesNotAdvanceDebounceState(t *testing.T) {
528 ctx := context.Background()
529 v := viper.New()
530 v.Set("server.gordon_domain", "publish.example.com")
531 v.Set("server.registry_port", 5000)
532 v.Set("server.max_proxy_body_size", "5MB")
533 v.Set("server.max_blob_chunk_size", "6MB")
534 v.Set("server.max_proxy_response_size", "7MB")
535 v.Set("server.max_concurrent_connections", 99)
536
537 publishErr := errors.New("publish failed")
538 reloadSvc := &reloadRecorder{}
539 proxySvc := &proxyRecorder{}
540 events := &eventBusRecorder{err: publishErr}
541
542 coord := newReloadCoordinator(v, reloadSvc, proxySvc, nil, events, nil, zerowrap.Default())
543
544 err := coord.Trigger(ctx)
545 require.ErrorIs(t, err, publishErr)
546
547 err = coord.Trigger(ctx)
548 require.ErrorIs(t, err, publishErr)
549
550 require.Equal(t, 2, reloadSvc.Calls())
551 require.Equal(t, 2, proxySvc.calls)
552 require.Equal(t, 2, events.Calls())
553 require.Equal(t, domain.EventConfigReload, events.eventType)
554}
555
556func TestReloadCoordinator_SerializesOverlappingReloadRequests(t *testing.T) {
557 ctx := context.Background()
558 v := viper.New()
559 v.Set("server.gordon_domain", "app.example.com")
560 v.Set("server.registry_port", 5000)
561 v.Set("server.max_proxy_body_size", "2MB")
562 v.Set("server.max_blob_chunk_size", "3MB")
563 v.Set("server.max_proxy_response_size", "4MB")
564 v.Set("server.max_concurrent_connections", 11)
565
566 started := make(chan struct{})
567 release := make(chan struct{})
568 var once sync.Once
569 reloadSvc := &reloadRecorder{onReload: func() {
570 once.Do(func() { close(started) })
571 <-release
572 }}
573 proxySvc := &proxyRecorder{}
574 coord := newReloadCoordinator(v, reloadSvc, proxySvc, nil, nil, nil, zerowrap.Default())
575
576 firstDone := make(chan struct{})
577 go func() {
578 _ = coord.Trigger(ctx)
579 close(firstDone)
580 }()
581
582 select {
583 case <-started:
584 case <-time.After(2 * time.Second):
585 t.Fatal("first reload did not start")
586 }
587
588 secondDone := make(chan struct{})
589 go func() {
590 _ = coord.Trigger(ctx)
591 close(secondDone)
592 }()
593
594 select {
595 case <-secondDone:
596 t.Fatal("second reload finished before the first was released")
597 case <-time.After(75 * time.Millisecond):
598 }
599
600 require.Equal(t, 1, reloadSvc.Calls())
601 close(release)
602
603 select {
604 case <-firstDone:
605 case <-time.After(2 * time.Second):
606 t.Fatal("first reload did not finish")
607 }
608
609 select {
610 case <-secondDone:
611 case <-time.After(2 * time.Second):
612 t.Fatal("second reload did not finish")
613 }
614
615 require.Equal(t, 1, reloadSvc.Calls())
616 assert.Equal(t, 1, proxySvc.calls)
617}