Merge pull request #137 from bnema/feat/durable-session-recovery
refactor(recovery): simplify durable session recovery
122 files · +10154 -7705
M cmd/vev-perf-harness/hangup_test.go +131 -0
| @@ -4,11 +4,13 @@ package main | |||||
| 4 | 4 | ||||
| 5 | import ( | 5 | import ( | ||
| 6 | "encoding/json" | 6 | "encoding/json" | ||
| 7 | + | "errors" | |||
| 7 | "fmt" | 8 | "fmt" | ||
| 8 | "os" | 9 | "os" | ||
| 9 | "os/exec" | 10 | "os/exec" | ||
| 10 | "path/filepath" | 11 | "path/filepath" | ||
| 11 | "strings" | 12 | "strings" | ||
| 13 | + | "syscall" | |||
| 12 | "testing" | 14 | "testing" | ||
| 13 | "time" | 15 | "time" | ||
| 14 | ) | 16 | ) | ||
| @@ -133,6 +135,135 @@ func TestClientTracePairsAfterPTYHangup(t *testing.T) { | |||||
| 133 | } | 135 | } | ||
| 134 | } | 136 | } | ||
| 135 | 137 | ||||
| 138 | + | // TestClientGracefulShutdownDrainsTracedStdioDescendant proves the launcher's | |||
| 139 | + | // normal client stop exits the attached shell before sending any signal. The | |||
| 140 | + | // client then closes and waits for its ssh transport, giving the traced _stdio | |||
| 141 | + | // descendant time to serialize its blocked receive's end mark and exit before | |||
| 142 | + | // the client is reaped. | |||
| 143 | + | func TestClientGracefulShutdownDrainsTracedStdioDescendant(t *testing.T) { | |||
| 144 | + | if testing.Short() { | |||
| 145 | + | t.Skip("public CLI stdio descendant lifecycle") | |||
| 146 | + | } | |||
| 147 | + | root, err := filepath.Abs(filepath.Join("..", "..")) | |||
| 148 | + | if err != nil { | |||
| 149 | + | t.Fatal(err) | |||
| 150 | + | } | |||
| 151 | + | runDir, err := os.MkdirTemp("", "vev-stdio-drain-") | |||
| 152 | + | if err != nil { | |||
| 153 | + | t.Fatal(err) | |||
| 154 | + | } | |||
| 155 | + | removeTestTree(t, runDir) | |||
| 156 | + | bin := filepath.Join(runDir, "vev") | |||
| 157 | + | build := exec.Command("go", "build", "-o", bin, "./") | |||
| 158 | + | build.Dir = root | |||
| 159 | + | if output, err := build.CombinedOutput(); err != nil { | |||
| 160 | + | t.Fatalf("build public CLI: %v\n%s", err, output) | |||
| 161 | + | } | |||
| 162 | + | ||||
| 163 | + | s := scenario{ID: "stdio-drain", Roles: []string{"daemon", "ssh_stdio_peer", "client"}} | |||
| 164 | + | maps := make([]processMapping, 0, len(s.Roles)) | |||
| 165 | + | for i, role := range s.Roles { | |||
| 166 | + | id := fmt.Sprintf("%s-r001-%s-%02d", safeName(s.ID), safeName(role), i+1) | |||
| 167 | + | path := filepath.Join(runDir, id+".jsonl") | |||
| 168 | + | f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) | |||
| 169 | + | if err != nil { | |||
| 170 | + | t.Fatal(err) | |||
| 171 | + | } | |||
| 172 | + | if err := f.Close(); err != nil { | |||
| 173 | + | t.Fatal(err) | |||
| 174 | + | } | |||
| 175 | + | maps = append(maps, processMapping{ProcessID: id, ClockDomain: id, TracePath: path, Role: role, Scenario: s.ID, Run: 1, Identity: "harness-assigned:" + id}) | |||
| 176 | + | } | |||
| 177 | + | ||||
| 178 | + | launcher := &cliLauncher{bin: bin} | |||
| 179 | + | var daemon, peer launchedProcess | |||
| 180 | + | var client *cliProcess | |||
| 181 | + | var peerMapping processMapping | |||
| 182 | + | for _, pm := range launchOrder(maps) { | |||
| 183 | + | var command roleCommand | |||
| 184 | + | switch pm.Role { | |||
| 185 | + | case "daemon": | |||
| 186 | + | command = roleCommand{Args: []string{"--daemon"}} | |||
| 187 | + | case "ssh_stdio_peer": | |||
| 188 | + | // An empty remote session requests an ephemeral session, avoiding a | |||
| 189 | + | // separate bootstrap client while still executing the traced _stdio mode. | |||
| 190 | + | command = roleCommand{Args: []string{"_stdio", ""}, Transport: transport{ID: "ssh_stdio", Kind: "ssh_stdio"}} | |||
| 191 | + | case "client": | |||
| 192 | + | command = roleCommand{Args: []string{"attach", "harness@127.0.0.1"}} | |||
| 193 | + | } | |||
| 194 | + | process, err := launcher.Launch(pm, command) | |||
| 195 | + | if err != nil { | |||
| 196 | + | t.Fatalf("launch %s: %v", pm.Role, err) | |||
| 197 | + | } | |||
| 198 | + | switch pm.Role { | |||
| 199 | + | case "daemon": | |||
| 200 | + | daemon = process | |||
| 201 | + | readyProcess, ok := process.(interface{ WaitReady() error }) | |||
| 202 | + | if !ok { | |||
| 203 | + | t.Fatalf("daemon process type %T does not support readiness", process) | |||
| 204 | + | } | |||
| 205 | + | if err := readyProcess.WaitReady(); err != nil { | |||
| 206 | + | t.Fatalf("daemon readiness: %v", err) | |||
| 207 | + | } | |||
| 208 | + | case "ssh_stdio_peer": | |||
| 209 | + | peer, peerMapping = process, pm | |||
| 210 | + | case "client": | |||
| 211 | + | var ok bool | |||
| 212 | + | client, ok = process.(*cliProcess) | |||
| 213 | + | if !ok { | |||
| 214 | + | t.Fatalf("client process type = %T, want *cliProcess", process) | |||
| 215 | + | } | |||
| 216 | + | } | |||
| 217 | + | } | |||
| 218 | + | if daemon == nil || peer == nil || client == nil { | |||
| 219 | + | t.Fatal("did not launch complete stdio process tree") | |||
| 220 | + | } | |||
| 221 | + | defer func() { | |||
| 222 | + | if err := peer.Close(); err != nil { | |||
| 223 | + | t.Errorf("close peer role: %v", err) | |||
| 224 | + | } | |||
| 225 | + | if err := daemon.Close(); err != nil { | |||
| 226 | + | t.Errorf("close daemon role: %v", err) | |||
| 227 | + | } | |||
| 228 | + | }() | |||
| 229 | + | defer func() { | |||
| 230 | + | if err := client.Close(); err != nil { | |||
| 231 | + | t.Errorf("close client role: %v", err) | |||
| 232 | + | } | |||
| 233 | + | }() | |||
| 234 | + | ||||
| 235 | + | if err := waitForInflightReceive(peerMapping.TracePath, 10*time.Second); err != nil { | |||
| 236 | + | t.Fatalf("stdio descendant never reached blocked receive: %v", err) | |||
| 237 | + | } | |||
| 238 | + | processGroup := client.cmd.Process.Pid | |||
| 239 | + | if err := client.Close(); err != nil { | |||
| 240 | + | t.Fatalf("graceful client close: %v", err) | |||
| 241 | + | } | |||
| 242 | + | if _, err := mergeProcessTraces([]processMapping{peerMapping}); err != nil { | |||
| 243 | + | t.Fatalf("stdio descendant trace did not drain: %v", err) | |||
| 244 | + | } | |||
| 245 | + | if err := waitForProcessGroupGone(processGroup, 10*time.Second); err != nil { | |||
| 246 | + | t.Fatal(err) | |||
| 247 | + | } | |||
| 248 | + | } | |||
| 249 | + | ||||
| 250 | + | func waitForProcessGroupGone(pgid int, timeout time.Duration) error { | |||
| 251 | + | deadline := time.Now().Add(timeout) | |||
| 252 | + | for { | |||
| 253 | + | err := syscall.Kill(-pgid, 0) | |||
| 254 | + | if errors.Is(err, syscall.ESRCH) { | |||
| 255 | + | return nil | |||
| 256 | + | } | |||
| 257 | + | if err != nil { | |||
| 258 | + | return fmt.Errorf("probe process group %d: %w", pgid, err) | |||
| 259 | + | } | |||
| 260 | + | if time.Now().After(deadline) { | |||
| 261 | + | return fmt.Errorf("process group %d remained after client shutdown", pgid) | |||
| 262 | + | } | |||
| 263 | + | time.Sleep(10 * time.Millisecond) | |||
| 264 | + | } | |||
| 265 | + | } | |||
| 266 | + | ||||
| 136 | // waitForInflightReceive polls a process trace until it holds an unpaired | 267 | // waitForInflightReceive polls a process trace until it holds an unpaired | ||
| 137 | // adapter_receive_start (more starts than ends). It tolerates a partially | 268 | // adapter_receive_start (more starts than ends). It tolerates a partially | ||
| 138 | // written trailing line while the traced process appends concurrently. | 269 | // written trailing line while the traced process appends concurrently. | ||
M cmd/vev-perf-harness/launcher.go +35 -37
| @@ -117,21 +117,22 @@ type terminalOutput interface { | |||||
| 117 | } | 117 | } | ||
| 118 | 118 | ||||
| 119 | type cliProcess struct { | 119 | type cliProcess struct { | ||
| 120 | - | cmd *exec.Cmd | 120 | + | cmd *exec.Cmd |
| 121 | - | pty io.ReadWriteCloser | 121 | + | pty io.ReadWriteCloser |
| 122 | - | output terminalOutput | 122 | + | output terminalOutput |
| 123 | - | chunks chan []byte | 123 | + | chunks chan []byte |
| 124 | - | done chan struct{} | 124 | + | done chan struct{} |
| 125 | - | closed sync.Once | 125 | + | closed sync.Once |
| 126 | - | waitErr chan error | 126 | + | waitErr chan error |
| 127 | - | waitTimeout func() <-chan time.Time | 127 | + | waitTimeout func() <-chan time.Time |
| 128 | - | forceCleanup func() | 128 | + | gracefulShutdown func() |
| 129 | - | forceKill func() | 129 | + | forceCleanup func() |
| 130 | - | shutdown func() error | 130 | + | forceKill func() |
| 131 | - | cleanupRuntime func() error | 131 | + | shutdown func() error |
| 132 | - | terminalReady bool | 132 | + | cleanupRuntime func() error |
| 133 | - | readyPath string | 133 | + | terminalReady bool |
| 134 | - | closeResult error | 134 | + | readyPath string |
| 135 | + | closeResult error | |||
| 135 | } | 136 | } | ||
| 136 | 137 | ||||
| 137 | func (l *cliLauncher) Launch(m processMapping, role roleCommand) (launchedProcess, error) { | 138 | func (l *cliLauncher) Launch(m processMapping, role roleCommand) (launchedProcess, error) { | ||
| @@ -203,6 +204,12 @@ func (l *cliLauncher) Launch(m processMapping, role roleCommand) (launchedProces | |||||
| 203 | } | 204 | } | ||
| 204 | _ = slave.Close() | 205 | _ = slave.Close() | ||
| 205 | p.pty, p.output, p.chunks, p.done = master, output, make(chan []byte, 32), make(chan struct{}) | 206 | p.pty, p.output, p.chunks, p.done = master, output, make(chan []byte, 32), make(chan struct{}) | ||
| 207 | + | // Stop workload generation through the attached shell before any signal. | |||
| 208 | + | // A normal shell exit makes the daemon detach the client; the client then | |||
| 209 | + | // closes and waits for its ssh transport, allowing a traced _stdio | |||
| 210 | + | // descendant to flush end marks and exit. Process-group signals are reserved | |||
| 211 | + | // for timeout escalation because they would kill the descendant concurrently. | |||
| 212 | + | p.gracefulShutdown = func() { _, _ = master.Write([]byte("exit\n")) } | |||
| 206 | go p.copyTerminal() | 213 | go p.copyTerminal() | ||
| 207 | } else { | 214 | } else { | ||
| 208 | // Every launched role owns a dedicated process group, so the bounded | 215 | // Every launched role owns a dedicated process group, so the bounded | ||
| @@ -629,11 +636,10 @@ func (e *ptyLocalEcho) applicationOutput(chunk []byte) []byte { | |||||
| 629 | // Close tears down a launched role along a bounded escalation ladder. Each rung | 636 | // Close tears down a launched role along a bounded escalation ladder. Each rung | ||
| 630 | // performs one action, then waits a single closeDeadline for the process to be | 637 | // performs one action, then waits a single closeDeadline for the process to be | ||
| 631 | // reaped before the next rung escalates. The graceful rungs — the public daemon | 638 | // reaped before the next rung escalates. The graceful rungs — the public daemon | ||
| 632 | - | // stop, the client shell exit, the controlling-terminal hangup, and SIGTERM — | 639 | + | // stop and a client shell exit — let a role close its blocked transport |
| 633 | - | // let a role close its blocked transport operations and serialize their end | 640 | + | // operations and drain descendants' end marks. Process-group SIGTERM is reserved |
| 634 | - | // marks; only a SIGKILL severs a process before it can, so only that rung fails | 641 | + | // for timeout escalation; only a SIGKILL rung fails teardown. Every path closes |
| 635 | - | // teardown. Every path closes the PTY exactly once and finalizes output and | 642 | + | // the PTY exactly once and finalizes output and runtime state. |
| 636 | - | // runtime state. | |||
| 637 | func (p *cliProcess) Close() error { | 643 | func (p *cliProcess) Close() error { | ||
| 638 | p.closed.Do(func() { | 644 | p.closed.Do(func() { | ||
| 639 | // 1. Drain terminal output for the whole teardown. Without an active | 645 | // 1. Drain terminal output for the whole teardown. Without an active | ||
| @@ -682,27 +688,19 @@ func (p *cliProcess) Close() error { | |||||
| 682 | _ = p.shutdown() | 688 | _ = p.shutdown() | ||
| 683 | waitForExit() | 689 | waitForExit() | ||
| 684 | } | 690 | } | ||
| 685 | - | // 3. Client role: a public terminal client owns a shell session. Ask that | 691 | + | // 3. Client role: stop the shell normally. Session teardown detaches the |
| 686 | - | // shell to exit so the client detaches and its transport finishes its | 692 | + | // client, which closes and waits for ssh/_stdio before it exits. |
| 687 | - | // receive span cleanly. | 693 | + | if p.gracefulShutdown != nil && !reaped() { |
| 688 | - | if p.pty != nil && !reaped() { | 694 | + | p.gracefulShutdown() |
| 689 | - | _, _ = p.pty.Write([]byte("exit\n")) | |||
| 690 | - | waitForExit() | |||
| 691 | - | } | |||
| 692 | - | // 4. Hang up the controlling terminal. The attach path installs a signal | |||
| 693 | - | // handler, so this SIGHUP is a graceful detach — not an abrupt kill — and | |||
| 694 | - | // the client still flushes its in-flight receive end mark. | |||
| 695 | - | if p.pty != nil && !reaped() { | |||
| 696 | - | closePTY() | |||
| 697 | waitForExit() | 695 | waitForExit() | ||
| 698 | } | 696 | } | ||
| 699 | - | // 5. SIGTERM the role's own process group. The daemon and, through its | 697 | + | // 4. Escalate a role that ignored graceful shutdown to process-group |
| 700 | - | // attach signal handler, the client both treat this as a graceful stop. | 698 | + | // SIGTERM. This remains bounded and cleans up hung descendants. |
| 701 | if !reaped() { | 699 | if !reaped() { | ||
| 702 | p.forceProcessGroupCleanup() | 700 | p.forceProcessGroupCleanup() | ||
| 703 | waitForExit() | 701 | waitForExit() | ||
| 704 | } | 702 | } | ||
| 705 | - | // 6. SIGKILL the role's process group. This severs the process; an | 703 | + | // 5. SIGKILL the role's process group. This severs the process; an |
| 706 | // in-flight observed operation cannot be closed, so its trace may truncate. | 704 | // in-flight observed operation cannot be closed, so its trace may truncate. | ||
| 707 | killed := false | 705 | killed := false | ||
| 708 | if !reaped() { | 706 | if !reaped() { | ||
| @@ -710,12 +708,12 @@ func (p *cliProcess) Close() error { | |||||
| 710 | p.forceProcessGroupKill() | 708 | p.forceProcessGroupKill() | ||
| 711 | waitForExit() | 709 | waitForExit() | ||
| 712 | } | 710 | } | ||
| 713 | - | // 7. A SIGKILL escalation cannot guarantee a complete trace. Fail teardown | 711 | + | // 6. A SIGKILL escalation cannot guarantee a complete trace. Fail teardown |
| 714 | // with a clear cause rather than letting the truncation surface later as a | 712 | // with a clear cause rather than letting the truncation surface later as a | ||
| 715 | // downstream merge mystery. The graceful rungs never error on their own. | 713 | // downstream merge mystery. The graceful rungs never error on their own. | ||
| 716 | if killed { | 714 | if killed { | ||
| 717 | if exited { | 715 | if exited { | ||
| 718 | - | p.closeResult = errors.New("public CLI process did not exit after hangup and SIGTERM; killed") | 716 | + | p.closeResult = errors.New("public CLI process did not exit after graceful shutdown and SIGTERM; killed") |
| 719 | } else { | 717 | } else { | ||
| 720 | p.closeResult = errors.New("timed out reaping public CLI process") | 718 | p.closeResult = errors.New("timed out reaping public CLI process") | ||
| 721 | } | 719 | } | ||
M cmd/vev-perf-harness/launcher_test.go +48 -34
| @@ -27,29 +27,22 @@ func (discardTerminalOutput) Close() error { return nil } | |||||
| 27 | func TestCLIProcessCloseDrainsQueuedTerminalOutputBeforeClientExit(t *testing.T) { | 27 | func TestCLIProcessCloseDrainsQueuedTerminalOutputBeforeClientExit(t *testing.T) { | ||
| 28 | harnessPTY, clientPTY := net.Pipe() | 28 | harnessPTY, clientPTY := net.Pipe() | ||
| 29 | t.Cleanup(func() { _ = clientPTY.Close() }) | 29 | t.Cleanup(func() { _ = clientPTY.Close() }) | ||
| 30 | - | waitErr := make(chan error) | 30 | + | waitErr := make(chan error, 1) |
| 31 | secondRead := make(chan struct{}) | 31 | secondRead := make(chan struct{}) | ||
| 32 | p := &cliProcess{ | 32 | p := &cliProcess{ | ||
| 33 | - | pty: harnessPTY, | 33 | + | pty: harnessPTY, |
| 34 | - | output: discardTerminalOutput{}, | 34 | + | output: discardTerminalOutput{}, |
| 35 | - | chunks: make(chan []byte, 1), | 35 | + | chunks: make(chan []byte, 1), |
| 36 | - | done: make(chan struct{}), | 36 | + | done: make(chan struct{}), |
| 37 | - | waitErr: waitErr, | 37 | + | waitErr: waitErr, |
| 38 | - | waitTimeout: func() <-chan time.Time { return time.After(time.Second) }, | 38 | + | waitTimeout: func() <-chan time.Time { return time.After(time.Second) }, |
| 39 | + | forceCleanup: func() { waitErr <- nil }, | |||
| 39 | } | 40 | } | ||
| 40 | go p.copyTerminal() | 41 | go p.copyTerminal() | ||
| 41 | go func() { | 42 | go func() { | ||
| 42 | _, _ = clientPTY.Write([]byte("first")) | 43 | _, _ = clientPTY.Write([]byte("first")) | ||
| 43 | _, _ = clientPTY.Write([]byte("second")) | 44 | _, _ = clientPTY.Write([]byte("second")) | ||
| 44 | close(secondRead) | 45 | close(secondRead) | ||
| 45 | - | buf := make([]byte, len("exit\n")) | |||
| 46 | - | if _, err := clientPTY.Read(buf); err != nil { | |||
| 47 | - | return | |||
| 48 | - | } | |||
| 49 | - | if _, err := clientPTY.Write([]byte("third")); err != nil { | |||
| 50 | - | return | |||
| 51 | - | } | |||
| 52 | - | waitErr <- nil | |||
| 53 | }() | 46 | }() | ||
| 54 | select { | 47 | select { | ||
| 55 | case <-secondRead: | 48 | case <-secondRead: | ||
| @@ -134,6 +127,26 @@ func TestCLIProcessCloseIsBoundedWhenPublicRoleDoesNotExit(t *testing.T) { | |||||
| 134 | } | 127 | } | ||
| 135 | } | 128 | } | ||
| 136 | 129 | ||||
| 130 | + | func TestCLIProcessCloseEscalatesHungClientAfterGracefulDrainTimeout(t *testing.T) { | |||
| 131 | + | deadline := make(chan time.Time) | |||
| 132 | + | close(deadline) | |||
| 133 | + | var graceful, term, kill int | |||
| 134 | + | p := &cliProcess{ | |||
| 135 | + | pty: &orderedPTY{close: func() {}}, | |||
| 136 | + | waitErr: make(chan error), | |||
| 137 | + | waitTimeout: func() <-chan time.Time { return deadline }, | |||
| 138 | + | gracefulShutdown: func() { graceful++ }, | |||
| 139 | + | forceCleanup: func() { term++ }, | |||
| 140 | + | forceKill: func() { kill++ }, | |||
| 141 | + | } | |||
| 142 | + | if err := p.Close(); err == nil { | |||
| 143 | + | t.Fatal("hung client close succeeded") | |||
| 144 | + | } | |||
| 145 | + | if graceful != 1 || term != 1 || kill != 1 { | |||
| 146 | + | t.Fatalf("graceful/TERM/KILL calls = %d/%d/%d, want 1/1/1", graceful, term, kill) | |||
| 147 | + | } | |||
| 148 | + | } | |||
| 149 | + | ||||
| 137 | func TestCLIProcessCloseEscalationOutcome(t *testing.T) { | 150 | func TestCLIProcessCloseEscalationOutcome(t *testing.T) { | ||
| 138 | tests := []struct { | 151 | tests := []struct { | ||
| 139 | name string | 152 | name string | ||
| @@ -252,6 +265,14 @@ func TestCLILauncherIsolatesRoleXDGEnvironmentAcrossRepetitions(t *testing.T) { | |||||
| 252 | if err != nil { | 265 | if err != nil { | ||
| 253 | t.Fatalf("run %d %s launch: %v", run, role, err) | 266 | t.Fatalf("run %d %s launch: %v", run, role, err) | ||
| 254 | } | 267 | } | ||
| 268 | + | // The fixture exits on its own. Synchronize with that exit instead of | |||
| 269 | + | // letting Close race its SIGTERM against the environment capture. | |||
| 270 | + | proc, ok := p.(*cliProcess) | |||
| 271 | + | if !ok { | |||
| 272 | + | t.Fatalf("run %d %s process type = %T, want *cliProcess", run, role, p) | |||
| 273 | + | } | |||
| 274 | + | exitErr := <-proc.waitErr | |||
| 275 | + | proc.waitErr <- exitErr | |||
| 255 | runtimeDir := launcher.runtimes[runDir] | 276 | runtimeDir := launcher.runtimes[runDir] | ||
| 256 | if err := p.Close(); err != nil { | 277 | if err := p.Close(); err != nil { | ||
| 257 | t.Fatalf("run %d %s close: %v", run, role, err) | 278 | t.Fatalf("run %d %s close: %v", run, role, err) | ||
| @@ -383,36 +404,29 @@ func TestTraceEnvironmentMatchesProcessMapping(t *testing.T) { | |||||
| 383 | } | 404 | } | ||
| 384 | } | 405 | } | ||
| 385 | 406 | ||||
| 386 | - | func TestCLIProcessCloseGracefulDetachCompletesSpanBeforeForcedCleanup(t *testing.T) { | 407 | + | func TestCLIProcessCloseStopsShellBeforeProcessGroupCleanup(t *testing.T) { |
| 387 | var order []string | 408 | var order []string | ||
| 388 | waitErr := make(chan error, 1) | 409 | waitErr := make(chan error, 1) | ||
| 389 | - | timeout := make(chan time.Time) | |||
| 390 | p := &cliProcess{ | 410 | p := &cliProcess{ | ||
| 391 | - | pty: &orderedPTY{ | 411 | + | pty: &orderedPTY{close: func() { order = append(order, "pty_closed") }}, |
| 392 | - | write: func(input []byte) { | 412 | + | output: &orderedOutput{close: func() { order = append(order, "output_closed") }}, |
| 393 | - | if string(input) != "exit\n" { | 413 | + | waitErr: waitErr, |
| 394 | - | t.Fatalf("graceful exit input=%q", input) | 414 | + | gracefulShutdown: func() { |
| 395 | - | } | 415 | + | order = append(order, "shell_exit") |
| 396 | - | order = append(order, "exit_written") | 416 | + | // The client closes and waits for its transport descendant before it |
| 397 | - | // The graceful client exit has closed its transport and emitted | 417 | + | // reports its own exit to the launcher. |
| 398 | - | // its adapter receive-end mark before cmd.Wait completes. | 418 | + | order = append(order, "descendant_receive_end") |
| 399 | - | order = append(order, "adapter_receive_end") | 419 | + | waitErr <- nil |
| 400 | - | waitErr <- nil | |||
| 401 | - | }, | |||
| 402 | - | close: func() { order = append(order, "pty_closed") }, | |||
| 403 | }, | 420 | }, | ||
| 404 | - | output: &orderedOutput{close: func() { order = append(order, "output_closed") }}, | |||
| 405 | - | waitErr: waitErr, | |||
| 406 | - | waitTimeout: func() <-chan time.Time { return timeout }, | |||
| 407 | forceCleanup: func() { | 421 | forceCleanup: func() { | ||
| 408 | - | order = append(order, "forced_process_group_cleanup") | 422 | + | t.Fatal("normal shutdown sent SIGTERM to the traced descendant process group") |
| 409 | }, | 423 | }, | ||
| 410 | } | 424 | } | ||
| 411 | 425 | ||||
| 412 | if err := p.Close(); err != nil { | 426 | if err := p.Close(); err != nil { | ||
| 413 | t.Fatal(err) | 427 | t.Fatal(err) | ||
| 414 | } | 428 | } | ||
| 415 | - | if !equalStrings(order, []string{"exit_written", "adapter_receive_end", "pty_closed", "output_closed"}) { | 429 | + | if !equalStrings(order, []string{"shell_exit", "descendant_receive_end", "pty_closed", "output_closed"}) { |
| 416 | t.Fatalf("cleanup order=%q", order) | 430 | t.Fatalf("cleanup order=%q", order) | ||
| 417 | } | 431 | } | ||
| 418 | } | 432 | } | ||
M docs/configuration.md +8 -0
| @@ -74,6 +74,14 @@ code.detach = DET | |||||
| 74 | 74 | ||||
| 75 | Invalid values log a warning and resolve that setting to its default on both initial load and reload. | 75 | Invalid values log a warning and resolve that setting to its default on both initial load and reload. | ||
| 76 | 76 | ||||
| 77 | + | ## Logs and durable state | |||
| 78 | + | ||||
| 79 | + | Set `VEV_LOG=debug`, `VEV_LOG=warn`, or `VEV_LOG=error` to change verbosity; the default is `info`. JSON-line logs such as `vev-daemon.log` live in `$XDG_STATE_HOME/vev`, or `~/.local/state/vev` when unset. The same state directory contains the strict session catalogue, migration journal, notices, and `snapshots/`. The lifecycle lock and socket live in `$XDG_RUNTIME_DIR/vev` (with platform runtime fallbacks). | |||
| 80 | + | ||||
| 81 | + | Recovery events include `lifecycle_owner_wait`, `lifecycle_owner_acquired`, `lifecycle_owner_released`, `catalogue_validated`, `catalogue_compaction_recovery_complete`, `session_restore_complete`, `fallback_checkpoint_promoted`, `snapshot_head_repair_complete`, `session_degraded`, `snapshot_maintenance_progress`, `interrupted_transaction_recovery_complete`, and `daemon_startup_complete`. | |||
| 82 | + | ||||
| 83 | + | Catalogue failure is fail-closed: vev does not publish an empty replacement daemon. Preserve the state directory, inspect `catalogue_validation_failed`, correct storage or ownership problems, and retry without editing catalogue files. See [Durable session recovery](durable-session-recovery.md) for explicit recovery commands, migration, diagnostics, and the committed checkpoint plus up to two direct fallbacks retention policy. | |||
| 84 | + | ||||
| 77 | ## Theme | 85 | ## Theme | ||
| 78 | 86 | ||||
| 79 | With `theme = auto`, vev follows the terminal's reported foreground, background, light/dark scheme, and ANSI palette. `theme.palette = on` is the default. `theme.accent = auto` derives one accent from the terminal's chromatic ANSI colors; repeated terminal colors are preferred, otherwise vev uses an eligible blue slot when available. `theme.accent = 0` through `theme.accent = 15` selects exactly that ANSI slot. Arbitrary RGB values and `off` are not accepted for `theme.accent`. | 87 | With `theme = auto`, vev follows the terminal's reported foreground, background, light/dark scheme, and ANSI palette. `theme.palette = on` is the default. `theme.accent = auto` derives one accent from the terminal's chromatic ANSI colors; repeated terminal colors are preferred, otherwise vev uses an eligible blue slot when available. `theme.accent = 0` through `theme.accent = 15` selects exactly that ANSI slot. Arbitrary RGB values and `off` are not accepted for `theme.accent`. | ||
A docs/durable-session-recovery.md +29 -0
| @@ -0,0 +1,29 @@ | |||||
| 1 | + | # Durable session recovery | |||
| 2 | + | ||||
| 3 | + | vev opens named-session state only while holding `$XDG_RUNTIME_DIR/vev/lifecycle.lock` (or the platform runtime fallback). Catalogue corruption prevents socket publication; vev never substitutes an empty healthy daemon. | |||
| 4 | + | ||||
| 5 | + | ## Session states | |||
| 6 | + | ||||
| 7 | + | `stopped` is safe fresh metadata, `restoring` is validation in progress, and `degraded` preserves uncertain state without starting a replacement shell. A degraded session remains visible in `vev ls`, but attach is refused until an explicit recovery action succeeds. | |||
| 8 | + | ||||
| 9 | + | ## Recovery commands | |||
| 10 | + | ||||
| 11 | + | - `vev cmd -s NAME session-recovery discard` | |||
| 12 | + | ||||
| 13 | + | Discard creates a new incarnation and retains the old record and snapshots under `snapshots/quarantine/` until an explicit later purge. | |||
| 14 | + | ||||
| 15 | + | ## Migration and retention | |||
| 16 | + | ||||
| 17 | + | The v0.x migration is resumable, retains legacy sources, and validates each referenced HEAD. Healthy sessions retain the committed checkpoint and up to two direct fallbacks; degraded, deleting, migrating, and unresolved transactions remain pinned. | |||
| 18 | + | ||||
| 19 | + | ## Fail-closed startup | |||
| 20 | + | ||||
| 21 | + | If the catalogue cannot be opened or validated, the daemon does not publish its socket. Do not delete or edit state files. Check `vev-daemon.log` for `catalogue_validation_failed`, preserve the complete state directory, correct filesystem ownership or capacity problems, and retry. For corruption, make a backup before attempting recovery or exporting data. | |||
| 22 | + | ||||
| 23 | + | A `lifecycle_owner_wait` event normally means another daemon is initializing or shutting down. Wait for ownership transfer; only terminate the prior process when its identity is known. The operating system releases `lifecycle.lock` when that process exits. | |||
| 24 | + | ||||
| 25 | + | ## Diagnostics | |||
| 26 | + | ||||
| 27 | + | Durable state and recovery journals are under `$XDG_STATE_HOME/vev` (default `~/.local/state/vev`). Snapshot quarantine is under its `snapshots/quarantine/` tree. Runtime ownership is under `$XDG_RUNTIME_DIR/vev`; logs are JSON lines in the state directory. | |||
| 28 | + | ||||
| 29 | + | Use session names, incarnation IDs, generation numbers, reason codes, and cursors from recovery events when diagnosing a failure. Recovery logs never include terminal or snapshot object contents. | |||
A internal/adapters/lifecycle/flock.go +105 -0
| @@ -0,0 +1,105 @@ | |||||
| 1 | + | //go:build unix | |||
| 2 | + | ||||
| 3 | + | // Package lifecycle provides exclusive process ownership of vev's durable | |||
| 4 | + | // state for the complete daemon lifetime. | |||
| 5 | + | package lifecycle | |||
| 6 | + | ||||
| 7 | + | import ( | |||
| 8 | + | "context" | |||
| 9 | + | "errors" | |||
| 10 | + | "fmt" | |||
| 11 | + | "os" | |||
| 12 | + | "path/filepath" | |||
| 13 | + | "sync" | |||
| 14 | + | "syscall" | |||
| 15 | + | "time" | |||
| 16 | + | ||||
| 17 | + | "github.com/bnema/vev/pkg/safedir" | |||
| 18 | + | ) | |||
| 19 | + | ||||
| 20 | + | // ErrBusy reports that another process currently owns durable session state. | |||
| 21 | + | var ErrBusy = errors.New("durable session state is busy") | |||
| 22 | + | ||||
| 23 | + | // Owner holds the lifecycle lock descriptor open until Release. | |||
| 24 | + | type Owner struct { | |||
| 25 | + | file *os.File | |||
| 26 | + | path string | |||
| 27 | + | once sync.Once | |||
| 28 | + | err error | |||
| 29 | + | } | |||
| 30 | + | ||||
| 31 | + | // Path returns the fixed lifecycle lock path in runtimeDir. | |||
| 32 | + | func Path(runtimeDir string) string { | |||
| 33 | + | return filepath.Join(runtimeDir, "lifecycle.lock") | |||
| 34 | + | } | |||
| 35 | + | ||||
| 36 | + | // TryAcquire attempts to take lifecycle ownership without waiting. | |||
| 37 | + | func TryAcquire(runtimeDir string) (*Owner, error) { | |||
| 38 | + | if err := safedir.EnsurePrivate(runtimeDir); err != nil { | |||
| 39 | + | return nil, fmt.Errorf("lifecycle: securing runtime directory: %w", err) | |||
| 40 | + | } | |||
| 41 | + | ||||
| 42 | + | path := Path(runtimeDir) | |||
| 43 | + | file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) | |||
| 44 | + | if err != nil { | |||
| 45 | + | return nil, fmt.Errorf("lifecycle: opening %s: %w", path, err) | |||
| 46 | + | } | |||
| 47 | + | if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { | |||
| 48 | + | closeErr := file.Close() | |||
| 49 | + | if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { | |||
| 50 | + | return nil, errors.Join(ErrBusy, closeErr) | |||
| 51 | + | } | |||
| 52 | + | return nil, errors.Join(fmt.Errorf("lifecycle: locking %s: %w", path, err), closeErr) | |||
| 53 | + | } | |||
| 54 | + | return &Owner{file: file, path: path}, nil | |||
| 55 | + | } | |||
| 56 | + | ||||
| 57 | + | // Acquire waits until lifecycle ownership is available or ctx is cancelled. | |||
| 58 | + | // Only lock contention is retried; filesystem and lock errors fail closed. | |||
| 59 | + | func Acquire(ctx context.Context, runtimeDir string, retry time.Duration) (*Owner, error) { | |||
| 60 | + | if retry <= 0 { | |||
| 61 | + | retry = time.Millisecond | |||
| 62 | + | } | |||
| 63 | + | for { | |||
| 64 | + | if err := ctx.Err(); err != nil { | |||
| 65 | + | return nil, err | |||
| 66 | + | } | |||
| 67 | + | owner, err := TryAcquire(runtimeDir) | |||
| 68 | + | if err == nil { | |||
| 69 | + | return owner, nil | |||
| 70 | + | } | |||
| 71 | + | if !errors.Is(err, ErrBusy) { | |||
| 72 | + | return nil, err | |||
| 73 | + | } | |||
| 74 | + | ||||
| 75 | + | timer := time.NewTimer(retry) | |||
| 76 | + | select { | |||
| 77 | + | case <-ctx.Done(): | |||
| 78 | + | timer.Stop() | |||
| 79 | + | return nil, ctx.Err() | |||
| 80 | + | case <-timer.C: | |||
| 81 | + | } | |||
| 82 | + | } | |||
| 83 | + | } | |||
| 84 | + | ||||
| 85 | + | // Release unlocks and closes the owner exactly once. | |||
| 86 | + | func (o *Owner) Release() error { | |||
| 87 | + | if o == nil { | |||
| 88 | + | return nil | |||
| 89 | + | } | |||
| 90 | + | o.once.Do(func() { | |||
| 91 | + | if o.file == nil { | |||
| 92 | + | return | |||
| 93 | + | } | |||
| 94 | + | unlockErr := syscall.Flock(int(o.file.Fd()), syscall.LOCK_UN) | |||
| 95 | + | closeErr := o.file.Close() | |||
| 96 | + | if unlockErr != nil { | |||
| 97 | + | unlockErr = fmt.Errorf("lifecycle: unlocking %s: %w", o.path, unlockErr) | |||
| 98 | + | } | |||
| 99 | + | if closeErr != nil { | |||
| 100 | + | closeErr = fmt.Errorf("lifecycle: closing %s: %w", o.path, closeErr) | |||
| 101 | + | } | |||
| 102 | + | o.err = errors.Join(unlockErr, closeErr) | |||
| 103 | + | }) | |||
| 104 | + | return o.err | |||
| 105 | + | } | |||
A internal/adapters/lifecycle/flock_test.go +95 -0
| @@ -0,0 +1,95 @@ | |||||
| 1 | + | //go:build unix | |||
| 2 | + | ||||
| 3 | + | package lifecycle | |||
| 4 | + | ||||
| 5 | + | import ( | |||
| 6 | + | "bufio" | |||
| 7 | + | "context" | |||
| 8 | + | "errors" | |||
| 9 | + | "fmt" | |||
| 10 | + | "io" | |||
| 11 | + | "os" | |||
| 12 | + | "os/exec" | |||
| 13 | + | "path/filepath" | |||
| 14 | + | "testing" | |||
| 15 | + | "time" | |||
| 16 | + | ||||
| 17 | + | "github.com/stretchr/testify/require" | |||
| 18 | + | ) | |||
| 19 | + | ||||
| 20 | + | func TestLifecycleLockProcessHelper(t *testing.T) { | |||
| 21 | + | if os.Getenv("VEV_LIFECYCLE_LOCK_HELPER") != "1" { | |||
| 22 | + | t.Skip("helper process") | |||
| 23 | + | } | |||
| 24 | + | ||||
| 25 | + | owner, err := TryAcquire(os.Getenv("VEV_LIFECYCLE_LOCK_DIR")) | |||
| 26 | + | require.NoError(t, err) | |||
| 27 | + | defer func() { require.NoError(t, owner.Release()) }() | |||
| 28 | + | ||||
| 29 | + | _, err = fmt.Fprintln(os.Stdout, "locked") | |||
| 30 | + | require.NoError(t, err) | |||
| 31 | + | _, err = io.Copy(io.Discard, os.Stdin) | |||
| 32 | + | require.NoError(t, err) | |||
| 33 | + | } | |||
| 34 | + | ||||
| 35 | + | func TestLifecycleLock(t *testing.T) { | |||
| 36 | + | t.Run("process contention and reacquire", testLifecycleLockProcessContentionAndReacquire) | |||
| 37 | + | t.Run("acquire cancellation", testLifecycleLockAcquireCancellation) | |||
| 38 | + | t.Run("unavailable runtime directory", testLifecycleLockUnavailableRuntimeDirectory) | |||
| 39 | + | } | |||
| 40 | + | ||||
| 41 | + | func testLifecycleLockProcessContentionAndReacquire(t *testing.T) { | |||
| 42 | + | runtimeDir := filepath.Join(t.TempDir(), "runtime") | |||
| 43 | + | cmd := exec.Command(os.Args[0], "-test.run=^TestLifecycleLockProcessHelper$") | |||
| 44 | + | cmd.Env = append(os.Environ(), | |||
| 45 | + | "VEV_LIFECYCLE_LOCK_HELPER=1", | |||
| 46 | + | "VEV_LIFECYCLE_LOCK_DIR="+runtimeDir, | |||
| 47 | + | ) | |||
| 48 | + | stdin, err := cmd.StdinPipe() | |||
| 49 | + | require.NoError(t, err) | |||
| 50 | + | stdout, err := cmd.StdoutPipe() | |||
| 51 | + | require.NoError(t, err) | |||
| 52 | + | cmd.Stderr = os.Stderr | |||
| 53 | + | require.NoError(t, cmd.Start()) | |||
| 54 | + | defer func() { | |||
| 55 | + | _ = stdin.Close() | |||
| 56 | + | _ = cmd.Wait() | |||
| 57 | + | }() | |||
| 58 | + | ||||
| 59 | + | line, err := bufio.NewReader(stdout).ReadString('\n') | |||
| 60 | + | require.NoError(t, err) | |||
| 61 | + | require.Equal(t, "locked\n", line) | |||
| 62 | + | ||||
| 63 | + | _, err = TryAcquire(runtimeDir) | |||
| 64 | + | require.ErrorIs(t, err, ErrBusy) | |||
| 65 | + | ||||
| 66 | + | require.NoError(t, stdin.Close()) | |||
| 67 | + | require.NoError(t, cmd.Wait()) | |||
| 68 | + | ||||
| 69 | + | owner, err := TryAcquire(runtimeDir) | |||
| 70 | + | require.NoError(t, err) | |||
| 71 | + | require.NoError(t, owner.Release()) | |||
| 72 | + | require.NoError(t, owner.Release(), "release is idempotent") | |||
| 73 | + | } | |||
| 74 | + | ||||
| 75 | + | func testLifecycleLockAcquireCancellation(t *testing.T) { | |||
| 76 | + | runtimeDir := filepath.Join(t.TempDir(), "runtime") | |||
| 77 | + | owner, err := TryAcquire(runtimeDir) | |||
| 78 | + | require.NoError(t, err) | |||
| 79 | + | defer func() { require.NoError(t, owner.Release()) }() | |||
| 80 | + | ||||
| 81 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 82 | + | cancel() | |||
| 83 | + | ||||
| 84 | + | _, err = Acquire(ctx, runtimeDir, time.Millisecond) | |||
| 85 | + | require.ErrorIs(t, err, context.Canceled) | |||
| 86 | + | } | |||
| 87 | + | ||||
| 88 | + | func testLifecycleLockUnavailableRuntimeDirectory(t *testing.T) { | |||
| 89 | + | runtimePath := t.TempDir() + "/not-a-directory" | |||
| 90 | + | require.NoError(t, os.WriteFile(runtimePath, []byte("occupied"), 0o600)) | |||
| 91 | + | ||||
| 92 | + | _, err := TryAcquire(runtimePath) | |||
| 93 | + | require.Error(t, err) | |||
| 94 | + | require.False(t, errors.Is(err, ErrBusy)) | |||
| 95 | + | } | |||
M internal/adapters/noticefile/noticefile.go +2 -7
| @@ -311,13 +311,8 @@ func appendNoticeFragment(line, fragment []byte) ([]byte, bool) { | |||||
| 311 | } | 311 | } | ||
| 312 | required := len(line) + len(fragment) | 312 | required := len(line) + len(fragment) | ||
| 313 | if required > cap(line) { | 313 | if required > cap(line) { | ||
| 314 | - | capacity := cap(line) * 2 | 314 | + | capacity := max(cap(line)*2, noticeReadBufferSize) |
| 315 | - | if capacity < noticeReadBufferSize { | 315 | + | capacity = min(capacity, maxNoticeRecordSize) |
| 316 | - | capacity = noticeReadBufferSize | |||
| 317 | - | } | |||
| 318 | - | if capacity > maxNoticeRecordSize { | |||
| 319 | - | capacity = maxNoticeRecordSize | |||
| 320 | - | } | |||
| 321 | grown := make([]byte, len(line), capacity) | 316 | grown := make([]byte, len(line), capacity) | ||
| 322 | copy(grown, line) | 317 | copy(grown, line) | ||
| 323 | line = grown | 318 | line = grown | ||
M internal/adapters/snapshot/atomic.go +7 -49
| @@ -8,19 +8,21 @@ import ( | |||||
| 8 | "path/filepath" | 8 | "path/filepath" | ||
| 9 | "syscall" | 9 | "syscall" | ||
| 10 | 10 | ||||
| 11 | + | "github.com/bnema/vev/internal/domain" | |||
| 11 | "github.com/bnema/vev/pkg/safedir" | 12 | "github.com/bnema/vev/pkg/safedir" | ||
| 12 | ) | 13 | ) | ||
| 13 | 14 | ||||
| 14 | - | func (r *Repository) ensureSession(key string) error { | 15 | + | func (r *Repository) ensureSession(id domain.IncarnationID) error { |
| 16 | + | sessionPath := r.sessionPath(id) | |||
| 15 | for _, directory := range []struct { | 17 | for _, directory := range []struct { | ||
| 16 | path string | 18 | path string | ||
| 17 | phase string | 19 | phase string | ||
| 18 | }{ | 20 | }{ | ||
| 19 | {r.dir, "repository"}, | 21 | {r.dir, "repository"}, | ||
| 20 | {filepath.Join(r.dir, repositorySessionsDir), "sessions"}, | 22 | {filepath.Join(r.dir, repositorySessionsDir), "sessions"}, | ||
| 21 | - | {r.sessionPath(key), "session"}, | 23 | + | {sessionPath, "session"}, |
| 22 | - | {filepath.Join(r.sessionPath(key), repositoryObjectsDir), "objects"}, | 24 | + | {filepath.Join(sessionPath, repositoryObjectsDir), "objects"}, |
| 23 | - | {filepath.Join(r.sessionPath(key), repositoryGenerations), "generations"}, | 25 | + | {filepath.Join(sessionPath, repositoryGenerations), "generations"}, |
| 24 | } { | 26 | } { | ||
| 25 | if err := r.ensurePrivateDirectoryPhase(directory.path, directory.phase); err != nil { | 27 | if err := r.ensurePrivateDirectoryPhase(directory.path, directory.phase); err != nil { | ||
| 26 | return fmt.Errorf("create snapshot repository directory: %w", err) | 28 | return fmt.Errorf("create snapshot repository directory: %w", err) | ||
| @@ -47,11 +49,6 @@ func (r *Repository) ensurePrivateDirectoryPhase(dir, phase string) (err error) | |||||
| 47 | return err | 49 | return err | ||
| 48 | } | 50 | } | ||
| 49 | if created { | 51 | if created { | ||
| 50 | - | if hook := r.hooks.syncDirectory; hook != nil { | |||
| 51 | - | if err := hook(filepath.Dir(dir)); err != nil { | |||
| 52 | - | return fmt.Errorf("%s parent directory sync: %w", phase, err) | |||
| 53 | - | } | |||
| 54 | - | } | |||
| 55 | if err := syncDirectory(filepath.Dir(dir)); err != nil { | 52 | if err := syncDirectory(filepath.Dir(dir)); err != nil { | ||
| 56 | return fmt.Errorf("%s parent directory sync: %w", phase, err) | 53 | return fmt.Errorf("%s parent directory sync: %w", phase, err) | ||
| 57 | } | 54 | } | ||
| @@ -191,36 +188,17 @@ func (r *Repository) withAtomicTemp(dir string, data []byte, publish func(string | |||||
| 191 | } | 188 | } | ||
| 192 | 189 | ||||
| 193 | func (r *Repository) createTemp(dir string) (*os.File, error) { | 190 | func (r *Repository) createTemp(dir string) (*os.File, error) { | ||
| 194 | - | if r.hooks.createTemp != nil { | |||
| 195 | - | if err := r.hooks.createTemp(dir); err != nil { | |||
| 196 | - | return nil, err | |||
| 197 | - | } | |||
| 198 | - | } | |||
| 199 | return r.createTempAt(dir) | 191 | return r.createTempAt(dir) | ||
| 200 | } | 192 | } | ||
| 201 | func (r *Repository) writeFile(f *os.File, data []byte) error { | 193 | func (r *Repository) writeFile(f *os.File, data []byte) error { | ||
| 202 | - | if r.hooks.writeTemp != nil { | |||
| 203 | - | if err := r.hooks.writeTemp(f.Name()); err != nil { | |||
| 204 | - | return err | |||
| 205 | - | } | |||
| 206 | - | } | |||
| 207 | _, err := f.Write(data) | 194 | _, err := f.Write(data) | ||
| 208 | return err | 195 | return err | ||
| 209 | } | 196 | } | ||
| 210 | func (r *Repository) syncFile(f *os.File) error { | 197 | func (r *Repository) syncFile(f *os.File) error { | ||
| 211 | - | if r.hooks.syncFile != nil { | |||
| 212 | - | if err := r.hooks.syncFile(f.Name()); err != nil { | |||
| 213 | - | return err | |||
| 214 | - | } | |||
| 215 | - | } | |||
| 216 | return f.Sync() | 198 | return f.Sync() | ||
| 217 | } | 199 | } | ||
| 218 | func (r *Repository) closeFile(f *os.File) error { | 200 | func (r *Repository) closeFile(f *os.File) error { | ||
| 219 | - | var injected error | 201 | + | return f.Close() |
| 220 | - | if r.hooks.closeFile != nil { | |||
| 221 | - | injected = r.hooks.closeFile(f.Name()) | |||
| 222 | - | } | |||
| 223 | - | return errors.Join(injected, f.Close()) | |||
| 224 | } | 202 | } | ||
| 225 | 203 | ||||
| 226 | // joinCloseError retains a primary operation failure and appends contextual | 204 | // joinCloseError retains a primary operation failure and appends contextual | ||
| @@ -232,11 +210,6 @@ func joinCloseError(primary *error, operation string, closeErr error) { | |||||
| 232 | } | 210 | } | ||
| 233 | 211 | ||||
| 234 | func (r *Repository) installImmutable(oldPath, newPath string) (err error) { | 212 | func (r *Repository) installImmutable(oldPath, newPath string) (err error) { | ||
| 235 | - | if r.hooks.installImmutable != nil { | |||
| 236 | - | if err := r.hooks.installImmutable(newPath); err != nil { | |||
| 237 | - | return err | |||
| 238 | - | } | |||
| 239 | - | } | |||
| 240 | oldRel, ok := r.repositoryRelative(oldPath) | 213 | oldRel, ok := r.repositoryRelative(oldPath) | ||
| 241 | if !ok { | 214 | if !ok { | ||
| 242 | return fmt.Errorf("snapshot path outside repository") | 215 | return fmt.Errorf("snapshot path outside repository") | ||
| @@ -253,11 +226,6 @@ func (r *Repository) installImmutable(oldPath, newPath string) (err error) { | |||||
| 253 | return root.Link(oldRel, newRel) | 226 | return root.Link(oldRel, newRel) | ||
| 254 | } | 227 | } | ||
| 255 | func (r *Repository) rename(oldPath, newPath string) (err error) { | 228 | func (r *Repository) rename(oldPath, newPath string) (err error) { | ||
| 256 | - | if r.hooks.rename != nil { | |||
| 257 | - | if err := r.hooks.rename(newPath); err != nil { | |||
| 258 | - | return err | |||
| 259 | - | } | |||
| 260 | - | } | |||
| 261 | oldRel, ok := r.repositoryRelative(oldPath) | 229 | oldRel, ok := r.repositoryRelative(oldPath) | ||
| 262 | if !ok { | 230 | if !ok { | ||
| 263 | return fmt.Errorf("snapshot path outside repository") | 231 | return fmt.Errorf("snapshot path outside repository") | ||
| @@ -274,11 +242,6 @@ func (r *Repository) rename(oldPath, newPath string) (err error) { | |||||
| 274 | return root.Rename(oldRel, newRel) | 242 | return root.Rename(oldRel, newRel) | ||
| 275 | } | 243 | } | ||
| 276 | func (r *Repository) remove(path string) (err error) { | 244 | func (r *Repository) remove(path string) (err error) { | ||
| 277 | - | if r.hooks.remove != nil { | |||
| 278 | - | if err := r.hooks.remove(path); err != nil { | |||
| 279 | - | return err | |||
| 280 | - | } | |||
| 281 | - | } | |||
| 282 | rel, ok := r.repositoryRelative(path) | 245 | rel, ok := r.repositoryRelative(path) | ||
| 283 | if !ok { | 246 | if !ok { | ||
| 284 | return fmt.Errorf("snapshot path outside repository") | 247 | return fmt.Errorf("snapshot path outside repository") | ||
| @@ -291,11 +254,6 @@ func (r *Repository) remove(path string) (err error) { | |||||
| 291 | return root.Remove(rel) | 254 | return root.Remove(rel) | ||
| 292 | } | 255 | } | ||
| 293 | func (r *Repository) syncDirectory(dir string) error { | 256 | func (r *Repository) syncDirectory(dir string) error { | ||
| 294 | - | if r.hooks.syncDirectory != nil { | |||
| 295 | - | if err := r.hooks.syncDirectory(dir); err != nil { | |||
| 296 | - | return err | |||
| 297 | - | } | |||
| 298 | - | } | |||
| 299 | return syncDirectory(dir) | 257 | return syncDirectory(dir) | ||
| 300 | } | 258 | } | ||
| 301 | func syncDirectory(dir string) error { | 259 | func syncDirectory(dir string) error { | ||
M internal/adapters/snapshot/atomic_test.go +24 -32
| @@ -8,51 +8,46 @@ import ( | |||||
| 8 | "testing" | 8 | "testing" | ||
| 9 | ) | 9 | ) | ||
| 10 | 10 | ||||
| 11 | - | func TestWriteImmutableUsesOneFinalDirectorySync(t *testing.T) { | 11 | + | func TestWriteImmutablePublishesAndReusesIdenticalData(t *testing.T) { |
| 12 | repo := NewRepository(privateDir(t)) | 12 | repo := NewRepository(privateDir(t)) | ||
| 13 | dir := filepath.Join(repo.dir, "immutable") | 13 | dir := filepath.Join(repo.dir, "immutable") | ||
| 14 | if err := repo.ensurePrivateDirectory(dir); err != nil { | 14 | if err := repo.ensurePrivateDirectory(dir); err != nil { | ||
| 15 | t.Fatal(err) | 15 | t.Fatal(err) | ||
| 16 | } | 16 | } | ||
| 17 | 17 | ||||
| 18 | - | syncs := 0 | 18 | + | path := filepath.Join(dir, "object") |
| 19 | - | repo.hooks.syncDirectory = func(got string) error { | |||
| 20 | - | if got == dir { | |||
| 21 | - | syncs++ | |||
| 22 | - | } | |||
| 23 | - | return nil | |||
| 24 | - | } | |||
| 25 | data := []byte("immutable") | 19 | data := []byte("immutable") | ||
| 26 | - | if err := repo.writeImmutable(filepath.Join(dir, "object"), data, func(existing []byte) error { | 20 | + | verify := func(existing []byte) error { |
| 27 | if !bytes.Equal(existing, data) { | 21 | if !bytes.Equal(existing, data) { | ||
| 28 | return errors.New("unexpected immutable data") | 22 | return errors.New("unexpected immutable data") | ||
| 29 | } | 23 | } | ||
| 30 | return nil | 24 | return nil | ||
| 31 | - | }); err != nil { | 25 | + | } |
| 26 | + | if err := repo.writeImmutable(path, data, verify); err != nil { | |||
| 32 | t.Fatal(err) | 27 | t.Fatal(err) | ||
| 33 | } | 28 | } | ||
| 34 | - | if syncs != 1 { | 29 | + | if err := repo.writeImmutable(path, data, verify); err != nil { |
| 35 | - | t.Fatalf("successful immutable directory syncs = %d, want 1", syncs) | 30 | + | t.Fatal(err) |
| 31 | + | } | |||
| 32 | + | got, err := os.ReadFile(path) | |||
| 33 | + | if err != nil { | |||
| 34 | + | t.Fatal(err) | |||
| 35 | + | } | |||
| 36 | + | if !bytes.Equal(got, data) { | |||
| 37 | + | t.Fatalf("immutable data = %q, want %q", got, data) | |||
| 36 | } | 38 | } | ||
| 37 | } | 39 | } | ||
| 38 | 40 | ||||
| 39 | - | func TestWithAtomicTempPreservesLifecycleOrder(t *testing.T) { | 41 | + | func TestWithAtomicTempRemovesPublishedTemporaryFile(t *testing.T) { |
| 40 | repo := NewRepository(privateDir(t)) | 42 | repo := NewRepository(privateDir(t)) | ||
| 41 | dir := filepath.Join(repo.dir, "temporary") | 43 | dir := filepath.Join(repo.dir, "temporary") | ||
| 42 | if err := repo.ensurePrivateDirectory(dir); err != nil { | 44 | if err := repo.ensurePrivateDirectory(dir); err != nil { | ||
| 43 | t.Fatal(err) | 45 | t.Fatal(err) | ||
| 44 | } | 46 | } | ||
| 45 | 47 | ||||
| 46 | - | var events []string | 48 | + | var temporary string |
| 47 | - | repo.hooks.createTemp = func(string) error { events = append(events, "create"); return nil } | |||
| 48 | - | repo.hooks.writeTemp = func(string) error { events = append(events, "write"); return nil } | |||
| 49 | - | repo.hooks.syncFile = func(string) error { events = append(events, "sync file"); return nil } | |||
| 50 | - | repo.hooks.closeFile = func(string) error { events = append(events, "close"); return nil } | |||
| 51 | - | repo.hooks.remove = func(string) error { events = append(events, "remove"); return nil } | |||
| 52 | - | repo.hooks.syncDirectory = func(string) error { events = append(events, "sync directory"); return nil } | |||
| 53 | - | ||||
| 54 | if err := repo.withAtomicTemp(dir, []byte("data"), func(path string) (bool, error) { | 49 | if err := repo.withAtomicTemp(dir, []byte("data"), func(path string) (bool, error) { | ||
| 55 | - | events = append(events, "publish") | 50 | + | temporary = path |
| 56 | info, err := os.Stat(path) | 51 | info, err := os.Stat(path) | ||
| 57 | if err != nil { | 52 | if err != nil { | ||
| 58 | return true, err | 53 | return true, err | ||
| @@ -64,21 +59,14 @@ func TestWithAtomicTempPreservesLifecycleOrder(t *testing.T) { | |||||
| 64 | }); err != nil { | 59 | }); err != nil { | ||
| 65 | t.Fatal(err) | 60 | t.Fatal(err) | ||
| 66 | } | 61 | } | ||
| 67 | - | want := []string{"create", "write", "sync file", "close", "publish", "remove", "sync directory"} | 62 | + | if _, err := os.Stat(temporary); !errors.Is(err, os.ErrNotExist) { |
| 68 | - | if len(events) != len(want) { | 63 | + | t.Fatalf("temporary file remains after publication: %v", err) |
| 69 | - | t.Fatalf("lifecycle = %v, want %v", events, want) | |||
| 70 | - | } | |||
| 71 | - | for i := range want { | |||
| 72 | - | if events[i] != want[i] { | |||
| 73 | - | t.Fatalf("lifecycle = %v, want %v", events, want) | |||
| 74 | - | } | |||
| 75 | } | 64 | } | ||
| 76 | } | 65 | } | ||
| 77 | 66 | ||||
| 78 | - | func TestWriteMutableDoesNotRemoveConsumedTemporaryFile(t *testing.T) { | 67 | + | func TestWriteMutableConsumesTemporaryFile(t *testing.T) { |
| 79 | repo := NewRepository(privateDir(t)) | 68 | repo := NewRepository(privateDir(t)) | ||
| 80 | path := filepath.Join(repo.dir, "HEAD") | 69 | path := filepath.Join(repo.dir, "HEAD") | ||
| 81 | - | repo.hooks.remove = func(string) error { return errors.New("remove must not run") } | |||
| 82 | 70 | ||||
| 83 | if err := repo.writeMutable(path, []byte("head")); err != nil { | 71 | if err := repo.writeMutable(path, []byte("head")); err != nil { | ||
| 84 | t.Fatalf("writeMutable error = %v", err) | 72 | t.Fatalf("writeMutable error = %v", err) | ||
| @@ -90,4 +78,8 @@ func TestWriteMutableDoesNotRemoveConsumedTemporaryFile(t *testing.T) { | |||||
| 90 | if !bytes.Equal(got, []byte("head")) { | 78 | if !bytes.Equal(got, []byte("head")) { | ||
| 91 | t.Fatalf("HEAD = %q, want %q", got, "head") | 79 | t.Fatalf("HEAD = %q, want %q", got, "head") | ||
| 92 | } | 80 | } | ||
| 81 | + | matches, err := filepath.Glob(filepath.Join(repo.dir, ".tmp-*")) | |||
| 82 | + | if err != nil || len(matches) != 0 { | |||
| 83 | + | t.Fatalf("temporary files after mutable write = %v, glob error = %v", matches, err) | |||
| 84 | + | } | |||
| 93 | } | 85 | } | ||
A internal/adapters/snapshot/direct_load_test.go +204 -0
| @@ -0,0 +1,204 @@ | |||||
| 1 | + | package snapshot | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "crypto/sha256" | |||
| 6 | + | "errors" | |||
| 7 | + | "os" | |||
| 8 | + | "path/filepath" | |||
| 9 | + | "testing" | |||
| 10 | + | ||||
| 11 | + | "github.com/bnema/vev/internal/domain" | |||
| 12 | + | "github.com/bnema/vev/internal/ports" | |||
| 13 | + | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | |||
| 14 | + | codec "github.com/bnema/vev/internal/usecase/snapshot" | |||
| 15 | + | "github.com/stretchr/testify/require" | |||
| 16 | + | ) | |||
| 17 | + | ||||
| 18 | + | func TestIncarnationNamespace(t *testing.T) { | |||
| 19 | + | repo := NewRepository(privateDir(t)) | |||
| 20 | + | id := domain.IncarnationID{1} | |||
| 21 | + | pub := incarnationPublication(t, id, "old", 1, nil) | |||
| 22 | + | if err := repo.Publish(context.Background(), pub); err != nil { | |||
| 23 | + | t.Fatal(err) | |||
| 24 | + | } | |||
| 25 | + | parent := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(pub.Manifest)} | |||
| 26 | + | pub = incarnationPublication(t, id, "new", 2, &parent) | |||
| 27 | + | if err := repo.Publish(context.Background(), pub); err != nil { | |||
| 28 | + | t.Fatal(err) | |||
| 29 | + | } | |||
| 30 | + | if got := filepath.Base(repo.sessionPath(id)); got != id.String() { | |||
| 31 | + | t.Fatalf("namespace = %q", got) | |||
| 32 | + | } | |||
| 33 | + | ||||
| 34 | + | wrong := pub | |||
| 35 | + | wrong.IncarnationID = domain.IncarnationID{2} | |||
| 36 | + | if err := repo.Publish(context.Background(), wrong); err == nil { | |||
| 37 | + | t.Fatal("mismatched publication incarnation accepted") | |||
| 38 | + | } | |||
| 39 | + | } | |||
| 40 | + | ||||
| 41 | + | func TestDirectCheckpointLoad(t *testing.T) { | |||
| 42 | + | repo := NewRepository(privateDir(t)) | |||
| 43 | + | id := domain.IncarnationID{1} | |||
| 44 | + | pub := incarnationPublication(t, id, "work", 1, nil) | |||
| 45 | + | if err := repo.Publish(context.Background(), pub); err != nil { | |||
| 46 | + | t.Fatal(err) | |||
| 47 | + | } | |||
| 48 | + | generations := filepath.Dir(repo.manifestPath(id, 1)) | |||
| 49 | + | for i := 2; i <= 5001; i++ { | |||
| 50 | + | if err := os.WriteFile(filepath.Join(generations, generationFilename(uint64(i))), []byte("stale"), 0o600); err != nil { | |||
| 51 | + | t.Fatal(err) | |||
| 52 | + | } | |||
| 53 | + | } | |||
| 54 | + | ref := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(pub.Manifest)} | |||
| 55 | + | got, err := repo.LoadCheckpoint(context.Background(), id, "work", ref) | |||
| 56 | + | if err != nil { | |||
| 57 | + | t.Fatal(err) | |||
| 58 | + | } | |||
| 59 | + | if got.Generation != 1 || got.IncarnationID != id { | |||
| 60 | + | t.Fatalf("generation = %#v", got) | |||
| 61 | + | } | |||
| 62 | + | } | |||
| 63 | + | ||||
| 64 | + | func TestReconcileCheckpoint(t *testing.T) { | |||
| 65 | + | repo := NewRepository(privateDir(t)) | |||
| 66 | + | id := domain.IncarnationID{1} | |||
| 67 | + | pub := incarnationPublication(t, id, "work", 1, nil) | |||
| 68 | + | if err := repo.Publish(context.Background(), pub); err != nil { | |||
| 69 | + | t.Fatal(err) | |||
| 70 | + | } | |||
| 71 | + | valid := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(pub.Manifest)} | |||
| 72 | + | if err := repo.ReconcileCheckpoint(context.Background(), id, valid); err != nil { | |||
| 73 | + | t.Fatal(err) | |||
| 74 | + | } | |||
| 75 | + | before, err := os.ReadFile(repo.headPath(id)) | |||
| 76 | + | if err != nil { | |||
| 77 | + | t.Fatal(err) | |||
| 78 | + | } | |||
| 79 | + | invalid := valid | |||
| 80 | + | invalid.ManifestDigest[0]++ | |||
| 81 | + | if err := repo.ReconcileCheckpoint(context.Background(), id, invalid); err == nil { | |||
| 82 | + | t.Fatal("invalid checkpoint repaired HEAD") | |||
| 83 | + | } | |||
| 84 | + | after, err := os.ReadFile(repo.headPath(id)) | |||
| 85 | + | if err != nil { | |||
| 86 | + | t.Fatal(err) | |||
| 87 | + | } | |||
| 88 | + | if string(after) != string(before) { | |||
| 89 | + | t.Fatal("invalid repair changed HEAD") | |||
| 90 | + | } | |||
| 91 | + | } | |||
| 92 | + | ||||
| 93 | + | func TestForwardOrphanRetryValidatesCheckpointBeforeCatalogueAdvance(t *testing.T) { | |||
| 94 | + | for _, tt := range []struct { | |||
| 95 | + | name string | |||
| 96 | + | mutate func(*testing.T, *Repository, ports.SnapshotPublication) | |||
| 97 | + | }{ | |||
| 98 | + | { | |||
| 99 | + | name: "missing object", | |||
| 100 | + | mutate: func(t *testing.T, repository *Repository, publication ports.SnapshotPublication) { | |||
| 101 | + | t.Helper() | |||
| 102 | + | require.NoError(t, os.Remove(repository.objectPath(publication.IncarnationID, publication.Objects[0].Digest))) | |||
| 103 | + | }, | |||
| 104 | + | }, | |||
| 105 | + | { | |||
| 106 | + | name: "corrupt object", | |||
| 107 | + | mutate: func(t *testing.T, repository *Repository, publication ports.SnapshotPublication) { | |||
| 108 | + | t.Helper() | |||
| 109 | + | require.NoError(t, os.WriteFile(repository.objectPath(publication.IncarnationID, publication.Objects[0].Digest), []byte("corrupt"), 0o600)) | |||
| 110 | + | }, | |||
| 111 | + | }, | |||
| 112 | + | { | |||
| 113 | + | name: "corrupt manifest digest", | |||
| 114 | + | mutate: func(t *testing.T, repository *Repository, publication ports.SnapshotPublication) { | |||
| 115 | + | t.Helper() | |||
| 116 | + | head, err := os.ReadFile(repository.headPath(publication.IncarnationID)) | |||
| 117 | + | require.NoError(t, err) | |||
| 118 | + | head[len(head)-1]++ | |||
| 119 | + | require.NoError(t, os.WriteFile(repository.headPath(publication.IncarnationID), head, 0o600)) | |||
| 120 | + | }, | |||
| 121 | + | }, | |||
| 122 | + | } { | |||
| 123 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 124 | + | id := domain.IncarnationID{1} | |||
| 125 | + | publication := incarnationPublication(t, id, "work", 1, nil) | |||
| 126 | + | commitError := errors.New("catalogue commit failed") | |||
| 127 | + | catalogue := &forwardOrphanCatalogue{ | |||
| 128 | + | record: domain.CatalogueRecord{Name: publication.Name, IncarnationID: id}, | |||
| 129 | + | replaceError: commitError, | |||
| 130 | + | } | |||
| 131 | + | repository := NewRepository(privateDir(t)) | |||
| 132 | + | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil) | |||
| 133 | + | ||||
| 134 | + | _, err := coordinator.PublishCheckpoint(t.Context(), publication.Name, publication) | |||
| 135 | + | require.ErrorIs(t, err, commitError) | |||
| 136 | + | tt.mutate(t, repository, publication) | |||
| 137 | + | _, err = coordinator.PublishCheckpoint(t.Context(), publication.Name, publication) | |||
| 138 | + | require.Error(t, err) | |||
| 139 | + | require.Nil(t, catalogue.record.Committed, "an invalid forward orphan must not become authoritative") | |||
| 140 | + | require.Equal(t, 1, catalogue.replaceCalls, "retry must not advance the catalogue") | |||
| 141 | + | }) | |||
| 142 | + | } | |||
| 143 | + | } | |||
| 144 | + | ||||
| 145 | + | type forwardOrphanCatalogue struct { | |||
| 146 | + | record domain.CatalogueRecord | |||
| 147 | + | replaceError error | |||
| 148 | + | replaceCalls int | |||
| 149 | + | } | |||
| 150 | + | ||||
| 151 | + | func (c *forwardOrphanCatalogue) Records() ([]domain.CatalogueRecord, error) { | |||
| 152 | + | return []domain.CatalogueRecord{c.record}, nil | |||
| 153 | + | } | |||
| 154 | + | ||||
| 155 | + | func (c *forwardOrphanCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) { | |||
| 156 | + | return c.record, c.record.Name == name, nil | |||
| 157 | + | } | |||
| 158 | + | ||||
| 159 | + | func (c *forwardOrphanCatalogue) Create(domain.CatalogueRecord) error { | |||
| 160 | + | return errors.New("unexpected create") | |||
| 161 | + | } | |||
| 162 | + | func (c *forwardOrphanCatalogue) UpdateMetadata(domain.CatalogueMetadataUpdate) error { | |||
| 163 | + | return errors.New("unexpected metadata update") | |||
| 164 | + | } | |||
| 165 | + | func (c *forwardOrphanCatalogue) Sync() error { return nil } | |||
| 166 | + | func (c *forwardOrphanCatalogue) Rename(string, domain.CatalogueRecord) error { | |||
| 167 | + | return errors.New("unexpected rename") | |||
| 168 | + | } | |||
| 169 | + | func (c *forwardOrphanCatalogue) Delete(string) error { return errors.New("unexpected delete") } | |||
| 170 | + | func (c *forwardOrphanCatalogue) Close() error { return nil } | |||
| 171 | + | ||||
| 172 | + | func (c *forwardOrphanCatalogue) Replace(name string, record domain.CatalogueRecord) error { | |||
| 173 | + | if name != c.record.Name || name != record.Name { | |||
| 174 | + | return errors.New("catalogue key mismatch") | |||
| 175 | + | } | |||
| 176 | + | c.replaceCalls++ | |||
| 177 | + | if c.replaceError != nil { | |||
| 178 | + | err := c.replaceError | |||
| 179 | + | c.replaceError = nil | |||
| 180 | + | return err | |||
| 181 | + | } | |||
| 182 | + | c.record = record | |||
| 183 | + | return nil | |||
| 184 | + | } | |||
| 185 | + | ||||
| 186 | + | func incarnationPublication(t *testing.T, id domain.IncarnationID, name string, generation uint64, parent *domain.CheckpointRef) ports.SnapshotPublication { | |||
| 187 | + | t.Helper() | |||
| 188 | + | tail, err := codec.MarshalObject(codec.HistoryTail, []byte("tail")) | |||
| 189 | + | if err != nil { | |||
| 190 | + | t.Fatal(err) | |||
| 191 | + | } | |||
| 192 | + | visible, err := codec.MarshalObject(codec.Visible, []byte("visible")) | |||
| 193 | + | if err != nil { | |||
| 194 | + | t.Fatal(err) | |||
| 195 | + | } | |||
| 196 | + | manifest, err := codec.MarshalManifest(codec.Manifest{ | |||
| 197 | + | Generation: generation, IncarnationID: id, ParentCheckpoint: parent, Name: name, | |||
| 198 | + | Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}, | |||
| 199 | + | }) | |||
| 200 | + | if err != nil { | |||
| 201 | + | t.Fatal(err) | |||
| 202 | + | } | |||
| 203 | + | return ports.SnapshotPublication{IncarnationID: id, Name: name, Generation: generation, ParentCheckpoint: parent, Manifest: manifest, Objects: []ports.SnapshotObject{tail, visible}} | |||
| 204 | + | } | |||
D internal/adapters/snapshot/dirent_cookie_darwin.go +0 -21
| @@ -1,21 +0,0 @@ | |||||
| 1 | - | //go:build darwin | |||
| 2 | - | ||||
| 3 | - | package snapshot | |||
| 4 | - | ||||
| 5 | - | import ( | |||
| 6 | - | "io" | |||
| 7 | - | "syscall" | |||
| 8 | - | "unsafe" | |||
| 9 | - | ) | |||
| 10 | - | ||||
| 11 | - | // syscall.ReadDirent on Darwin implements Getdirentries by storing the number | |||
| 12 | - | // of returned entries in the descriptor seek offset. Dirent.Seekoff is not | |||
| 13 | - | // that cursor. A max-sized record buffer lets each call be drained before the | |||
| 14 | - | // descriptor-maintained count is saved. | |||
| 15 | - | const maintenanceDirentBufferSize = int(unsafe.Sizeof(syscall.Dirent{})) | |||
| 16 | - | ||||
| 17 | - | func directoryCookie(file maintenanceDirectory, _ *syscall.Dirent) (int64, error) { | |||
| 18 | - | return file.Seek(0, io.SeekCurrent) | |||
| 19 | - | } | |||
| 20 | - | ||||
| 21 | - | func drainMaintenanceDirentBatch() bool { return true } | |||
D internal/adapters/snapshot/dirent_cookie_darwin_test.go +0 -20
| @@ -1,20 +0,0 @@ | |||||
| 1 | - | //go:build darwin | |||
| 2 | - | ||||
| 3 | - | package snapshot | |||
| 4 | - | ||||
| 5 | - | import ( | |||
| 6 | - | "syscall" | |||
| 7 | - | "testing" | |||
| 8 | - | ) | |||
| 9 | - | ||||
| 10 | - | func TestDirectoryCookieUsesDescriptorEntryCount(t *testing.T) { | |||
| 11 | - | file := fakeMaintenanceDirectory{seekOffset: 42} | |||
| 12 | - | dirent := syscall.Dirent{Seekoff: 99} | |||
| 13 | - | got, err := directoryCookie(file, &dirent) | |||
| 14 | - | if err != nil { | |||
| 15 | - | t.Fatal(err) | |||
| 16 | - | } | |||
| 17 | - | if got != 42 { | |||
| 18 | - | t.Fatalf("directoryCookie() = %d, want descriptor entry count 42", got) | |||
| 19 | - | } | |||
| 20 | - | } | |||
D internal/adapters/snapshot/dirent_cookie_linux.go +0 -14
| @@ -1,14 +0,0 @@ | |||||
| 1 | - | //go:build linux | |||
| 2 | - | ||||
| 3 | - | package snapshot | |||
| 4 | - | ||||
| 5 | - | import "syscall" | |||
| 6 | - | ||||
| 7 | - | const maintenanceDirentBufferSize = 8192 | |||
| 8 | - | ||||
| 9 | - | // directoryCookie is d_off, the resumable getdents(2) cookie for Linux. | |||
| 10 | - | func directoryCookie(_ maintenanceDirectory, dirent *syscall.Dirent) (int64, error) { | |||
| 11 | - | return dirent.Off, nil | |||
| 12 | - | } | |||
| 13 | - | ||||
| 14 | - | func drainMaintenanceDirentBatch() bool { return false } | |||
D internal/adapters/snapshot/dirent_cookie_linux_test.go +0 -19
| @@ -1,19 +0,0 @@ | |||||
| 1 | - | //go:build linux | |||
| 2 | - | ||||
| 3 | - | package snapshot | |||
| 4 | - | ||||
| 5 | - | import ( | |||
| 6 | - | "syscall" | |||
| 7 | - | "testing" | |||
| 8 | - | ) | |||
| 9 | - | ||||
| 10 | - | func TestDirectoryCookieUsesLinuxOffset(t *testing.T) { | |||
| 11 | - | dirent := syscall.Dirent{Off: 42} | |||
| 12 | - | got, err := directoryCookie(fakeMaintenanceDirectory{}, &dirent) | |||
| 13 | - | if err != nil { | |||
| 14 | - | t.Fatal(err) | |||
| 15 | - | } | |||
| 16 | - | if got != 42 { | |||
| 17 | - | t.Fatalf("directoryCookie() = %d, want 42", got) | |||
| 18 | - | } | |||
| 19 | - | } | |||
D internal/adapters/snapshot/fd_count_darwin_test.go +0 -28
| @@ -1,28 +0,0 @@ | |||||
| 1 | - | //go:build darwin | |||
| 2 | - | ||||
| 3 | - | package snapshot | |||
| 4 | - | ||||
| 5 | - | import ( | |||
| 6 | - | "os" | |||
| 7 | - | "testing" | |||
| 8 | - | ) | |||
| 9 | - | ||||
| 10 | - | // macOS exposes the calling process descriptor table through /dev/fd. | |||
| 11 | - | // Readdirnames avoids os.ReadDir's fstatat calls for descriptors that can | |||
| 12 | - | // disappear while the directory is being enumerated. | |||
| 13 | - | func openDescriptorCount(t *testing.T) int { | |||
| 14 | - | t.Helper() | |||
| 15 | - | dir, err := os.Open("/dev/fd") | |||
| 16 | - | if err != nil { | |||
| 17 | - | t.Fatal(err) | |||
| 18 | - | } | |||
| 19 | - | names, readErr := dir.Readdirnames(-1) | |||
| 20 | - | closeErr := dir.Close() | |||
| 21 | - | if readErr != nil { | |||
| 22 | - | t.Fatal(readErr) | |||
| 23 | - | } | |||
| 24 | - | if closeErr != nil { | |||
| 25 | - | t.Fatal(closeErr) | |||
| 26 | - | } | |||
| 27 | - | return len(names) | |||
| 28 | - | } | |||
D internal/adapters/snapshot/fd_count_linux_test.go +0 -19
| @@ -1,19 +0,0 @@ | |||||
| 1 | - | //go:build linux | |||
| 2 | - | ||||
| 3 | - | package snapshot | |||
| 4 | - | ||||
| 5 | - | import ( | |||
| 6 | - | "os" | |||
| 7 | - | "testing" | |||
| 8 | - | ) | |||
| 9 | - | ||||
| 10 | - | // openDescriptorCount checks the actual Linux descriptor table, not a mocked | |||
| 11 | - | // lifecycle signal, so the maintenance leak assertion remains strict. | |||
| 12 | - | func openDescriptorCount(t *testing.T) int { | |||
| 13 | - | t.Helper() | |||
| 14 | - | entries, err := os.ReadDir("/proc/self/fd") | |||
| 15 | - | if err != nil { | |||
| 16 | - | t.Fatal(err) | |||
| 17 | - | } | |||
| 18 | - | return len(entries) | |||
| 19 | - | } | |||
D internal/adapters/snapshot/legacy.go +0 -399
| @@ -1,399 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "bytes" | |||
| 5 | - | "context" | |||
| 6 | - | "crypto/sha256" | |||
| 7 | - | "errors" | |||
| 8 | - | "fmt" | |||
| 9 | - | "io" | |||
| 10 | - | "os" | |||
| 11 | - | "path/filepath" | |||
| 12 | - | "sort" | |||
| 13 | - | "strings" | |||
| 14 | - | ||||
| 15 | - | "github.com/bnema/vev/internal/ports" | |||
| 16 | - | ) | |||
| 17 | - | ||||
| 18 | - | const ( | |||
| 19 | - | legacyDeleteMarkerPrefix = ".legacy-import-receipt-" | |||
| 20 | - | maxLegacyDeleteMarkers = maxLegacySnapshotFiles | |||
| 21 | - | maxLegacyDeleteNameBytes = 200 | |||
| 22 | - | ) | |||
| 23 | - | ||||
| 24 | - | type legacyDeleteMarker struct { | |||
| 25 | - | name string | |||
| 26 | - | digest [sha256.Size]byte | |||
| 27 | - | } | |||
| 28 | - | ||||
| 29 | - | type legacyDirectory interface { | |||
| 30 | - | ReadDir(int) ([]os.DirEntry, error) | |||
| 31 | - | Close() error | |||
| 32 | - | } | |||
| 33 | - | ||||
| 34 | - | func (r *Repository) openLegacyDirectory(path string) (legacyDirectory, error) { | |||
| 35 | - | if r.hooks.openLegacyDirectory != nil { | |||
| 36 | - | return r.hooks.openLegacyDirectory(path) | |||
| 37 | - | } | |||
| 38 | - | return r.openDirectory(path) | |||
| 39 | - | } | |||
| 40 | - | ||||
| 41 | - | // readLegacyImportCandidate reports whether an untrusted root entry can be | |||
| 42 | - | // safely imported. Legacy scanning skips entries that disappear or fail secure | |||
| 43 | - | // descriptor validation so one hostile .snap file does not block other imports. | |||
| 44 | - | func (r *Repository) readLegacyImportCandidate(path string) ([]byte, bool) { | |||
| 45 | - | data, err := r.readBounded(path) | |||
| 46 | - | return data, err == nil | |||
| 47 | - | } | |||
| 48 | - | ||||
| 49 | - | // LoadLegacy reads only pre-incremental root .snap files. It is deliberately | |||
| 50 | - | // isolated from repository reads so incremental directories are never traversed. | |||
| 51 | - | func (r *Repository) LoadLegacy(ctx context.Context) (out []ports.LegacySnapshot, err error) { | |||
| 52 | - | if err := ctx.Err(); err != nil { | |||
| 53 | - | return nil, err | |||
| 54 | - | } | |||
| 55 | - | // One budget covers marker recovery and import. Every directory entry costs | |||
| 56 | - | // work, including unrelated attacker-created names, so neither phase can be | |||
| 57 | - | // made unbounded by a huge legacy root. | |||
| 58 | - | budget := maxDirectoryTraversalEntries | |||
| 59 | - | // A marker is an already-authorized deletion, not an import candidate. Finish | |||
| 60 | - | // it before exposing any legacy blobs so process restart cannot republish one. | |||
| 61 | - | if err := r.cleanupPendingLegacyDeletes(ctx, &budget); err != nil { | |||
| 62 | - | return nil, err | |||
| 63 | - | } | |||
| 64 | - | f, err := r.openLegacyDirectory(r.dir) | |||
| 65 | - | if errors.Is(err, os.ErrNotExist) { | |||
| 66 | - | return nil, nil | |||
| 67 | - | } | |||
| 68 | - | if err != nil { | |||
| 69 | - | return nil, fmt.Errorf("read legacy snapshot directory: %w", safeFilesystemError(err)) | |||
| 70 | - | } | |||
| 71 | - | defer func() { | |||
| 72 | - | if closeErr := f.Close(); closeErr != nil { | |||
| 73 | - | err = errors.Join(err, fmt.Errorf("close legacy snapshot directory: %w", safeFilesystemError(closeErr))) | |||
| 74 | - | } | |||
| 75 | - | }() | |||
| 76 | - | ||||
| 77 | - | out = make([]ports.LegacySnapshot, 0, maxLegacySnapshotFiles) | |||
| 78 | - | files := 0 | |||
| 79 | - | total := 0 | |||
| 80 | - | for { | |||
| 81 | - | entries, readErr := f.ReadDir(maintenanceBatch) | |||
| 82 | - | for _, entry := range entries { | |||
| 83 | - | if err := ctx.Err(); err != nil { | |||
| 84 | - | return nil, err | |||
| 85 | - | } | |||
| 86 | - | if budget == 0 { | |||
| 87 | - | return nil, ErrDirectoryTraversalBudget | |||
| 88 | - | } | |||
| 89 | - | budget-- | |||
| 90 | - | if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".snap") { | |||
| 91 | - | continue | |||
| 92 | - | } | |||
| 93 | - | files++ | |||
| 94 | - | if files > maxLegacySnapshotFiles { | |||
| 95 | - | return nil, fmt.Errorf("legacy snapshot import has too many files (maximum %d)", maxLegacySnapshotFiles) | |||
| 96 | - | } | |||
| 97 | - | path := filepath.Join(r.dir, entry.Name()) | |||
| 98 | - | data, importable := r.readLegacyImportCandidate(path) | |||
| 99 | - | if !importable { | |||
| 100 | - | continue | |||
| 101 | - | } | |||
| 102 | - | if len(data) > maxLegacySnapshotBytes-total { | |||
| 103 | - | return nil, fmt.Errorf("legacy snapshot import exceeds aggregate byte limit (%d bytes)", maxLegacySnapshotBytes) | |||
| 104 | - | } | |||
| 105 | - | total += len(data) | |||
| 106 | - | name := strings.TrimSuffix(entry.Name(), ".snap") | |||
| 107 | - | if strings.HasPrefix(name, "@") || !safeNameRE.MatchString(name) { | |||
| 108 | - | name = "" | |||
| 109 | - | } | |||
| 110 | - | killed, tombstoneErr := r.tombstoned(name) | |||
| 111 | - | if tombstoneErr != nil { | |||
| 112 | - | return nil, fmt.Errorf("read killed session marker: %w", safeFilesystemError(tombstoneErr)) | |||
| 113 | - | } | |||
| 114 | - | if killed { | |||
| 115 | - | continue | |||
| 116 | - | } | |||
| 117 | - | out = append(out, ports.LegacySnapshot{Name: name, Data: data}) | |||
| 118 | - | } | |||
| 119 | - | if errors.Is(readErr, io.EOF) { | |||
| 120 | - | break | |||
| 121 | - | } | |||
| 122 | - | if readErr != nil { | |||
| 123 | - | return nil, fmt.Errorf("read legacy snapshot directory: %w", safeFilesystemError(readErr)) | |||
| 124 | - | } | |||
| 125 | - | } | |||
| 126 | - | sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) | |||
| 127 | - | return out, nil | |||
| 128 | - | } | |||
| 129 | - | ||||
| 130 | - | // DeleteVerifiedLegacy records a hash-bound import receipt before deleting the | |||
| 131 | - | // source. LoadLegacy consumes surviving receipts after restart, but only when | |||
| 132 | - | // the source still has the exact bytes that were verified and published. | |||
| 133 | - | func (r *Repository) DeleteVerifiedLegacy(ctx context.Context, blob ports.LegacySnapshot) error { | |||
| 134 | - | if err := ctx.Err(); err != nil { | |||
| 135 | - | return err | |||
| 136 | - | } | |||
| 137 | - | filename := filenameForName(blob.Name) | |||
| 138 | - | lock := r.lockSession("legacy-sync:" + filename) | |||
| 139 | - | defer r.unlockSession(lock) | |||
| 140 | - | if err := ctx.Err(); err != nil { | |||
| 141 | - | return err | |||
| 142 | - | } | |||
| 143 | - | receipt := legacyDeleteMarker{name: blob.Name, digest: sha256.Sum256(blob.Data)} | |||
| 144 | - | if err := r.authorizeLegacyDelete(receipt); err != nil { | |||
| 145 | - | return err | |||
| 146 | - | } | |||
| 147 | - | return r.deleteAuthorizedLegacy(ctx, filename, receipt) | |||
| 148 | - | } | |||
| 149 | - | ||||
| 150 | - | // DeleteLegacy is for deliberate session purges, which do not represent a | |||
| 151 | - | // verified import and therefore must not create an import receipt. | |||
| 152 | - | func (r *Repository) DeleteLegacy(ctx context.Context, name string) error { | |||
| 153 | - | if err := ctx.Err(); err != nil { | |||
| 154 | - | return err | |||
| 155 | - | } | |||
| 156 | - | filename := filenameForName(name) | |||
| 157 | - | lock := r.lockSession("legacy-sync:" + filename) | |||
| 158 | - | defer r.unlockSession(lock) | |||
| 159 | - | return r.deleteLegacyFile(ctx, filename) | |||
| 160 | - | } | |||
| 161 | - | ||||
| 162 | - | func legacyDeleteMarkerFilename(key string) string { return legacyDeleteMarkerPrefix + key } | |||
| 163 | - | ||||
| 164 | - | func (r *Repository) authorizeLegacyDelete(receipt legacyDeleteMarker) error { | |||
| 165 | - | if err := r.ensurePrivateDirectory(r.dir); err != nil { | |||
| 166 | - | return fmt.Errorf("create legacy import receipt directory: %w", safeFilesystemError(err)) | |||
| 167 | - | } | |||
| 168 | - | path := filepath.Join(r.dir, legacyDeleteMarkerFilename(sessionKey(receipt.name))) | |||
| 169 | - | data := append(receipt.digest[:], receipt.name...) | |||
| 170 | - | if err := r.writeImmutable(path, data, func(data []byte) error { | |||
| 171 | - | if len(data) != sha256.Size+len(receipt.name) || string(data[sha256.Size:]) != receipt.name || !bytes.Equal(data[:sha256.Size], receipt.digest[:]) { | |||
| 172 | - | return fmt.Errorf("legacy import receipt identity mismatch") | |||
| 173 | - | } | |||
| 174 | - | return nil | |||
| 175 | - | }); err != nil { | |||
| 176 | - | return fmt.Errorf("write legacy import receipt: %w", safeFilesystemError(err)) | |||
| 177 | - | } | |||
| 178 | - | return nil | |||
| 179 | - | } | |||
| 180 | - | ||||
| 181 | - | // deleteAuthorizedLegacy deletes only a source whose bytes match the durable | |||
| 182 | - | // receipt. A replaced source invalidates the receipt and remains importable. | |||
| 183 | - | func (r *Repository) deleteAuthorizedLegacy(ctx context.Context, filename string, receipt legacyDeleteMarker) error { | |||
| 184 | - | if _, pending := r.pendingLegacySync.Load(filename); pending { | |||
| 185 | - | if err := r.syncPendingLegacy(ctx, filename); err != nil { | |||
| 186 | - | return err | |||
| 187 | - | } | |||
| 188 | - | return r.clearLegacyDeleteAuthorization(receipt.name) | |||
| 189 | - | } | |||
| 190 | - | path := filepath.Join(r.dir, filename) | |||
| 191 | - | data, err := r.readBounded(path) | |||
| 192 | - | if err != nil && !errors.Is(err, os.ErrNotExist) { | |||
| 193 | - | return fmt.Errorf("read legacy snapshot for deletion: %w", safeFilesystemError(err)) | |||
| 194 | - | } | |||
| 195 | - | if err == nil && sha256.Sum256(data) != receipt.digest { | |||
| 196 | - | return r.clearLegacyDeleteAuthorization(receipt.name) | |||
| 197 | - | } | |||
| 198 | - | return r.deleteLegacyFile(ctx, filename, receipt.name) | |||
| 199 | - | } | |||
| 200 | - | ||||
| 201 | - | func (r *Repository) deleteLegacyFile(ctx context.Context, filename string, clearReceipt ...string) error { | |||
| 202 | - | if _, pending := r.pendingLegacySync.Load(filename); pending { | |||
| 203 | - | if err := r.syncPendingLegacy(ctx, filename); err != nil { | |||
| 204 | - | return err | |||
| 205 | - | } | |||
| 206 | - | } else { | |||
| 207 | - | path := filepath.Join(r.dir, filename) | |||
| 208 | - | if err := r.remove(path); err != nil { | |||
| 209 | - | if !errors.Is(err, os.ErrNotExist) { | |||
| 210 | - | return fmt.Errorf("delete legacy snapshot: %w", safeFilesystemError(err)) | |||
| 211 | - | } | |||
| 212 | - | // A missing repository has no deletion to make durable. This preserves | |||
| 213 | - | // idempotence for explicit purges before the snapshot root exists. | |||
| 214 | - | if _, statErr := os.Lstat(r.dir); errors.Is(statErr, os.ErrNotExist) { | |||
| 215 | - | return nil | |||
| 216 | - | } else if statErr != nil { | |||
| 217 | - | return fmt.Errorf("stat legacy snapshot directory: %w", safeFilesystemError(statErr)) | |||
| 218 | - | } | |||
| 219 | - | } | |||
| 220 | - | r.pendingLegacySync.Store(filename, struct{}{}) | |||
| 221 | - | if err := r.syncPendingLegacy(ctx, filename); err != nil { | |||
| 222 | - | return err | |||
| 223 | - | } | |||
| 224 | - | } | |||
| 225 | - | if len(clearReceipt) != 0 { | |||
| 226 | - | return r.clearLegacyDeleteAuthorization(clearReceipt[0]) | |||
| 227 | - | } | |||
| 228 | - | return nil | |||
| 229 | - | } | |||
| 230 | - | ||||
| 231 | - | func (r *Repository) clearLegacyDeleteAuthorization(name string) error { | |||
| 232 | - | path := filepath.Join(r.dir, legacyDeleteMarkerFilename(sessionKey(name))) | |||
| 233 | - | if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { | |||
| 234 | - | return fmt.Errorf("clear legacy import receipt: %w", safeFilesystemError(err)) | |||
| 235 | - | } | |||
| 236 | - | if err := r.syncDirectory(r.dir); err != nil { | |||
| 237 | - | return fmt.Errorf("sync legacy import receipt directory: %w", safeFilesystemError(err)) | |||
| 238 | - | } | |||
| 239 | - | return nil | |||
| 240 | - | } | |||
| 241 | - | ||||
| 242 | - | func (r *Repository) cleanupPendingLegacyDeletes(ctx context.Context, budget *int) error { | |||
| 243 | - | markers, err := r.pendingLegacyDeleteMarkers(ctx, budget) | |||
| 244 | - | if errors.Is(err, os.ErrNotExist) { | |||
| 245 | - | return nil | |||
| 246 | - | } | |||
| 247 | - | if err != nil { | |||
| 248 | - | return err | |||
| 249 | - | } | |||
| 250 | - | for _, marker := range markers { | |||
| 251 | - | filename := filenameForName(marker.name) | |||
| 252 | - | lock := r.lockSession("legacy-sync:" + filename) | |||
| 253 | - | err := r.deleteAuthorizedLegacy(ctx, filename, marker) | |||
| 254 | - | r.unlockSession(lock) | |||
| 255 | - | if err != nil { | |||
| 256 | - | return fmt.Errorf("retry authorized legacy deletion: %w", err) | |||
| 257 | - | } | |||
| 258 | - | } | |||
| 259 | - | return nil | |||
| 260 | - | } | |||
| 261 | - | ||||
| 262 | - | func (r *Repository) pendingLegacyDeleteMarkers(ctx context.Context, budget *int) (markers []legacyDeleteMarker, err error) { | |||
| 263 | - | f, err := r.openLegacyDirectory(r.dir) | |||
| 264 | - | if err != nil { | |||
| 265 | - | return nil, err | |||
| 266 | - | } | |||
| 267 | - | defer func() { | |||
| 268 | - | if closeErr := f.Close(); closeErr != nil { | |||
| 269 | - | err = errors.Join(err, fmt.Errorf("close legacy deletion marker directory: %w", safeFilesystemError(closeErr))) | |||
| 270 | - | } | |||
| 271 | - | }() | |||
| 272 | - | markers = make([]legacyDeleteMarker, 0, maxLegacyDeleteMarkers) | |||
| 273 | - | for { | |||
| 274 | - | entries, readErr := f.ReadDir(maintenanceBatch) | |||
| 275 | - | for _, entry := range entries { | |||
| 276 | - | if err := ctx.Err(); err != nil { | |||
| 277 | - | return nil, err | |||
| 278 | - | } | |||
| 279 | - | if *budget == 0 { | |||
| 280 | - | return nil, ErrDirectoryTraversalBudget | |||
| 281 | - | } | |||
| 282 | - | *budget-- | |||
| 283 | - | if entry.IsDir() || !strings.HasPrefix(entry.Name(), legacyDeleteMarkerPrefix) { | |||
| 284 | - | continue | |||
| 285 | - | } | |||
| 286 | - | if len(markers) == maxLegacyDeleteMarkers { | |||
| 287 | - | return nil, fmt.Errorf("legacy deletion retry has too many markers (maximum %d)", maxLegacyDeleteMarkers) | |||
| 288 | - | } | |||
| 289 | - | key := strings.TrimPrefix(entry.Name(), legacyDeleteMarkerPrefix) | |||
| 290 | - | if !canonicalSessionKey(key) { | |||
| 291 | - | return nil, fmt.Errorf("invalid legacy deletion marker") | |||
| 292 | - | } | |||
| 293 | - | path := filepath.Join(r.dir, entry.Name()) | |||
| 294 | - | data, dataErr := r.readBounded(path) | |||
| 295 | - | if dataErr != nil || len(data) < sha256.Size || len(data)-sha256.Size > maxLegacyDeleteNameBytes { | |||
| 296 | - | return nil, fmt.Errorf("invalid legacy import receipt") | |||
| 297 | - | } | |||
| 298 | - | name := string(data[sha256.Size:]) | |||
| 299 | - | if sessionKey(name) != key { | |||
| 300 | - | return nil, fmt.Errorf("invalid legacy import receipt") | |||
| 301 | - | } | |||
| 302 | - | marker := legacyDeleteMarker{name: name} | |||
| 303 | - | copy(marker.digest[:], data[:sha256.Size]) | |||
| 304 | - | markers = append(markers, marker) | |||
| 305 | - | } | |||
| 306 | - | if errors.Is(readErr, io.EOF) { | |||
| 307 | - | break | |||
| 308 | - | } | |||
| 309 | - | if readErr != nil { | |||
| 310 | - | return nil, fmt.Errorf("read legacy deletion markers: %w", safeFilesystemError(readErr)) | |||
| 311 | - | } | |||
| 312 | - | } | |||
| 313 | - | return markers, nil | |||
| 314 | - | } | |||
| 315 | - | ||||
| 316 | - | // Tombstone durably marks name as logically killed before either snapshot | |||
| 317 | - | // source is deleted. Startup consults this marker before importing legacy or | |||
| 318 | - | // restoring incremental state, so a partial offline kill cannot resurrect it. | |||
| 319 | - | func (r *Repository) Tombstone(ctx context.Context, name string) error { | |||
| 320 | - | if err := ctx.Err(); err != nil { | |||
| 321 | - | return err | |||
| 322 | - | } | |||
| 323 | - | key := sessionKey(name) | |||
| 324 | - | lock := r.lockSession(key) | |||
| 325 | - | defer r.unlockSession(lock) | |||
| 326 | - | if err := ctx.Err(); err != nil { | |||
| 327 | - | return err | |||
| 328 | - | } | |||
| 329 | - | if err := r.ensurePrivateDirectory(r.dir); err != nil { | |||
| 330 | - | return fmt.Errorf("create killed session marker directory: %w", safeFilesystemError(err)) | |||
| 331 | - | } | |||
| 332 | - | path := filepath.Join(r.dir, tombstoneFilename(key)) | |||
| 333 | - | if err := r.writeImmutable(path, []byte(name), func(data []byte) error { | |||
| 334 | - | if string(data) != name { | |||
| 335 | - | return fmt.Errorf("killed session marker identity mismatch") | |||
| 336 | - | } | |||
| 337 | - | return nil | |||
| 338 | - | }); err != nil { | |||
| 339 | - | return fmt.Errorf("write killed session marker: %w", safeFilesystemError(err)) | |||
| 340 | - | } | |||
| 341 | - | return nil | |||
| 342 | - | } | |||
| 343 | - | ||||
| 344 | - | // DeleteTombstone removes a marker only after both sources have been durably | |||
| 345 | - | // deleted. A missing marker is idempotent for offline-kill retries. | |||
| 346 | - | func (r *Repository) DeleteTombstone(ctx context.Context, name string) error { | |||
| 347 | - | if err := ctx.Err(); err != nil { | |||
| 348 | - | return err | |||
| 349 | - | } | |||
| 350 | - | key := sessionKey(name) | |||
| 351 | - | lock := r.lockSession(key) | |||
| 352 | - | defer r.unlockSession(lock) | |||
| 353 | - | if err := ctx.Err(); err != nil { | |||
| 354 | - | return err | |||
| 355 | - | } | |||
| 356 | - | if err := r.remove(filepath.Join(r.dir, tombstoneFilename(key))); errors.Is(err, os.ErrNotExist) { | |||
| 357 | - | return nil | |||
| 358 | - | } else if err != nil { | |||
| 359 | - | return fmt.Errorf("delete killed session marker: %w", safeFilesystemError(err)) | |||
| 360 | - | } | |||
| 361 | - | if err := r.syncDirectory(r.dir); err != nil { | |||
| 362 | - | return fmt.Errorf("sync killed session marker directory: %w", safeFilesystemError(err)) | |||
| 363 | - | } | |||
| 364 | - | return nil | |||
| 365 | - | } | |||
| 366 | - | ||||
| 367 | - | func tombstoneFilename(key string) string { return ".killed-" + key } | |||
| 368 | - | ||||
| 369 | - | func (r *Repository) tombstoned(name string) (bool, error) { | |||
| 370 | - | if name == "" { | |||
| 371 | - | return false, nil | |||
| 372 | - | } | |||
| 373 | - | path := filepath.Join(r.dir, tombstoneFilename(sessionKey(name))) | |||
| 374 | - | if hook := r.hooks.beforeTombstoneCheck; hook != nil { | |||
| 375 | - | hook(path) | |||
| 376 | - | } | |||
| 377 | - | data, err := r.readBounded(path) | |||
| 378 | - | if errors.Is(err, os.ErrNotExist) { | |||
| 379 | - | return false, nil | |||
| 380 | - | } | |||
| 381 | - | if err != nil { | |||
| 382 | - | return false, err | |||
| 383 | - | } | |||
| 384 | - | if string(data) != name { | |||
| 385 | - | return false, fmt.Errorf("killed session marker identity mismatch") | |||
| 386 | - | } | |||
| 387 | - | return true, nil | |||
| 388 | - | } | |||
| 389 | - | ||||
| 390 | - | func (r *Repository) syncPendingLegacy(ctx context.Context, filename string) error { | |||
| 391 | - | if err := ctx.Err(); err != nil { | |||
| 392 | - | return err | |||
| 393 | - | } | |||
| 394 | - | if err := r.syncDirectory(r.dir); err != nil { | |||
| 395 | - | return fmt.Errorf("sync legacy snapshot directory: %w", safeFilesystemError(err)) | |||
| 396 | - | } | |||
| 397 | - | r.pendingLegacySync.Delete(filename) | |||
| 398 | - | return nil | |||
| 399 | - | } | |||
D internal/adapters/snapshot/legacy_pending_delete_test.go +0 -63
| @@ -1,63 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "errors" | |||
| 6 | - | "os" | |||
| 7 | - | "path/filepath" | |||
| 8 | - | "testing" | |||
| 9 | - | ||||
| 10 | - | "github.com/stretchr/testify/require" | |||
| 11 | - | ||||
| 12 | - | "github.com/bnema/vev/internal/ports" | |||
| 13 | - | ) | |||
| 14 | - | ||||
| 15 | - | func TestRepositoryLoadLegacyCompletesAuthorizedDeleteAfterRestart(t *testing.T) { | |||
| 16 | - | repo := NewRepository(privateDir(t)) | |||
| 17 | - | require.NoError(t, os.Mkdir(repo.dir, 0o700)) | |||
| 18 | - | path := filepath.Join(repo.dir, filenameForName("named")) | |||
| 19 | - | require.NoError(t, os.WriteFile(path, []byte("legacy"), 0o600)) | |||
| 20 | - | ||||
| 21 | - | injected := errors.New("unlink failed") | |||
| 22 | - | repo.hooks.remove = func(path string) error { | |||
| 23 | - | if filepath.Base(path) == filenameForName("named") { | |||
| 24 | - | return injected | |||
| 25 | - | } | |||
| 26 | - | return os.Remove(path) | |||
| 27 | - | } | |||
| 28 | - | // The injected unlink fault is after receipt persistence, simulating a | |||
| 29 | - | // crash boundary where publication was verified but source cleanup was not. | |||
| 30 | - | require.ErrorIs(t, repo.DeleteVerifiedLegacy(context.Background(), ports.LegacySnapshot{Name: "named", Data: []byte("legacy")}), injected) | |||
| 31 | - | ||||
| 32 | - | // A new repository has no in-memory retry state. The durable receipt must | |||
| 33 | - | // fence import and complete deletion without another publication. | |||
| 34 | - | repo = NewRepository(repo.dir) | |||
| 35 | - | legacy, err := repo.LoadLegacy(context.Background()) | |||
| 36 | - | require.NoError(t, err) | |||
| 37 | - | require.Empty(t, legacy) | |||
| 38 | - | _, err = os.Stat(path) | |||
| 39 | - | require.ErrorIs(t, err, os.ErrNotExist) | |||
| 40 | - | } | |||
| 41 | - | ||||
| 42 | - | func TestRepositoryReceiptDoesNotDeleteReplacedLegacySource(t *testing.T) { | |||
| 43 | - | repo := NewRepository(privateDir(t)) | |||
| 44 | - | require.NoError(t, os.Mkdir(repo.dir, 0o700)) | |||
| 45 | - | path := filepath.Join(repo.dir, filenameForName("named")) | |||
| 46 | - | require.NoError(t, os.WriteFile(path, []byte("verified"), 0o600)) | |||
| 47 | - | ||||
| 48 | - | injected := errors.New("unlink failed") | |||
| 49 | - | repo.hooks.remove = func(path string) error { | |||
| 50 | - | if filepath.Base(path) == filenameForName("named") { | |||
| 51 | - | return injected | |||
| 52 | - | } | |||
| 53 | - | return os.Remove(path) | |||
| 54 | - | } | |||
| 55 | - | require.ErrorIs(t, repo.DeleteVerifiedLegacy(context.Background(), ports.LegacySnapshot{Name: "named", Data: []byte("verified")}), injected) | |||
| 56 | - | require.NoError(t, os.WriteFile(path, []byte("replacement"), 0o600)) | |||
| 57 | - | ||||
| 58 | - | repo = NewRepository(repo.dir) | |||
| 59 | - | legacy, err := repo.LoadLegacy(context.Background()) | |||
| 60 | - | require.NoError(t, err) | |||
| 61 | - | require.Equal(t, []ports.LegacySnapshot{{Name: "named", Data: []byte("replacement")}}, legacy) | |||
| 62 | - | require.FileExists(t, path) | |||
| 63 | - | } | |||
D internal/adapters/snapshot/legacy_test.go +0 -141
| @@ -1,141 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "errors" | |||
| 6 | - | "fmt" | |||
| 7 | - | "io" | |||
| 8 | - | "os" | |||
| 9 | - | "path/filepath" | |||
| 10 | - | "strings" | |||
| 11 | - | "testing" | |||
| 12 | - | ) | |||
| 13 | - | ||||
| 14 | - | func TestRepositoryLegacyReadsOnlyBoundedRootSnapshots(t *testing.T) { | |||
| 15 | - | repo := NewRepository(privateDir(t)) | |||
| 16 | - | if err := os.MkdirAll(filepath.Join(repo.dir, repositorySessionsDir), 0o700); err != nil { | |||
| 17 | - | t.Fatal(err) | |||
| 18 | - | } | |||
| 19 | - | if err := os.WriteFile(filepath.Join(repo.dir, "named.snap"), []byte("legacy"), 0o600); err != nil { | |||
| 20 | - | t.Fatal(err) | |||
| 21 | - | } | |||
| 22 | - | if err := os.WriteFile(filepath.Join(repo.dir, filenameForName("unsafe/name")), []byte("hashed"), 0o600); err != nil { | |||
| 23 | - | t.Fatal(err) | |||
| 24 | - | } | |||
| 25 | - | if err := os.WriteFile(filepath.Join(repo.dir, repositorySessionsDir, "ignored.snap"), []byte("incremental"), 0o600); err != nil { | |||
| 26 | - | t.Fatal(err) | |||
| 27 | - | } | |||
| 28 | - | ||||
| 29 | - | got, err := repo.LoadLegacy(context.Background()) | |||
| 30 | - | if err != nil { | |||
| 31 | - | t.Fatal(err) | |||
| 32 | - | } | |||
| 33 | - | if len(got) != 2 || got[0].Name != "" || got[1].Name != "named" { | |||
| 34 | - | t.Fatalf("LoadLegacy = %#v, want root safe and hashed snapshots", got) | |||
| 35 | - | } | |||
| 36 | - | } | |||
| 37 | - | ||||
| 38 | - | func TestRepositoryLoadLegacyReportsDirectoryCloseError(t *testing.T) { | |||
| 39 | - | for _, tt := range []struct { | |||
| 40 | - | name string | |||
| 41 | - | failOpen int | |||
| 42 | - | operation string | |||
| 43 | - | }{ | |||
| 44 | - | {name: "pending deletion markers", failOpen: 1, operation: "close legacy deletion marker directory"}, | |||
| 45 | - | {name: "legacy snapshots", failOpen: 2, operation: "close legacy snapshot directory"}, | |||
| 46 | - | } { | |||
| 47 | - | t.Run(tt.name, func(t *testing.T) { | |||
| 48 | - | closeErr := errors.New("injected directory close failure") | |||
| 49 | - | repo := NewRepository(privateDir(t)) | |||
| 50 | - | openCalls := 0 | |||
| 51 | - | repo.hooks.openLegacyDirectory = func(string) (legacyDirectory, error) { | |||
| 52 | - | openCalls++ | |||
| 53 | - | if openCalls == tt.failOpen { | |||
| 54 | - | return fakeLegacyDirectory{closeErr: closeErr}, nil | |||
| 55 | - | } | |||
| 56 | - | return fakeLegacyDirectory{}, nil | |||
| 57 | - | } | |||
| 58 | - | ||||
| 59 | - | _, err := repo.LoadLegacy(context.Background()) | |||
| 60 | - | if !errors.Is(err, closeErr) { | |||
| 61 | - | t.Fatalf("LoadLegacy error = %v, want close error", err) | |||
| 62 | - | } | |||
| 63 | - | if !strings.Contains(err.Error(), tt.operation) { | |||
| 64 | - | t.Fatalf("LoadLegacy error = %v, want %q", err, tt.operation) | |||
| 65 | - | } | |||
| 66 | - | }) | |||
| 67 | - | } | |||
| 68 | - | } | |||
| 69 | - | ||||
| 70 | - | type fakeLegacyDirectory struct { | |||
| 71 | - | closeErr error | |||
| 72 | - | } | |||
| 73 | - | ||||
| 74 | - | func (d fakeLegacyDirectory) ReadDir(int) ([]os.DirEntry, error) { return nil, io.EOF } | |||
| 75 | - | func (d fakeLegacyDirectory) Close() error { return d.closeErr } | |||
| 76 | - | ||||
| 77 | - | func TestRepositoryLegacyRejectsTooManyAndTooLargeAggregateSnapshots(t *testing.T) { | |||
| 78 | - | t.Run("count", func(t *testing.T) { | |||
| 79 | - | repo := NewRepository(privateDir(t)) | |||
| 80 | - | if err := os.Mkdir(repo.dir, 0o700); err != nil { | |||
| 81 | - | t.Fatal(err) | |||
| 82 | - | } | |||
| 83 | - | for i := 0; i < maxLegacySnapshotFiles+1; i++ { | |||
| 84 | - | if err := os.WriteFile(filepath.Join(repo.dir, fmt.Sprintf("%03d.snap", i)), []byte("legacy"), 0o600); err != nil { | |||
| 85 | - | t.Fatal(err) | |||
| 86 | - | } | |||
| 87 | - | } | |||
| 88 | - | if _, err := repo.LoadLegacy(context.Background()); err == nil || !strings.Contains(err.Error(), "too many") { | |||
| 89 | - | t.Fatalf("LoadLegacy error = %v, want actionable count limit", err) | |||
| 90 | - | } | |||
| 91 | - | }) | |||
| 92 | - | t.Run("aggregate bytes", func(t *testing.T) { | |||
| 93 | - | repo := NewRepository(privateDir(t)) | |||
| 94 | - | if err := os.Mkdir(repo.dir, 0o700); err != nil { | |||
| 95 | - | t.Fatal(err) | |||
| 96 | - | } | |||
| 97 | - | for i, size := range []int{maxLegacySnapshotBytes / 2, maxLegacySnapshotBytes/2 + 1} { | |||
| 98 | - | if err := os.WriteFile(filepath.Join(repo.dir, fmt.Sprintf("%d.snap", i)), make([]byte, size), 0o600); err != nil { | |||
| 99 | - | t.Fatal(err) | |||
| 100 | - | } | |||
| 101 | - | } | |||
| 102 | - | if _, err := repo.LoadLegacy(context.Background()); err == nil || !strings.Contains(err.Error(), "aggregate") { | |||
| 103 | - | t.Fatalf("LoadLegacy error = %v, want actionable aggregate limit", err) | |||
| 104 | - | } | |||
| 105 | - | }) | |||
| 106 | - | } | |||
| 107 | - | ||||
| 108 | - | func TestRepositoryDeleteLegacyMissingFileDoesNotRequireRoot(t *testing.T) { | |||
| 109 | - | repo := NewRepository(privateDir(t)) | |||
| 110 | - | if err := repo.DeleteLegacy(context.Background(), "missing"); err != nil { | |||
| 111 | - | t.Fatalf("DeleteLegacy missing file: %v", err) | |||
| 112 | - | } | |||
| 113 | - | } | |||
| 114 | - | ||||
| 115 | - | func TestRepositoryDeleteLegacySurfacesRootSyncAndCanRetry(t *testing.T) { | |||
| 116 | - | repo := NewRepository(privateDir(t)) | |||
| 117 | - | if err := os.Mkdir(repo.dir, 0o700); err != nil { | |||
| 118 | - | t.Fatal(err) | |||
| 119 | - | } | |||
| 120 | - | if err := os.WriteFile(filepath.Join(repo.dir, filenameForName("named")), []byte("legacy"), 0o600); err != nil { | |||
| 121 | - | t.Fatal(err) | |||
| 122 | - | } | |||
| 123 | - | injected := errors.New("root sync") | |||
| 124 | - | syncCalls := 0 | |||
| 125 | - | repo.hooks.syncDirectory = func(string) error { | |||
| 126 | - | syncCalls++ | |||
| 127 | - | if syncCalls == 1 { | |||
| 128 | - | return injected | |||
| 129 | - | } | |||
| 130 | - | return nil | |||
| 131 | - | } | |||
| 132 | - | if err := repo.DeleteLegacy(context.Background(), "named"); !errors.Is(err, injected) { | |||
| 133 | - | t.Fatalf("DeleteLegacy error = %v, want root sync error", err) | |||
| 134 | - | } | |||
| 135 | - | if err := repo.DeleteLegacy(context.Background(), "named"); err != nil { | |||
| 136 | - | t.Fatalf("retry DeleteLegacy: %v", err) | |||
| 137 | - | } | |||
| 138 | - | if syncCalls != 2 { | |||
| 139 | - | t.Fatalf("sync calls = %d, want retry to sync absent file", syncCalls) | |||
| 140 | - | } | |||
| 141 | - | } | |||
M internal/adapters/snapshot/load.go +48 -219
| @@ -6,252 +6,83 @@ import ( | |||||
| 6 | "encoding/binary" | 6 | "encoding/binary" | ||
| 7 | "errors" | 7 | "errors" | ||
| 8 | "fmt" | 8 | "fmt" | ||
| 9 | - | "io" | |||
| 10 | "os" | 9 | "os" | ||
| 11 | - | "path/filepath" | |||
| 12 | - | "sort" | |||
| 13 | 10 | ||||
| 11 | + | "github.com/bnema/vev/internal/domain" | |||
| 14 | "github.com/bnema/vev/internal/ports" | 12 | "github.com/bnema/vev/internal/ports" | ||
| 15 | codec "github.com/bnema/vev/internal/usecase/snapshot" | 13 | codec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 16 | ) | 14 | ) | ||
| 17 | 15 | ||||
| 18 | - | // Directory scans are deliberately finite. Snapshot directories are attacker | 16 | + | var ErrInvalidHEAD = errors.New("invalid snapshot HEAD") |
| 19 | - | // controlled state, so callers get a retryable error instead of retaining an | |||
| 20 | - | // arbitrary number of names or spending an arbitrary amount of work. | |||
| 21 | - | const ( | |||
| 22 | - | directoryTraversalBatch = 64 | |||
| 23 | - | maxDirectoryTraversalEntries = 4096 | |||
| 24 | - | ) | |||
| 25 | - | ||||
| 26 | - | var ( | |||
| 27 | - | ErrDirectoryTraversalBudget = errors.New("snapshot directory traversal budget exceeded") | |||
| 28 | - | ErrInvalidHEAD = errors.New("invalid snapshot HEAD") | |||
| 29 | - | ) | |||
| 30 | 17 | ||||
| 31 | - | func (r *Repository) List(ctx context.Context) ([]string, error) { | 18 | + | func (r *Repository) LoadCheckpoint(ctx context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, error) { |
| 32 | if err := ctx.Err(); err != nil { | 19 | if err := ctx.Err(); err != nil { | ||
| 33 | - | return nil, err | 20 | + | return ports.SnapshotGeneration{}, err |
| 34 | - | } | |||
| 35 | - | budget := maxDirectoryTraversalEntries | |||
| 36 | - | out := make([]string, 0) | |||
| 37 | - | sessions := filepath.Join(r.dir, repositorySessionsDir) | |||
| 38 | - | err := r.walkDirectory(ctx, sessions, &budget, func(entry os.DirEntry) error { | |||
| 39 | - | if !entry.IsDir() || !canonicalSessionKey(entry.Name()) { | |||
| 40 | - | return nil | |||
| 41 | - | } | |||
| 42 | - | generation, ok, err := r.loadNewestGeneration(ctx, entry.Name(), &budget) | |||
| 43 | - | if err != nil || !ok { | |||
| 44 | - | return err | |||
| 45 | - | } | |||
| 46 | - | out = append(out, generation.Name) | |||
| 47 | - | return nil | |||
| 48 | - | }) | |||
| 49 | - | if errors.Is(err, os.ErrNotExist) { | |||
| 50 | - | return nil, nil | |||
| 51 | } | 21 | } | ||
| 22 | + | key, err := incarnationKey(id) | |||
| 52 | if err != nil { | 23 | if err != nil { | ||
| 53 | - | return nil, fmt.Errorf("read snapshot sessions: %w", err) | |||
| 54 | - | } | |||
| 55 | - | sort.Strings(out) | |||
| 56 | - | return out, nil | |||
| 57 | - | } | |||
| 58 | - | ||||
| 59 | - | func (r *Repository) Load(ctx context.Context, name string) (ports.SnapshotGeneration, error) { | |||
| 60 | - | if err := ctx.Err(); err != nil { | |||
| 61 | return ports.SnapshotGeneration{}, err | 24 | return ports.SnapshotGeneration{}, err | ||
| 62 | } | 25 | } | ||
| 63 | - | key := sessionKey(name) | |||
| 64 | lock := r.lockSession(key) | 26 | lock := r.lockSession(key) | ||
| 65 | defer r.unlockSession(lock) | 27 | defer r.unlockSession(lock) | ||
| 66 | - | if err := ctx.Err(); err != nil { | 28 | + | generation, _, err := r.loadCheckpointLocked(ctx, id, name, ref) |
| 67 | - | return ports.SnapshotGeneration{}, err | 29 | + | return generation, err |
| 68 | - | } | |||
| 69 | - | killed, err := r.tombstoned(name) | |||
| 70 | - | if err != nil { | |||
| 71 | - | return ports.SnapshotGeneration{}, fmt.Errorf("read killed session marker: %w", err) | |||
| 72 | - | } | |||
| 73 | - | if killed { | |||
| 74 | - | return ports.SnapshotGeneration{}, fmt.Errorf("snapshot session %q is killed", name) | |||
| 75 | - | } | |||
| 76 | - | return r.load(ctx, name, key) | |||
| 77 | } | 30 | } | ||
| 78 | 31 | ||||
| 79 | - | func (r *Repository) load(ctx context.Context, name, key string) (ports.SnapshotGeneration, error) { | 32 | + | func (r *Repository) loadCheckpointLocked(ctx context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, codec.Manifest, error) { |
| 80 | - | preferred, preferredDigest, err := r.readHead(key) | 33 | + | if ref.Generation == 0 || ref.ManifestDigest == ([32]byte{}) { |
| 81 | - | skipped := errors.Is(err, ErrInvalidHEAD) | 34 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid checkpoint reference") |
| 82 | - | if err != nil && !errors.Is(err, os.ErrNotExist) && !errors.Is(err, ErrInvalidHEAD) { | |||
| 83 | - | return ports.SnapshotGeneration{}, fmt.Errorf("read snapshot HEAD: %w", err) | |||
| 84 | } | 35 | } | ||
| 85 | - | if preferred != 0 { | 36 | + | data, err := r.readBounded(r.manifestPath(id, ref.Generation)) |
| 86 | - | got, err := r.loadGeneration(ctx, name, key, preferred) | |||
| 87 | - | if err == nil && sha256.Sum256(got.Manifest) == preferredDigest { | |||
| 88 | - | return got, nil | |||
| 89 | - | } | |||
| 90 | - | skipped = true | |||
| 91 | - | } | |||
| 92 | - | ||||
| 93 | - | budget := maxDirectoryTraversalEntries | |||
| 94 | - | candidates, err := r.generationCandidates(ctx, key, preferred, &budget) | |||
| 95 | if err != nil { | 37 | if err != nil { | ||
| 96 | - | return ports.SnapshotGeneration{}, err | 38 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, err |
| 97 | - | } | |||
| 98 | - | for _, generation := range candidates { | |||
| 99 | - | got, err := r.loadGeneration(ctx, name, key, generation) | |||
| 100 | - | if err != nil { | |||
| 101 | - | skipped = true | |||
| 102 | - | continue | |||
| 103 | - | } | |||
| 104 | - | got.Fallback = skipped || preferred != 0 | |||
| 105 | - | return got, nil | |||
| 106 | - | } | |||
| 107 | - | return ports.SnapshotGeneration{}, fmt.Errorf("no complete snapshot generation for %q", name) | |||
| 108 | - | } | |||
| 109 | - | ||||
| 110 | - | func (r *Repository) currentGeneration(ctx context.Context, name, key string) (uint64, []byte, error) { | |||
| 111 | - | generation, digest, err := r.readHead(key) | |||
| 112 | - | if err == nil { | |||
| 113 | - | got, err := r.loadGeneration(ctx, name, key, generation) | |||
| 114 | - | if err == nil && sha256.Sum256(got.Manifest) == digest { | |||
| 115 | - | return generation, got.Manifest, nil | |||
| 116 | - | } | |||
| 117 | - | } else if !errors.Is(err, os.ErrNotExist) && !errors.Is(err, ErrInvalidHEAD) { | |||
| 118 | - | return 0, nil, fmt.Errorf("read snapshot HEAD: %w", err) | |||
| 119 | } | 39 | } | ||
| 120 | - | budget := maxDirectoryTraversalEntries | 40 | + | if sha256.Sum256(data) != ref.ManifestDigest { |
| 121 | - | candidates, err := r.generationCandidates(ctx, key, 0, &budget) | 41 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("manifest digest mismatch") |
| 122 | - | if err != nil { | |||
| 123 | - | return 0, nil, err | |||
| 124 | } | 42 | } | ||
| 125 | - | for _, generation := range candidates { | 43 | + | manifest, err := codec.UnmarshalManifest(data) |
| 126 | - | got, err := r.loadGeneration(ctx, name, key, generation) | 44 | + | if err != nil || manifest.IncarnationID != id || manifest.Generation != ref.Generation { |
| 127 | - | if err == nil { | 45 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid manifest") |
| 128 | - | return generation, got.Manifest, nil | |||
| 129 | - | } | |||
| 130 | } | 46 | } | ||
| 131 | - | return 0, nil, nil | 47 | + | refs := manifestRefs(manifest) |
| 132 | - | } | 48 | + | if refs == nil || !withinGenerationBudget(len(data), refs) { |
| 133 | - | 49 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("snapshot generation too large") | |
| 134 | - | // loadNewestGeneration validates a single bounded candidate batch in strict | |||
| 135 | - | // descending generation order. It is used by List, whose name is discovered | |||
| 136 | - | // from each manifest rather than supplied by a caller. | |||
| 137 | - | func (r *Repository) loadNewestGeneration(ctx context.Context, key string, budget *int) (ports.SnapshotGeneration, bool, error) { | |||
| 138 | - | candidates, err := r.generationCandidates(ctx, key, 0, budget) | |||
| 139 | - | if err != nil { | |||
| 140 | - | return ports.SnapshotGeneration{}, false, err | |||
| 141 | } | 50 | } | ||
| 142 | - | for _, generation := range candidates { | 51 | + | objects := make(map[ports.SnapshotDigest][]byte, len(refs)) |
| 143 | - | data, err := r.readBounded(r.manifestPath(key, generation)) | 52 | + | for digest, objectRef := range refs { |
| 144 | - | if err != nil { | 53 | + | if err := ctx.Err(); err != nil { |
| 145 | - | continue | 54 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, err |
| 146 | - | } | |||
| 147 | - | manifest, err := codec.UnmarshalManifest(data) | |||
| 148 | - | if err != nil || sessionKey(manifest.Name) != key { | |||
| 149 | - | continue | |||
| 150 | } | 55 | } | ||
| 151 | - | killed, err := r.tombstoned(manifest.Name) | 56 | + | object, err := r.readBounded(r.objectPath(id, digest)) |
| 152 | if err != nil { | 57 | if err != nil { | ||
| 153 | - | return ports.SnapshotGeneration{}, false, fmt.Errorf("read killed session marker: %w", err) | 58 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, err |
| 154 | - | } | |||
| 155 | - | if killed { | |||
| 156 | - | return ports.SnapshotGeneration{}, false, nil | |||
| 157 | } | 59 | } | ||
| 158 | - | got, err := r.loadGeneration(ctx, manifest.Name, key, generation) | 60 | + | if sha256.Sum256(object) != digest || !validObject(object, objectRef) { |
| 159 | - | if err == nil { | 61 | + | return ports.SnapshotGeneration{}, codec.Manifest{}, fmt.Errorf("invalid object") |
| 160 | - | return got, true, nil | |||
| 161 | } | 62 | } | ||
| 63 | + | objects[digest] = object | |||
| 162 | } | 64 | } | ||
| 163 | - | return ports.SnapshotGeneration{}, false, nil | 65 | + | return ports.SnapshotGeneration{IncarnationID: id, Name: name, Generation: ref.Generation, ParentCheckpoint: manifest.ParentCheckpoint, Manifest: data, Objects: objects}, manifest, nil |
| 164 | } | 66 | } | ||
| 165 | 67 | ||||
| 166 | - | // generationCandidates reads the generations directory once per load attempt. | 68 | + | func (r *Repository) ReconcileCheckpoint(ctx context.Context, id domain.IncarnationID, ref ports.CheckpointRef) error { |
| 167 | - | // The shared traversal budget bounds both retained names and work; sorting the | 69 | + | if err := ctx.Err(); err != nil { |
| 168 | - | // bounded batch preserves strict newest-to-oldest fallback without rescanning. | 70 | + | return err |
| 169 | - | func (r *Repository) generationCandidates(ctx context.Context, key string, exclude uint64, budget *int) ([]uint64, error) { | |||
| 170 | - | candidates := make([]uint64, 0) | |||
| 171 | - | err := r.walkDirectory(ctx, filepath.Join(r.sessionPath(key), repositoryGenerations), budget, func(entry os.DirEntry) error { | |||
| 172 | - | if entry.IsDir() { | |||
| 173 | - | return nil | |||
| 174 | - | } | |||
| 175 | - | generation, ok := parseGenerationFilename(entry.Name()) | |||
| 176 | - | if ok && generation != exclude { | |||
| 177 | - | candidates = append(candidates, generation) | |||
| 178 | - | } | |||
| 179 | - | return nil | |||
| 180 | - | }) | |||
| 181 | - | if errors.Is(err, os.ErrNotExist) { | |||
| 182 | - | return candidates, nil | |||
| 183 | - | } | |||
| 184 | - | if err != nil { | |||
| 185 | - | return nil, err | |||
| 186 | } | 71 | } | ||
| 187 | - | sort.Slice(candidates, func(i, j int) bool { return candidates[i] > candidates[j] }) | 72 | + | key, err := incarnationKey(id) |
| 188 | - | return candidates, nil | |||
| 189 | - | } | |||
| 190 | - | ||||
| 191 | - | // walkDirectory is the common finite directory traversal primitive for reads. | |||
| 192 | - | // File.ReadDir maintains a cursor on one descriptor and returns at most one | |||
| 193 | - | // small batch, so no full directory enumeration is retained in memory. | |||
| 194 | - | func (r *Repository) walkDirectory(ctx context.Context, dir string, budget *int, visit func(os.DirEntry) error) (err error) { | |||
| 195 | - | file, err := r.openDirectory(dir) | |||
| 196 | if err != nil { | 73 | if err != nil { | ||
| 197 | return err | 74 | return err | ||
| 198 | } | 75 | } | ||
| 199 | - | defer func() { | 76 | + | lock := r.lockSession(key) |
| 200 | - | if closeErr := file.Close(); err == nil && closeErr != nil { | 77 | + | defer r.unlockSession(lock) |
| 201 | - | err = closeErr | 78 | + | _, manifest, err := r.loadCheckpointLocked(ctx, id, "", ref) |
| 202 | - | } | |||
| 203 | - | }() | |||
| 204 | - | for { | |||
| 205 | - | if err := ctx.Err(); err != nil { | |||
| 206 | - | return err | |||
| 207 | - | } | |||
| 208 | - | entries, readErr := file.ReadDir(directoryTraversalBatch) | |||
| 209 | - | for _, entry := range entries { | |||
| 210 | - | if *budget == 0 { | |||
| 211 | - | return ErrDirectoryTraversalBudget | |||
| 212 | - | } | |||
| 213 | - | *budget-- | |||
| 214 | - | if err := visit(entry); err != nil { | |||
| 215 | - | return err | |||
| 216 | - | } | |||
| 217 | - | } | |||
| 218 | - | if errors.Is(readErr, io.EOF) { | |||
| 219 | - | return nil | |||
| 220 | - | } | |||
| 221 | - | if readErr != nil { | |||
| 222 | - | return readErr | |||
| 223 | - | } | |||
| 224 | - | } | |||
| 225 | - | } | |||
| 226 | - | ||||
| 227 | - | func (r *Repository) loadGeneration(ctx context.Context, name, key string, generation uint64) (ports.SnapshotGeneration, error) { | |||
| 228 | - | data, err := r.readBounded(r.manifestPath(key, generation)) | |||
| 229 | if err != nil { | 79 | if err != nil { | ||
| 230 | - | return ports.SnapshotGeneration{}, err | 80 | + | return fmt.Errorf("validate checkpoint before HEAD repair: %w", err) |
| 231 | - | } | |||
| 232 | - | manifest, err := codec.UnmarshalManifest(data) | |||
| 233 | - | if err != nil || manifest.Name != name || manifest.Generation != generation { | |||
| 234 | - | return ports.SnapshotGeneration{}, fmt.Errorf("invalid manifest") | |||
| 235 | } | 81 | } | ||
| 236 | - | refs := manifestRefs(manifest) | 82 | + | if manifest.Name == "" { |
| 237 | - | if refs == nil || !withinGenerationBudget(len(data), refs) { | 83 | + | return fmt.Errorf("validate checkpoint before HEAD repair: invalid manifest name") |
| 238 | - | return ports.SnapshotGeneration{}, fmt.Errorf("snapshot generation too large") | |||
| 239 | } | 84 | } | ||
| 240 | - | objects := make(map[ports.SnapshotDigest][]byte, len(refs)) | 85 | + | return r.writeMutable(r.headPath(id), marshalHead(ref.Generation, ref.ManifestDigest)) |
| 241 | - | for digest, ref := range refs { | |||
| 242 | - | if err := ctx.Err(); err != nil { | |||
| 243 | - | return ports.SnapshotGeneration{}, err | |||
| 244 | - | } | |||
| 245 | - | object, err := r.readBounded(r.objectPath(key, digest)) | |||
| 246 | - | if err != nil { | |||
| 247 | - | return ports.SnapshotGeneration{}, err | |||
| 248 | - | } | |||
| 249 | - | if sha256.Sum256(object) != digest || !validObject(object, ref) { | |||
| 250 | - | return ports.SnapshotGeneration{}, fmt.Errorf("invalid object") | |||
| 251 | - | } | |||
| 252 | - | objects[digest] = object | |||
| 253 | - | } | |||
| 254 | - | return ports.SnapshotGeneration{Name: manifest.Name, Generation: generation, Manifest: data, Objects: objects}, nil | |||
| 255 | } | 86 | } | ||
| 256 | 87 | ||||
| 257 | func marshalHead(generation uint64, digest ports.SnapshotDigest) []byte { | 88 | func marshalHead(generation uint64, digest ports.SnapshotDigest) []byte { | ||
| @@ -261,17 +92,15 @@ func marshalHead(generation uint64, digest ports.SnapshotDigest) []byte { | |||||
| 261 | copy(out[12:], digest[:]) | 92 | copy(out[12:], digest[:]) | ||
| 262 | return out | 93 | return out | ||
| 263 | } | 94 | } | ||
| 264 | - | func (r *Repository) readHead(key string) (uint64, ports.SnapshotDigest, error) { | 95 | + | func (r *Repository) readHead(id domain.IncarnationID) (uint64, ports.SnapshotDigest, error) { |
| 265 | - | return r.readHeadWithRoot(nil, key) | 96 | + | return r.readHeadWithRoot(nil, id) |
| 266 | } | 97 | } | ||
| 267 | 98 | ||||
| 268 | - | func (r *Repository) readHeadWithRoot(root *os.Root, key string) (uint64, ports.SnapshotDigest, error) { | 99 | + | func (r *Repository) readHeadWithRoot(root *os.Root, id domain.IncarnationID) (uint64, ports.SnapshotDigest, error) { |
| 269 | - | path := r.headPath(key) | 100 | + | return r.readHeadAt(root, r.headPath(id)) |
| 270 | - | if hook := r.hooks.beforeHeadRead; hook != nil { | 101 | + | } |
| 271 | - | if err := hook(path); err != nil { | 102 | + | |
| 272 | - | return 0, ports.SnapshotDigest{}, err | 103 | + | func (r *Repository) readHeadAt(root *os.Root, path string) (uint64, ports.SnapshotDigest, error) { |
| 273 | - | } | |||
| 274 | - | } | |||
| 275 | var data []byte | 104 | var data []byte | ||
| 276 | var err error | 105 | var err error | ||
| 277 | if root == nil { | 106 | if root == nil { | ||
D internal/adapters/snapshot/load_bounds_test.go +0 -112
| @@ -1,112 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "errors" | |||
| 6 | - | "fmt" | |||
| 7 | - | "os" | |||
| 8 | - | "path/filepath" | |||
| 9 | - | "testing" | |||
| 10 | - | ) | |||
| 11 | - | ||||
| 12 | - | func TestRepositoryListStopsAtTraversalBudget(t *testing.T) { | |||
| 13 | - | repo := NewRepository(privateDir(t)) | |||
| 14 | - | sessions := filepath.Join(repo.dir, repositorySessionsDir) | |||
| 15 | - | if err := os.MkdirAll(sessions, 0o700); err != nil { | |||
| 16 | - | t.Fatal(err) | |||
| 17 | - | } | |||
| 18 | - | for i := 0; i < maxDirectoryTraversalEntries+1; i++ { | |||
| 19 | - | if err := os.Mkdir(filepath.Join(sessions, fmt.Sprintf("hostile-%05d", i)), 0o700); err != nil { | |||
| 20 | - | t.Fatal(err) | |||
| 21 | - | } | |||
| 22 | - | } | |||
| 23 | - | ||||
| 24 | - | _, err := repo.List(context.Background()) | |||
| 25 | - | if !errors.Is(err, ErrDirectoryTraversalBudget) { | |||
| 26 | - | t.Fatalf("List error = %v, want directory traversal budget", err) | |||
| 27 | - | } | |||
| 28 | - | } | |||
| 29 | - | ||||
| 30 | - | func TestRepositoryLoadStopsAtGenerationTraversalBudget(t *testing.T) { | |||
| 31 | - | repo := NewRepository(privateDir(t)) | |||
| 32 | - | key := sessionKey("named") | |||
| 33 | - | generations := filepath.Join(repo.sessionPath(key), repositoryGenerations) | |||
| 34 | - | if err := os.MkdirAll(generations, 0o700); err != nil { | |||
| 35 | - | t.Fatal(err) | |||
| 36 | - | } | |||
| 37 | - | for generation := uint64(1); generation <= uint64(maxDirectoryTraversalEntries+1); generation++ { | |||
| 38 | - | if err := os.WriteFile(filepath.Join(generations, generationFilename(generation)), nil, 0o600); err != nil { | |||
| 39 | - | t.Fatal(err) | |||
| 40 | - | } | |||
| 41 | - | } | |||
| 42 | - | ||||
| 43 | - | _, err := repo.Load(context.Background(), "named") | |||
| 44 | - | if !errors.Is(err, ErrDirectoryTraversalBudget) { | |||
| 45 | - | t.Fatalf("Load error = %v, want directory traversal budget", err) | |||
| 46 | - | } | |||
| 47 | - | } | |||
| 48 | - | ||||
| 49 | - | func TestRepositoryLoadFallsBackNewestToOldestWithoutGenerationEnumeration(t *testing.T) { | |||
| 50 | - | repo := NewRepository(privateDir(t)) | |||
| 51 | - | for generation := uint64(1); generation <= 3; generation++ { | |||
| 52 | - | if err := repo.Publish(context.Background(), repositoryPublication(t, "named", generation, []byte(fmt.Sprintf("state-%d", generation)))); err != nil { | |||
| 53 | - | t.Fatal(err) | |||
| 54 | - | } | |||
| 55 | - | } | |||
| 56 | - | key := sessionKey("named") | |||
| 57 | - | publication := repositoryPublication(t, "named", 3, []byte("state-3")) | |||
| 58 | - | if err := os.Remove(repo.objectPath(key, publication.Objects[0].Digest)); err != nil { | |||
| 59 | - | t.Fatal(err) | |||
| 60 | - | } | |||
| 61 | - | ||||
| 62 | - | got, err := repo.Load(context.Background(), "named") | |||
| 63 | - | if err != nil { | |||
| 64 | - | t.Fatal(err) | |||
| 65 | - | } | |||
| 66 | - | if got.Generation != 2 || !got.Fallback { | |||
| 67 | - | t.Fatalf("Load = generation %d fallback %v, want generation 2 fallback true", got.Generation, got.Fallback) | |||
| 68 | - | } | |||
| 69 | - | } | |||
| 70 | - | ||||
| 71 | - | func TestRepositoryMaintainDoesNotCollectBeyondRetainedMetadataBudget(t *testing.T) { | |||
| 72 | - | repo := NewRepository(privateDir(t)) | |||
| 73 | - | for generation := uint64(1); generation <= uint64(maxMaintenanceMarkedGenerations+1); generation++ { | |||
| 74 | - | if err := repo.Publish(context.Background(), repositoryPublication(t, "named", generation, []byte(fmt.Sprintf("state-%d", generation)))); err != nil { | |||
| 75 | - | t.Fatal(err) | |||
| 76 | - | } | |||
| 77 | - | } | |||
| 78 | - | for pass := 0; pass < maxMaintenanceMarkedGenerations*3; pass++ { | |||
| 79 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 80 | - | t.Fatal(err) | |||
| 81 | - | } | |||
| 82 | - | if repo.maintenanceSessions[sessionKey("named")] == nil { | |||
| 83 | - | break | |||
| 84 | - | } | |||
| 85 | - | if pass == maxMaintenanceMarkedGenerations*3-1 { | |||
| 86 | - | t.Fatal("maintenance did not complete after bounded object-shard traversal") | |||
| 87 | - | } | |||
| 88 | - | } | |||
| 89 | - | if _, err := os.Lstat(repo.manifestPath(sessionKey("named"), 1)); err != nil { | |||
| 90 | - | t.Fatalf("old manifest collected after mark budget overflow: %v", err) | |||
| 91 | - | } | |||
| 92 | - | if state := repo.maintenanceSessions[sessionKey("named")]; state != nil { | |||
| 93 | - | t.Fatalf("overflow maintenance state retained = %#v, want reset", state) | |||
| 94 | - | } | |||
| 95 | - | } | |||
| 96 | - | ||||
| 97 | - | func TestMaintenanceMarkStateIsCapped(t *testing.T) { | |||
| 98 | - | references := &sessionMaintenance{marked: make(map[uint64]manifestMaintenance)} | |||
| 99 | - | if !references.canRetainManifest(maxMaintenanceReferences) { | |||
| 100 | - | t.Fatal("state rejected references at its documented ceiling") | |||
| 101 | - | } | |||
| 102 | - | if references.canRetainManifest(1) { | |||
| 103 | - | t.Fatal("state accepted references beyond its documented ceiling") | |||
| 104 | - | } | |||
| 105 | - | generations := &sessionMaintenance{marked: make(map[uint64]manifestMaintenance)} | |||
| 106 | - | for generation := uint64(1); generation <= maxMaintenanceMarkedGenerations; generation++ { | |||
| 107 | - | generations.marked[generation] = manifestMaintenance{} | |||
| 108 | - | } | |||
| 109 | - | if generations.canRetainManifest(0) { | |||
| 110 | - | t.Fatal("state accepted generations beyond its documented ceiling") | |||
| 111 | - | } | |||
| 112 | - | } | |||
D internal/adapters/snapshot/load_major_test.go +0 -88
| @@ -1,88 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "os" | |||
| 6 | - | "syscall" | |||
| 7 | - | "testing" | |||
| 8 | - | ||||
| 9 | - | "github.com/stretchr/testify/require" | |||
| 10 | - | ) | |||
| 11 | - | ||||
| 12 | - | func TestRepositoryLoadPropagatesOperationalHeadErrors(t *testing.T) { | |||
| 13 | - | repo := NewRepository(privateDir(t)) | |||
| 14 | - | require.NoError(t, repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state")))) | |||
| 15 | - | ||||
| 16 | - | repo.hooks.beforeHeadRead = func(string) error { return os.ErrPermission } | |||
| 17 | - | ||||
| 18 | - | _, err := repo.Load(context.Background(), "named") | |||
| 19 | - | require.ErrorIs(t, err, os.ErrPermission) | |||
| 20 | - | } | |||
| 21 | - | ||||
| 22 | - | func TestRepositoryCurrentGenerationPropagatesOperationalHeadErrors(t *testing.T) { | |||
| 23 | - | repo := NewRepository(privateDir(t)) | |||
| 24 | - | require.NoError(t, repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state")))) | |||
| 25 | - | ||||
| 26 | - | repo.hooks.beforeHeadRead = func(string) error { return syscall.EIO } | |||
| 27 | - | ||||
| 28 | - | _, _, err := repo.currentGeneration(context.Background(), "named", sessionKey("named")) | |||
| 29 | - | require.ErrorIs(t, err, syscall.EIO) | |||
| 30 | - | } | |||
| 31 | - | ||||
| 32 | - | func TestRepositoryLoadFallsBackFromInvalidHead(t *testing.T) { | |||
| 33 | - | repo := NewRepository(privateDir(t)) | |||
| 34 | - | require.NoError(t, repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state")))) | |||
| 35 | - | require.NoError(t, os.WriteFile(repo.headPath(sessionKey("named")), []byte("invalid"), 0o600)) | |||
| 36 | - | ||||
| 37 | - | got, err := repo.Load(context.Background(), "named") | |||
| 38 | - | require.NoError(t, err) | |||
| 39 | - | require.Equal(t, uint64(1), got.Generation) | |||
| 40 | - | require.True(t, got.Fallback) | |||
| 41 | - | } | |||
| 42 | - | ||||
| 43 | - | func TestRepositoryLoadFallsBackAcrossMultipleCorruptCandidates(t *testing.T) { | |||
| 44 | - | repo := NewRepository(privateDir(t)) | |||
| 45 | - | for generation := uint64(1); generation <= 4; generation++ { | |||
| 46 | - | require.NoError(t, repo.Publish(context.Background(), repositoryPublication(t, "named", generation, []byte{byte(generation)}))) | |||
| 47 | - | } | |||
| 48 | - | key := sessionKey("named") | |||
| 49 | - | for _, generation := range []uint64{4, 3} { | |||
| 50 | - | publication := repositoryPublication(t, "named", generation, []byte{byte(generation)}) | |||
| 51 | - | require.NoError(t, os.Remove(repo.objectPath(key, publication.Objects[0].Digest))) | |||
| 52 | - | } | |||
| 53 | - | ||||
| 54 | - | got, err := repo.Load(context.Background(), "named") | |||
| 55 | - | require.NoError(t, err) | |||
| 56 | - | require.Equal(t, uint64(2), got.Generation) | |||
| 57 | - | require.True(t, got.Fallback) | |||
| 58 | - | } | |||
| 59 | - | ||||
| 60 | - | func TestRepositoryLoadHoldsSessionLockAcrossTombstoneCheck(t *testing.T) { | |||
| 61 | - | repo := NewRepository(privateDir(t)) | |||
| 62 | - | require.NoError(t, repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state")))) | |||
| 63 | - | ||||
| 64 | - | tombstoneCheckStarted := make(chan struct{}) | |||
| 65 | - | releaseTombstoneCheck := make(chan struct{}) | |||
| 66 | - | repo.hooks.beforeTombstoneCheck = func(string) { | |||
| 67 | - | close(tombstoneCheckStarted) | |||
| 68 | - | <-releaseTombstoneCheck | |||
| 69 | - | } | |||
| 70 | - | loadDone := make(chan error, 1) | |||
| 71 | - | go func() { | |||
| 72 | - | _, err := repo.Load(context.Background(), "named") | |||
| 73 | - | loadDone <- err | |||
| 74 | - | }() | |||
| 75 | - | <-tombstoneCheckStarted | |||
| 76 | - | ||||
| 77 | - | tombstoneDone := make(chan error, 1) | |||
| 78 | - | go func() { tombstoneDone <- repo.Tombstone(context.Background(), "named") }() | |||
| 79 | - | select { | |||
| 80 | - | case err := <-tombstoneDone: | |||
| 81 | - | t.Fatalf("Tombstone completed before Load released the session lock: %v", err) | |||
| 82 | - | default: | |||
| 83 | - | } | |||
| 84 | - | ||||
| 85 | - | close(releaseTombstoneCheck) | |||
| 86 | - | require.NoError(t, <-loadDone) | |||
| 87 | - | require.NoError(t, <-tombstoneDone) | |||
| 88 | - | } | |||
D internal/adapters/snapshot/maintenance.go +0 -129
| @@ -1,129 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "errors" | |||
| 6 | - | "fmt" | |||
| 7 | - | "os" | |||
| 8 | - | "path/filepath" | |||
| 9 | - | ) | |||
| 10 | - | ||||
| 11 | - | const maintenanceBatch = 64 | |||
| 12 | - | ||||
| 13 | - | // These caps bound metadata retained across resumable GC passes. If a session | |||
| 14 | - | // exceeds either cap, marking remains conservative and no destructive sweep or | |||
| 15 | - | // generation collection occurs in that cycle. | |||
| 16 | - | const ( | |||
| 17 | - | maxMaintenanceMarkedGenerations = 128 | |||
| 18 | - | maxMaintenanceReferences = 8192 | |||
| 19 | - | ) | |||
| 20 | - | ||||
| 21 | - | // Maintain reaps a bounded amount of stale state. Continuation handles are | |||
| 22 | - | // repository-owned, so a stable prefix cannot starve entries later in a large | |||
| 23 | - | // directory. Cancellation and errors discard those handles. | |||
| 24 | - | func (r *Repository) Maintain(ctx context.Context) (err error) { | |||
| 25 | - | r.maintenanceMu.Lock() | |||
| 26 | - | defer r.maintenanceMu.Unlock() | |||
| 27 | - | if err := ctx.Err(); err != nil { | |||
| 28 | - | r.resetMaintenance() | |||
| 29 | - | return err | |||
| 30 | - | } | |||
| 31 | - | defer func() { | |||
| 32 | - | if err != nil { | |||
| 33 | - | r.resetMaintenance() | |||
| 34 | - | } | |||
| 35 | - | }() | |||
| 36 | - | if r.maintenanceCursors == nil { | |||
| 37 | - | r.maintenanceCursors = make(map[string]*maintenanceCursor) | |||
| 38 | - | r.maintenanceSessions = make(map[string]*sessionMaintenance) | |||
| 39 | - | } | |||
| 40 | - | ||||
| 41 | - | budget := maintenanceBatch | |||
| 42 | - | if _, err := r.removeTemps(ctx, r.dir, &budget, "root-temps"); err != nil || budget == 0 { | |||
| 43 | - | return err | |||
| 44 | - | } | |||
| 45 | - | if r.maintenanceQuarantine != nil { | |||
| 46 | - | changed, done, err := r.maintainQuarantine(ctx, &budget) | |||
| 47 | - | if err != nil { | |||
| 48 | - | return err | |||
| 49 | - | } | |||
| 50 | - | if changed { | |||
| 51 | - | if err := r.syncDirectory(filepath.Join(r.dir, repositorySessionsDir)); err != nil { | |||
| 52 | - | return fmt.Errorf("sync snapshot maintenance directory: %w", safeFilesystemError(err)) | |||
| 53 | - | } | |||
| 54 | - | } | |||
| 55 | - | if done { | |||
| 56 | - | r.maintenanceQuarantine = nil | |||
| 57 | - | } | |||
| 58 | - | return nil | |||
| 59 | - | } | |||
| 60 | - | // Resume an incomplete session before discovering another one. This keeps | |||
| 61 | - | // retained marks (and their session lock/epoch references) bounded even | |||
| 62 | - | // when a hostile repository contains arbitrarily many partial sessions. | |||
| 63 | - | for key := range r.maintenanceSessions { | |||
| 64 | - | lock := r.lockSession(key) | |||
| 65 | - | err = r.maintainSession(ctx, key, &budget) | |||
| 66 | - | r.unlockSession(lock) | |||
| 67 | - | return err | |||
| 68 | - | } | |||
| 69 | - | sessions := filepath.Join(r.dir, repositorySessionsDir) | |||
| 70 | - | entries, _, err := r.readMaintenanceDir(sessions, maintenanceBatch, "sessions") | |||
| 71 | - | if errors.Is(err, os.ErrNotExist) { | |||
| 72 | - | return nil | |||
| 73 | - | } | |||
| 74 | - | if err != nil { | |||
| 75 | - | return fmt.Errorf("read snapshot maintenance sessions: %w", safeFilesystemError(err)) | |||
| 76 | - | } | |||
| 77 | - | for i, entry := range entries { | |||
| 78 | - | if budget == 0 { | |||
| 79 | - | r.requeueMaintenanceEntries(sessions, "sessions", entries[i:]) | |||
| 80 | - | return nil | |||
| 81 | - | } | |||
| 82 | - | if err := ctx.Err(); err != nil { | |||
| 83 | - | return err | |||
| 84 | - | } | |||
| 85 | - | path := filepath.Join(sessions, entry.name) | |||
| 86 | - | if isQuarantine(entry.name) { | |||
| 87 | - | r.maintenanceQuarantine = &quarantineMaintenance{root: path, current: path} | |||
| 88 | - | changed, done, err := r.maintainQuarantine(ctx, &budget) | |||
| 89 | - | if err != nil && !errors.Is(err, os.ErrNotExist) { | |||
| 90 | - | return fmt.Errorf("remove snapshot quarantine %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err)) | |||
| 91 | - | } | |||
| 92 | - | if changed { | |||
| 93 | - | if err := ctx.Err(); err != nil { | |||
| 94 | - | return err | |||
| 95 | - | } | |||
| 96 | - | if err := r.syncDirectory(sessions); err != nil { | |||
| 97 | - | return fmt.Errorf("sync snapshot maintenance directory for %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err)) | |||
| 98 | - | } | |||
| 99 | - | } | |||
| 100 | - | if done { | |||
| 101 | - | r.maintenanceQuarantine = nil | |||
| 102 | - | } | |||
| 103 | - | if i+1 < len(entries) { | |||
| 104 | - | r.requeueMaintenanceEntries(sessions, "sessions", entries[i+1:]) | |||
| 105 | - | } | |||
| 106 | - | return nil | |||
| 107 | - | } | |||
| 108 | - | if !entry.isDir || !canonicalSessionKey(entry.name) { | |||
| 109 | - | continue | |||
| 110 | - | } | |||
| 111 | - | lock := r.lockSession(entry.name) | |||
| 112 | - | err = r.maintainSession(ctx, entry.name, &budget) | |||
| 113 | - | r.unlockSession(lock) | |||
| 114 | - | if err != nil { | |||
| 115 | - | return err | |||
| 116 | - | } | |||
| 117 | - | if r.maintenanceSessions[entry.name] != nil { | |||
| 118 | - | return nil | |||
| 119 | - | } | |||
| 120 | - | } | |||
| 121 | - | return nil | |||
| 122 | - | } | |||
| 123 | - | ||||
| 124 | - | // maintenanceCursor is deliberately reopenable metadata, not an open file. | |||
| 125 | - | // On Linux, d_off is the seek cookie returned by getdents(2); seeking a newly | |||
| 126 | - | // opened descriptor to it resumes the directory stream. Go's File.ReadDir | |||
| 127 | - | // cannot be used here: its 8 KiB internal getdents buffer advances the kernel | |||
| 128 | - | // offset beyond a short ReadDir result (see os/dir_unix.go), so saving SeekCur | |||
| 129 | - | // after ReadDir would skip buffered entries. | |||
D internal/adapters/snapshot/maintenance_cursor.go +0 -279
| @@ -1,279 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "errors" | |||
| 6 | - | "fmt" | |||
| 7 | - | "io" | |||
| 8 | - | "os" | |||
| 9 | - | "path/filepath" | |||
| 10 | - | "strings" | |||
| 11 | - | "syscall" | |||
| 12 | - | "unsafe" | |||
| 13 | - | ) | |||
| 14 | - | ||||
| 15 | - | // maxMaintenanceCursors bounds all retained directory seek cookies, including | |||
| 16 | - | // cursors for attacker-controlled object shards. Maintenance processes only | |||
| 17 | - | // one shard at a time, so normal operation stays well below this ceiling. | |||
| 18 | - | const maxMaintenanceCursors = 8 | |||
| 19 | - | ||||
| 20 | - | var errMaintenanceCursorLimit = errors.New("snapshot maintenance cursor limit") | |||
| 21 | - | ||||
| 22 | - | type maintenanceCursor struct { | |||
| 23 | - | offset int64 | |||
| 24 | - | pending []maintenanceDirEntry | |||
| 25 | - | done bool | |||
| 26 | - | } | |||
| 27 | - | ||||
| 28 | - | type maintenanceDirEntry struct { | |||
| 29 | - | name string | |||
| 30 | - | isDir bool | |||
| 31 | - | } | |||
| 32 | - | ||||
| 33 | - | func (r *Repository) resetMaintenance() { | |||
| 34 | - | for key := range r.maintenanceSessions { | |||
| 35 | - | r.clearSessionMaintenance(key) | |||
| 36 | - | } | |||
| 37 | - | r.maintenanceCursors = nil | |||
| 38 | - | r.maintenanceSessions = nil | |||
| 39 | - | r.maintenanceQuarantine = nil | |||
| 40 | - | } | |||
| 41 | - | ||||
| 42 | - | func maintenanceCursorID(dir, purpose string) string { return purpose + "\x00" + dir } | |||
| 43 | - | ||||
| 44 | - | // readMaintenanceDir advances one named directory cursor. Its continuation is | |||
| 45 | - | // only a Linux seek cookie and at most one call's worth of pending names. The | |||
| 46 | - | // descriptor used to read a call is always closed before this method returns. | |||
| 47 | - | // Callers must return entries they could not process with | |||
| 48 | - | // requeueMaintenanceEntries before yielding for budget exhaustion. | |||
| 49 | - | func (r *Repository) readMaintenanceDir(dir string, n int, purpose string) ([]maintenanceDirEntry, bool, error) { | |||
| 50 | - | id := maintenanceCursorID(dir, purpose) | |||
| 51 | - | cursor := r.maintenanceCursors[id] | |||
| 52 | - | if cursor != nil && len(cursor.pending) != 0 { | |||
| 53 | - | count := min(n, len(cursor.pending)) | |||
| 54 | - | entries := cursor.pending[:count] | |||
| 55 | - | cursor.pending = cursor.pending[count:] | |||
| 56 | - | done := cursor.done && len(cursor.pending) == 0 | |||
| 57 | - | if done { | |||
| 58 | - | delete(r.maintenanceCursors, id) | |||
| 59 | - | } | |||
| 60 | - | return entries, done, nil | |||
| 61 | - | } | |||
| 62 | - | if cursor == nil { | |||
| 63 | - | if len(r.maintenanceCursors) >= maxMaintenanceCursors { | |||
| 64 | - | return nil, false, errMaintenanceCursorLimit | |||
| 65 | - | } | |||
| 66 | - | cursor = &maintenanceCursor{} | |||
| 67 | - | r.maintenanceCursors[id] = cursor | |||
| 68 | - | } | |||
| 69 | - | ||||
| 70 | - | entries, done, err := r.readMaintenanceDirent(dir, n, cursor) | |||
| 71 | - | if err != nil { | |||
| 72 | - | delete(r.maintenanceCursors, id) | |||
| 73 | - | return nil, false, err | |||
| 74 | - | } | |||
| 75 | - | if done { | |||
| 76 | - | delete(r.maintenanceCursors, id) | |||
| 77 | - | } | |||
| 78 | - | return entries, done, nil | |||
| 79 | - | } | |||
| 80 | - | ||||
| 81 | - | // maintenanceDirectory is the small syscall surface needed to resume a | |||
| 82 | - | // bounded getdents cursor. It permits deterministic fault injection without | |||
| 83 | - | // retaining directory descriptors between maintenance passes. | |||
| 84 | - | type maintenanceDirectory interface { | |||
| 85 | - | Seek(int64, int) (int64, error) | |||
| 86 | - | ReadDirent([]byte) (int, error) | |||
| 87 | - | Close() error | |||
| 88 | - | } | |||
| 89 | - | ||||
| 90 | - | type osMaintenanceDirectory struct{ file *os.File } | |||
| 91 | - | ||||
| 92 | - | func (d osMaintenanceDirectory) Seek(offset int64, whence int) (int64, error) { | |||
| 93 | - | return syscall.Seek(int(d.file.Fd()), offset, whence) | |||
| 94 | - | } | |||
| 95 | - | ||||
| 96 | - | func (d osMaintenanceDirectory) ReadDirent(buffer []byte) (int, error) { | |||
| 97 | - | return syscall.ReadDirent(int(d.file.Fd()), buffer) | |||
| 98 | - | } | |||
| 99 | - | ||||
| 100 | - | func (d osMaintenanceDirectory) Close() error { return d.file.Close() } | |||
| 101 | - | ||||
| 102 | - | func (r *Repository) openMaintenanceDirectory(dir string) (maintenanceDirectory, error) { | |||
| 103 | - | if open := r.hooks.openMaintenanceDirectory; open != nil { | |||
| 104 | - | return open(dir) | |||
| 105 | - | } | |||
| 106 | - | file, err := r.openDirectory(dir) | |||
| 107 | - | if err != nil { | |||
| 108 | - | return nil, err | |||
| 109 | - | } | |||
| 110 | - | return osMaintenanceDirectory{file: file}, nil | |||
| 111 | - | } | |||
| 112 | - | ||||
| 113 | - | // maintenanceDirectoryError adds stable operation context and removes any | |||
| 114 | - | // filesystem path while retaining the underlying cause for errors.Is. | |||
| 115 | - | func maintenanceDirectoryError(operation string, err error) error { | |||
| 116 | - | return fmt.Errorf("%s: %w", operation, safeFilesystemError(err)) | |||
| 117 | - | } | |||
| 118 | - | ||||
| 119 | - | // readMaintenanceDirent saves a resumable directory cursor. Linux d_off lets | |||
| 120 | - | // a large getdents batch stop at limit. Darwin's syscall.ReadDirent instead | |||
| 121 | - | // stores an entry count in the descriptor offset, so every returned buffer | |||
| 122 | - | // must be drained before that descriptor-maintained count is saved. | |||
| 123 | - | func (r *Repository) readMaintenanceDirent(dir string, limit int, cursor *maintenanceCursor) (entries []maintenanceDirEntry, done bool, err error) { | |||
| 124 | - | return r.readMaintenanceDirentWithDrain(dir, limit, cursor, drainMaintenanceDirentBatch(), directoryCookie) | |||
| 125 | - | } | |||
| 126 | - | ||||
| 127 | - | // readMaintenanceDirentWithDrain contains the shared traversal logic. The | |||
| 128 | - | // explicit parameters permit platform-independent verification of Darwin's | |||
| 129 | - | // buffer-draining semantics with deterministic fake directories. | |||
| 130 | - | func (r *Repository) readMaintenanceDirentWithDrain(dir string, limit int, cursor *maintenanceCursor, drainBuffer bool, cookie func(maintenanceDirectory, *syscall.Dirent) (int64, error)) (entries []maintenanceDirEntry, done bool, err error) { | |||
| 131 | - | file, err := r.openMaintenanceDirectory(dir) | |||
| 132 | - | if err != nil { | |||
| 133 | - | return nil, false, maintenanceDirectoryError("open maintenance directory", err) | |||
| 134 | - | } | |||
| 135 | - | defer func() { | |||
| 136 | - | if closeErr := file.Close(); closeErr != nil { | |||
| 137 | - | closeErr = maintenanceDirectoryError("close maintenance directory", closeErr) | |||
| 138 | - | if err != nil { | |||
| 139 | - | err = errors.Join(err, closeErr) | |||
| 140 | - | } else { | |||
| 141 | - | entries = nil | |||
| 142 | - | done = false | |||
| 143 | - | err = closeErr | |||
| 144 | - | } | |||
| 145 | - | } | |||
| 146 | - | }() | |||
| 147 | - | if cursor.offset != 0 { | |||
| 148 | - | if _, err := file.Seek(cursor.offset, io.SeekStart); err != nil { | |||
| 149 | - | return nil, false, maintenanceDirectoryError("seek maintenance directory", err) | |||
| 150 | - | } | |||
| 151 | - | } | |||
| 152 | - | ||||
| 153 | - | entries = make([]maintenanceDirEntry, 0, limit) | |||
| 154 | - | buffer := make([]byte, maintenanceDirentBufferSize) | |||
| 155 | - | for { | |||
| 156 | - | count, err := file.ReadDirent(buffer) | |||
| 157 | - | if err != nil { | |||
| 158 | - | return nil, false, maintenanceDirectoryError("read maintenance directory", err) | |||
| 159 | - | } | |||
| 160 | - | if count < 0 || count > len(buffer) { | |||
| 161 | - | return nil, false, syscall.EIO | |||
| 162 | - | } | |||
| 163 | - | if count == 0 { | |||
| 164 | - | return entries, true, nil | |||
| 165 | - | } | |||
| 166 | - | for data := buffer[:count]; len(data) != 0 && (len(entries) < limit || drainBuffer); { | |||
| 167 | - | record, nameBytes, reclen, decodeErr := decodeMaintenanceDirent(data) | |||
| 168 | - | if decodeErr != nil { | |||
| 169 | - | return nil, false, decodeErr | |||
| 170 | - | } | |||
| 171 | - | data = data[reclen:] | |||
| 172 | - | // Advance past every raw record, including dot and disappeared entries. | |||
| 173 | - | offset, cookieErr := cookie(file, record) | |||
| 174 | - | if cookieErr != nil { | |||
| 175 | - | return nil, false, maintenanceDirectoryError("tell maintenance directory", cookieErr) | |||
| 176 | - | } | |||
| 177 | - | cursor.offset = offset | |||
| 178 | - | if end := strings.IndexByte(string(nameBytes), 0); end >= 0 { | |||
| 179 | - | nameBytes = nameBytes[:end] | |||
| 180 | - | } | |||
| 181 | - | name := string(nameBytes) | |||
| 182 | - | if name == "." || name == ".." || name == "" { | |||
| 183 | - | continue | |||
| 184 | - | } | |||
| 185 | - | stat, err := r.stat(filepath.Join(dir, name)) | |||
| 186 | - | if errors.Is(err, os.ErrNotExist) { | |||
| 187 | - | continue | |||
| 188 | - | } | |||
| 189 | - | if err != nil { | |||
| 190 | - | return nil, false, maintenanceDirectoryError("stat maintenance directory entry", err) | |||
| 191 | - | } | |||
| 192 | - | entries = append(entries, maintenanceDirEntry{name: name, isDir: stat.IsDir()}) | |||
| 193 | - | } | |||
| 194 | - | if len(entries) >= limit { | |||
| 195 | - | if drainBuffer && len(entries) > limit { | |||
| 196 | - | // Darwin has already advanced the descriptor past this entire | |||
| 197 | - | // buffer, so retain only this final buffer's excess. | |||
| 198 | - | cursor.pending = append(cursor.pending, entries[limit:]...) | |||
| 199 | - | } | |||
| 200 | - | return entries[:limit], false, nil | |||
| 201 | - | } | |||
| 202 | - | } | |||
| 203 | - | } | |||
| 204 | - | ||||
| 205 | - | // decodeMaintenanceDirent copies a bounded raw record into aligned storage | |||
| 206 | - | // before reading its fields. syscall.ReadDirent writes to a byte buffer, whose | |||
| 207 | - | // start and record boundaries are not guaranteed to satisfy syscall.Dirent's | |||
| 208 | - | // alignment requirements on Darwin. | |||
| 209 | - | func decodeMaintenanceDirent(data []byte) (*syscall.Dirent, []byte, int, error) { | |||
| 210 | - | nameOffset := int(unsafe.Offsetof(syscall.Dirent{}.Name)) | |||
| 211 | - | if len(data) < nameOffset { | |||
| 212 | - | return nil, nil, 0, syscall.EIO | |||
| 213 | - | } | |||
| 214 | - | ||||
| 215 | - | var record syscall.Dirent | |||
| 216 | - | header := unsafe.Slice((*byte)(unsafe.Pointer(&record)), nameOffset) | |||
| 217 | - | copy(header, data[:nameOffset]) | |||
| 218 | - | reclen := int(record.Reclen) | |||
| 219 | - | if reclen < nameOffset || reclen > len(data) || reclen > int(unsafe.Sizeof(record)) { | |||
| 220 | - | return nil, nil, 0, syscall.EIO | |||
| 221 | - | } | |||
| 222 | - | ||||
| 223 | - | recordBytes := unsafe.Slice((*byte)(unsafe.Pointer(&record)), reclen) | |||
| 224 | - | copy(recordBytes, data[:reclen]) | |||
| 225 | - | name := unsafe.Slice((*byte)(unsafe.Pointer(&record.Name[0])), reclen-nameOffset) | |||
| 226 | - | return &record, name, reclen, nil | |||
| 227 | - | } | |||
| 228 | - | ||||
| 229 | - | func (r *Repository) requeueMaintenanceEntries(dir, purpose string, entries []maintenanceDirEntry) { | |||
| 230 | - | if len(entries) == 0 { | |||
| 231 | - | return | |||
| 232 | - | } | |||
| 233 | - | id := maintenanceCursorID(dir, purpose) | |||
| 234 | - | cursor := r.maintenanceCursors[id] | |||
| 235 | - | if cursor == nil { | |||
| 236 | - | cursor = &maintenanceCursor{done: true} | |||
| 237 | - | r.maintenanceCursors[id] = cursor | |||
| 238 | - | } | |||
| 239 | - | cursor.pending = append(entries, cursor.pending...) | |||
| 240 | - | } | |||
| 241 | - | ||||
| 242 | - | func (r *Repository) removeTemps(ctx context.Context, dir string, budget *int, purpose string) (bool, error) { | |||
| 243 | - | if *budget == 0 { | |||
| 244 | - | return false, nil | |||
| 245 | - | } | |||
| 246 | - | entries, done, err := r.readMaintenanceDir(dir, maintenanceBatch, purpose) | |||
| 247 | - | if errors.Is(err, os.ErrNotExist) { | |||
| 248 | - | return true, nil | |||
| 249 | - | } | |||
| 250 | - | if err != nil { | |||
| 251 | - | return false, fmt.Errorf("read snapshot maintenance directory: %w", safeFilesystemError(err)) | |||
| 252 | - | } | |||
| 253 | - | for i, entry := range entries { | |||
| 254 | - | if *budget == 0 { | |||
| 255 | - | r.requeueMaintenanceEntries(dir, purpose, entries[i:]) | |||
| 256 | - | return false, nil | |||
| 257 | - | } | |||
| 258 | - | if err := ctx.Err(); err != nil { | |||
| 259 | - | return false, err | |||
| 260 | - | } | |||
| 261 | - | if entry.isDir || !strings.HasPrefix(entry.name, ".tmp-") { | |||
| 262 | - | continue | |||
| 263 | - | } | |||
| 264 | - | path := filepath.Join(dir, entry.name) | |||
| 265 | - | if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { | |||
| 266 | - | return false, fmt.Errorf("remove snapshot maintenance file %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err)) | |||
| 267 | - | } | |||
| 268 | - | if err := ctx.Err(); err != nil { | |||
| 269 | - | return false, err | |||
| 270 | - | } | |||
| 271 | - | if err := r.syncDirectory(dir); err != nil { | |||
| 272 | - | return false, fmt.Errorf("sync snapshot maintenance directory for %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err)) | |||
| 273 | - | } | |||
| 274 | - | *budget-- | |||
| 275 | - | } | |||
| 276 | - | return done, nil | |||
| 277 | - | } | |||
| 278 | - | ||||
| 279 | - | // maintenancePath avoids exposing the repository root while retaining a | |||
M internal/adapters/snapshot/maintenance_delete.go +25 -169
| @@ -4,195 +4,51 @@ import ( | |||||
| 4 | "context" | 4 | "context" | ||
| 5 | "errors" | 5 | "errors" | ||
| 6 | "fmt" | 6 | "fmt" | ||
| 7 | - | "io" | |||
| 8 | "os" | 7 | "os" | ||
| 9 | "path/filepath" | 8 | "path/filepath" | ||
| 10 | - | "strings" | 9 | + | |
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 11 | ) | 11 | ) | ||
| 12 | 12 | ||||
| 13 | - | // Delete makes a session unavailable by durably moving it out of the canonical | 13 | + | // DeleteIncarnation removes one incarnation-keyed snapshot namespace. It is |
| 14 | - | // namespace. Maintain reaps the private quarantine later; Delete never restores | 14 | + | // retry-idempotent: an incarnation already absent is a successful deletion. |
| 15 | - | // a quarantined directory. | 15 | + | func (r *Repository) DeleteIncarnation(ctx context.Context, id domain.IncarnationID) error { |
| 16 | - | func (r *Repository) Delete(ctx context.Context, name string) error { | |||
| 17 | if err := ctx.Err(); err != nil { | 16 | if err := ctx.Err(); err != nil { | ||
| 18 | return err | 17 | return err | ||
| 19 | } | 18 | } | ||
| 20 | - | key := sessionKey(name) | 19 | + | key, err := incarnationKey(id) |
| 21 | - | lock := r.lockSession(key) | 20 | + | if err != nil { |
| 22 | - | defer r.unlockSession(lock) | |||
| 23 | - | if err := ctx.Err(); err != nil { | |||
| 24 | return err | 21 | return err | ||
| 25 | } | 22 | } | ||
| 23 | + | lock := r.lockSession(key) | |||
| 24 | + | defer r.unlockSession(lock) | |||
| 26 | 25 | ||||
| 27 | - | canonical := r.sessionPath(key) | 26 | + | path := r.sessionPath(id) |
| 28 | - | sessions := filepath.Dir(canonical) | 27 | + | if err := r.removeIncarnationLocked(id); err != nil { |
| 29 | - | // A prior rename may have succeeded while its parent sync failed. Complete | 28 | + | return fmt.Errorf("remove snapshot incarnation %s: %w", id.String(), safeFilesystemError(err)) |
| 30 | - | // that durability boundary before considering a canonical directory: this | |||
| 31 | - | // also leaves a newly recreated session untouched. | |||
| 32 | - | pending, err := r.pendingQuarantine(sessions, key) | |||
| 33 | - | if err != nil { | |||
| 34 | - | return fmt.Errorf("stat deleting snapshot session %q: %w", key, safeFilesystemError(err)) | |||
| 35 | } | 29 | } | ||
| 36 | - | if pending { | |||
| 37 | - | if err := ctx.Err(); err != nil { | |||
| 38 | - | return err | |||
| 39 | - | } | |||
| 40 | - | if err := r.syncDirectory(sessions); err != nil { | |||
| 41 | - | return fmt.Errorf("sync deleted snapshot session directory %q: %w", key, safeFilesystemError(err)) | |||
| 42 | - | } | |||
| 43 | - | return nil | |||
| 44 | - | } | |||
| 45 | - | if dir, err := r.openDirectory(canonical); errors.Is(err, os.ErrNotExist) { | |||
| 46 | - | return nil | |||
| 47 | - | } else if err != nil { | |||
| 48 | - | return fmt.Errorf("stat snapshot session %q: %w", key, safeFilesystemError(err)) | |||
| 49 | - | } else if err := dir.Close(); err != nil { | |||
| 50 | - | return fmt.Errorf("close snapshot session %q: %w", key, safeFilesystemError(err)) | |||
| 51 | - | } | |||
| 52 | - | quarantine := filepath.Join(sessions, deletingSessionName(key)) | |||
| 53 | if err := ctx.Err(); err != nil { | 30 | if err := ctx.Err(); err != nil { | ||
| 54 | return err | 31 | return err | ||
| 55 | } | 32 | } | ||
| 56 | - | // A renamed session can be replaced by a later publication. Invalidate any | 33 | + | parent := filepath.Dir(path) |
| 57 | - | // mark made for the old namespace before that replacement becomes possible. | 34 | + | if _, err := os.Lstat(parent); errors.Is(err, os.ErrNotExist) { |
| 58 | - | r.invalidateStorageEpoch(key) | 35 | + | return nil |
| 59 | - | if err := r.rename(canonical, quarantine); err != nil { | 36 | + | } else if err != nil { |
| 60 | - | return fmt.Errorf("quarantine snapshot session %q: %w", key, safeFilesystemError(err)) | |||
| 61 | - | } | |||
| 62 | - | if err := ctx.Err(); err != nil { | |||
| 63 | return err | 37 | return err | ||
| 64 | } | 38 | } | ||
| 65 | - | if err := r.syncDirectory(sessions); err != nil { | 39 | + | return r.syncDirectory(parent) |
| 66 | - | return fmt.Errorf("sync deleted snapshot session directory %q: %w", key, safeFilesystemError(err)) | |||
| 67 | - | } | |||
| 68 | - | return nil | |||
| 69 | } | 40 | } | ||
| 70 | 41 | ||||
| 71 | - | // deletingSessionName is a deterministic pending-delete record. Its presence | 42 | + | // removeIncarnationLocked removes an incarnation through a root-pinned path. |
| 72 | - | // is checked directly rather than scanning an attacker-controlled sessions | 43 | + | // The caller must hold the incarnation's session lock. |
| 73 | - | // directory, and it prevents a retry from deleting a session republished after | 44 | + | func (r *Repository) removeIncarnationLocked(id domain.IncarnationID) (err error) { |
| 74 | - | // the initial rename reached disk but its directory sync failed. | 45 | + | root, err := r.openRoot() |
| 75 | - | func deletingSessionName(key string) string { return ".deleting-" + key } | |||
| 76 | - | ||||
| 77 | - | func (r *Repository) pendingQuarantine(dir, key string) (bool, error) { | |||
| 78 | - | path := filepath.Join(dir, deletingSessionName(key)) | |||
| 79 | - | if hook := r.hooks.beforePendingQuarantineCheck; hook != nil { | |||
| 80 | - | hook(path) | |||
| 81 | - | } | |||
| 82 | - | file, err := r.openDirectory(path) | |||
| 83 | if errors.Is(err, os.ErrNotExist) { | 46 | if errors.Is(err, os.ErrNotExist) { | ||
| 84 | - | return false, nil | 47 | + | return nil |
| 85 | } | 48 | } | ||
| 86 | if err != nil { | 49 | if err != nil { | ||
| 87 | - | return false, err | 50 | + | return err |
| 88 | - | } | |||
| 89 | - | return true, file.Close() | |||
| 90 | - | } | |||
| 91 | - | ||||
| 92 | - | // Maintain reaps a bounded amount of stale state. Continuation handles are | |||
| 93 | - | // repository-owned, so a stable prefix cannot starve entries later in a large | |||
| 94 | - | // directory. Cancellation and errors discard those handles: a restarted | |||
| 95 | - | // repository (and the next fresh cycle) always begins at a safe boundary. | |||
| 96 | - | func isQuarantine(name string) bool { return strings.HasPrefix(name, ".deleting-") } | |||
| 97 | - | ||||
| 98 | - | // quarantineMaintenance retains one root-to-current path, never an unbounded | |||
| 99 | - | // stack of directory handles or names. Parent paths are recovered after a | |||
| 100 | - | // removal, while descent resumes from current on the next budgeted step. | |||
| 101 | - | type quarantineMaintenance struct { | |||
| 102 | - | root string | |||
| 103 | - | current string | |||
| 104 | - | } | |||
| 105 | - | ||||
| 106 | - | // consumeQuarantineWork charges every filesystem operation that can be driven | |||
| 107 | - | // by quarantine contents. This makes deep and wide trees subject to exactly | |||
| 108 | - | // the same maintenanceBatch ceiling as removal work. | |||
| 109 | - | func (r *Repository) consumeQuarantineWork(budget *int, operation string) bool { | |||
| 110 | - | if *budget == 0 { | |||
| 111 | - | return false | |||
| 112 | - | } | |||
| 113 | - | *budget-- | |||
| 114 | - | if hook := r.hooks.beforeMaintenanceWork; hook != nil { | |||
| 115 | - | hook(operation) | |||
| 116 | - | } | |||
| 117 | - | return true | |||
| 118 | - | } | |||
| 119 | - | ||||
| 120 | - | // maintainQuarantine makes one bounded depth-first traversal step at a time. | |||
| 121 | - | // It deliberately reopens a directory to select its first child: every | |||
| 122 | - | // successful child removal changes that prefix, so no directory cursor is | |||
| 123 | - | // retained at each hostile nesting level. | |||
| 124 | - | func (r *Repository) maintainQuarantine(ctx context.Context, budget *int) (changed, done bool, err error) { | |||
| 125 | - | state := r.maintenanceQuarantine | |||
| 126 | - | if state == nil { | |||
| 127 | - | return false, true, nil | |||
| 128 | - | } | |||
| 129 | - | for *budget > 0 { | |||
| 130 | - | if err := ctx.Err(); err != nil { | |||
| 131 | - | return changed, false, err | |||
| 132 | - | } | |||
| 133 | - | if !r.consumeQuarantineWork(budget, "stat") { | |||
| 134 | - | return changed, false, nil | |||
| 135 | - | } | |||
| 136 | - | stat, err := r.stat(state.current) | |||
| 137 | - | if errors.Is(err, os.ErrNotExist) { | |||
| 138 | - | if state.current == state.root { | |||
| 139 | - | return changed, true, nil | |||
| 140 | - | } | |||
| 141 | - | state.current = filepath.Dir(state.current) | |||
| 142 | - | continue | |||
| 143 | - | } | |||
| 144 | - | if err != nil { | |||
| 145 | - | return changed, false, err | |||
| 146 | - | } | |||
| 147 | - | if !stat.IsDir() { | |||
| 148 | - | if !r.consumeQuarantineWork(budget, "remove") { | |||
| 149 | - | return changed, false, nil | |||
| 150 | - | } | |||
| 151 | - | if err := r.remove(state.current); err != nil && !errors.Is(err, os.ErrNotExist) { | |||
| 152 | - | return changed, false, err | |||
| 153 | - | } | |||
| 154 | - | changed = true | |||
| 155 | - | state.current = filepath.Dir(state.current) | |||
| 156 | - | continue | |||
| 157 | - | } | |||
| 158 | - | ||||
| 159 | - | if !r.consumeQuarantineWork(budget, "open") { | |||
| 160 | - | return changed, false, nil | |||
| 161 | - | } | |||
| 162 | - | dir, err := r.openDirectory(state.current) | |||
| 163 | - | if err != nil { | |||
| 164 | - | return changed, false, err | |||
| 165 | - | } | |||
| 166 | - | if !r.consumeQuarantineWork(budget, "read") { | |||
| 167 | - | _ = dir.Close() | |||
| 168 | - | return changed, false, nil | |||
| 169 | - | } | |||
| 170 | - | entries, readErr := dir.ReadDir(1) | |||
| 171 | - | closeErr := dir.Close() | |||
| 172 | - | if readErr != nil && !errors.Is(readErr, io.EOF) { | |||
| 173 | - | return changed, false, readErr | |||
| 174 | - | } | |||
| 175 | - | if closeErr != nil { | |||
| 176 | - | return changed, false, closeErr | |||
| 177 | - | } | |||
| 178 | - | if len(entries) == 0 { | |||
| 179 | - | if !r.consumeQuarantineWork(budget, "remove") { | |||
| 180 | - | return changed, false, nil | |||
| 181 | - | } | |||
| 182 | - | if err := r.remove(state.current); err != nil && !errors.Is(err, os.ErrNotExist) { | |||
| 183 | - | return changed, false, err | |||
| 184 | - | } | |||
| 185 | - | changed = true | |||
| 186 | - | if state.current == state.root { | |||
| 187 | - | return changed, true, nil | |||
| 188 | - | } | |||
| 189 | - | state.current = filepath.Dir(state.current) | |||
| 190 | - | continue | |||
| 191 | - | } | |||
| 192 | - | if !r.consumeQuarantineWork(budget, "descend") { | |||
| 193 | - | return changed, false, nil | |||
| 194 | - | } | |||
| 195 | - | state.current = filepath.Join(state.current, entries[0].Name()) | |||
| 196 | } | 51 | } | ||
| 197 | - | return changed, false, nil | 52 | + | defer func() { joinCloseError(&err, "close snapshot root", r.closeRoot(root)) }() |
| 53 | + | return root.RemoveAll(filepath.Join(repositorySessionsDir, id.String())) | |||
| 198 | } | 54 | } | ||
D internal/adapters/snapshot/maintenance_error.go +0 -38
| @@ -1,38 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "errors" | |||
| 5 | - | "os" | |||
| 6 | - | "path/filepath" | |||
| 7 | - | "strings" | |||
| 8 | - | ) | |||
| 9 | - | ||||
| 10 | - | func maintenancePath(root, path string) string { | |||
| 11 | - | rel, err := filepath.Rel(root, path) | |||
| 12 | - | if err != nil || rel == "." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || rel == ".." { | |||
| 13 | - | return filepath.Base(path) | |||
| 14 | - | } | |||
| 15 | - | return rel | |||
| 16 | - | } | |||
| 17 | - | ||||
| 18 | - | type safePathError struct { | |||
| 19 | - | op string | |||
| 20 | - | cause error | |||
| 21 | - | } | |||
| 22 | - | ||||
| 23 | - | func (e safePathError) Error() string { return e.op + ": " + e.cause.Error() } | |||
| 24 | - | func (e safePathError) Unwrap() error { return e.cause } | |||
| 25 | - | ||||
| 26 | - | // safeFilesystemError preserves error matching without including a filesystem | |||
| 27 | - | // path from terminal-facing error text. | |||
| 28 | - | func safeFilesystemError(err error) error { | |||
| 29 | - | var linkError *os.LinkError | |||
| 30 | - | if errors.As(err, &linkError) { | |||
| 31 | - | return safePathError{op: linkError.Op, cause: linkError.Err} | |||
| 32 | - | } | |||
| 33 | - | var pathError *os.PathError | |||
| 34 | - | if errors.As(err, &pathError) { | |||
| 35 | - | return safePathError{op: pathError.Op, cause: pathError.Err} | |||
| 36 | - | } | |||
| 37 | - | return err | |||
| 38 | - | } | |||
M internal/adapters/snapshot/maintenance_gc.go +201 -328
| @@ -7,408 +7,281 @@ import ( | |||||
| 7 | "fmt" | 7 | "fmt" | ||
| 8 | "os" | 8 | "os" | ||
| 9 | "path/filepath" | 9 | "path/filepath" | ||
| 10 | - | "sort" | |||
| 11 | - | "strings" | |||
| 12 | 10 | ||||
| 11 | + | "github.com/bnema/vev/internal/domain" | |||
| 13 | "github.com/bnema/vev/internal/ports" | 12 | "github.com/bnema/vev/internal/ports" | ||
| 14 | codec "github.com/bnema/vev/internal/usecase/snapshot" | 13 | codec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 15 | ) | 14 | ) | ||
| 16 | 15 | ||||
| 17 | - | type manifestMaintenance struct { | 16 | + | // CollectGarbage applies the complete snapshot retention policy. The keep map |
| 18 | - | refs map[ports.SnapshotDigest]codec.ObjectRef | 17 | + | // must come from a catalogue that loaded and validated successfully: only then |
| 19 | - | complete bool | 18 | + | // is an incarnation absent from keep known to be an orphan. |
| 20 | - | } | 19 | + | func (r *Repository) CollectGarbage(ctx context.Context, keep map[domain.IncarnationID]domain.CheckpointRef) error { |
| 21 | - | ||||
| 22 | - | type sessionMaintenance struct { | |||
| 23 | - | lock *sessionMutex | |||
| 24 | - | token string | |||
| 25 | - | epoch uint64 | |||
| 26 | - | marked map[uint64]manifestMaintenance | |||
| 27 | - | referenceCount int | |||
| 28 | - | uncertain bool | |||
| 29 | - | conservative bool | |||
| 30 | - | markDone bool | |||
| 31 | - | manifestQueue []uint64 | |||
| 32 | - | objectTempShard string | |||
| 33 | - | objectTempsDone bool | |||
| 34 | - | sweepShard string | |||
| 35 | - | sweepRootDone bool | |||
| 36 | - | } | |||
| 37 | - | ||||
| 38 | - | // canRetainManifest reserves bounded state before retaining a manifest's | |||
| 39 | - | // reference map. The caller must leave the map unretained when it returns | |||
| 40 | - | // false, and make the cycle conservative instead. | |||
| 41 | - | func (s *sessionMaintenance) canRetainManifest(referenceCount int) bool { | |||
| 42 | - | if referenceCount < 0 || len(s.marked) >= maxMaintenanceMarkedGenerations { | |||
| 43 | - | return false | |||
| 44 | - | } | |||
| 45 | - | if referenceCount > maxMaintenanceReferences-s.referenceCount { | |||
| 46 | - | return false | |||
| 47 | - | } | |||
| 48 | - | s.referenceCount += referenceCount | |||
| 49 | - | return true | |||
| 50 | - | } | |||
| 51 | - | ||||
| 52 | - | func (r *Repository) maintainSession(ctx context.Context, key string, budget *int) error { | |||
| 53 | if err := ctx.Err(); err != nil { | 20 | if err := ctx.Err(); err != nil { | ||
| 54 | return err | 21 | return err | ||
| 55 | } | 22 | } | ||
| 56 | - | state, err := r.sessionMaintenanceState(key) | 23 | + | sessions := filepath.Join(r.dir, repositorySessionsDir) |
| 57 | - | if err != nil { | 24 | + | entries, err := r.readGarbageCollectionDirectory(sessions) |
| 58 | - | return err | 25 | + | if errors.Is(err, os.ErrNotExist) { |
| 26 | + | return nil | |||
| 59 | } | 27 | } | ||
| 60 | - | if _, err := r.removeTemps(ctx, filepath.Join(r.sessionPath(key), repositoryGenerations), budget, "generation-temps:"+key); err != nil || *budget == 0 { | 28 | + | if err != nil { |
| 61 | - | return err | 29 | + | return fmt.Errorf("read snapshot incarnations: %w", safeFilesystemError(err)) |
| 62 | } | 30 | } | ||
| 63 | - | if !state.objectTempsDone { | 31 | + | |
| 64 | - | done, err := r.removeObjectTemps(ctx, key, state, budget) | 32 | + | var collected error |
| 65 | - | if err != nil || !done || *budget == 0 { | 33 | + | for _, entry := range entries { |
| 66 | - | return err | 34 | + | if err := ctx.Err(); err != nil { |
| 35 | + | return errors.Join(collected, err) | |||
| 67 | } | 36 | } | ||
| 68 | - | } | 37 | + | if !entry.IsDir() { |
| 69 | - | if !state.markDone { | 38 | + | continue |
| 70 | - | if err := r.markSession(ctx, key, state); err != nil { | |||
| 71 | - | return err | |||
| 72 | } | 39 | } | ||
| 73 | - | if !state.markDone { | 40 | + | var id domain.IncarnationID |
| 74 | - | return nil | 41 | + | if err := id.UnmarshalText([]byte(entry.Name())); err != nil { |
| 42 | + | continue | |||
| 43 | + | } | |||
| 44 | + | committed, known := keep[id] | |||
| 45 | + | if !known { | |||
| 46 | + | key, keyErr := incarnationKey(id) | |||
| 47 | + | if keyErr != nil { | |||
| 48 | + | collected = errors.Join(collected, keyErr) | |||
| 49 | + | continue | |||
| 50 | + | } | |||
| 51 | + | lock := r.lockSession(key) | |||
| 52 | + | _, known = keep[id] | |||
| 53 | + | if !known { | |||
| 54 | + | err = r.removeIncarnationLocked(id) | |||
| 55 | + | } | |||
| 56 | + | r.unlockSession(lock) | |||
| 57 | + | if err != nil { | |||
| 58 | + | collected = errors.Join(collected, fmt.Errorf("remove orphan snapshot incarnation %s: %w", id.String(), safeFilesystemError(err))) | |||
| 59 | + | continue | |||
| 60 | + | } | |||
| 61 | + | if !known { | |||
| 62 | + | r.log.Info("snapshot_garbage_collected", "incarnation", id.String(), "action", "remove-incarnation") | |||
| 63 | + | } | |||
| 64 | + | continue | |||
| 65 | + | } | |||
| 66 | + | if err := r.pruneGenerations(ctx, id, committed); err != nil { | |||
| 67 | + | collected = errors.Join(collected, err) | |||
| 75 | } | 68 | } | ||
| 76 | } | 69 | } | ||
| 77 | - | if state.epoch != r.storageEpoch(key) { | 70 | + | if err := r.syncDirectory(sessions); err != nil { |
| 78 | - | // A publication may have left an unpublished manifest after this mark. | 71 | + | collected = errors.Join(collected, fmt.Errorf("sync snapshot sessions directory: %w", safeFilesystemError(err))) |
| 79 | - | // Discard all mark and sweep cursors; the next pass starts from it. | |||
| 80 | - | r.clearSessionMaintenance(key) | |||
| 81 | - | return nil | |||
| 82 | - | } | |||
| 83 | - | if state.uncertain { | |||
| 84 | - | // An unreadable, invalid, or unretained manifest may reference any | |||
| 85 | - | // existing blob. Do not collect manifests or sweep in this cycle; retry | |||
| 86 | - | // marking from a fresh directory pass. | |||
| 87 | - | r.clearSessionMaintenance(key) | |||
| 88 | - | return nil | |||
| 89 | - | } | |||
| 90 | - | if err := r.removeObsoleteManifests(ctx, key, state, budget); err != nil || *budget == 0 { | |||
| 91 | - | return err | |||
| 92 | - | } | |||
| 93 | - | if err := r.sweepSession(ctx, key, state, budget); err != nil { | |||
| 94 | - | return err | |||
| 95 | } | 72 | } | ||
| 96 | - | return nil | 73 | + | return collected |
| 97 | } | 74 | } | ||
| 98 | 75 | ||||
| 99 | - | func (r *Repository) sessionMaintenanceState(key string) (*sessionMaintenance, error) { | 76 | + | // pruneGenerations keeps the committed generation and its immediate |
| 100 | - | token, conservative, err := r.maintenanceToken(key) | 77 | + | // predecessor. Everything else, including a forward orphan newer than the |
| 78 | + | // catalogue commit, is removed before unreferenced objects are swept. | |||
| 79 | + | func (r *Repository) pruneGenerations(ctx context.Context, id domain.IncarnationID, committed domain.CheckpointRef) error { | |||
| 80 | + | key, err := incarnationKey(id) | |||
| 101 | if err != nil { | 81 | if err != nil { | ||
| 102 | - | return nil, err | 82 | + | return err |
| 103 | - | } | |||
| 104 | - | state := r.maintenanceSessions[key] | |||
| 105 | - | if state == nil || state.token != token || state.epoch != r.storageEpoch(key) { | |||
| 106 | - | r.clearSessionMaintenance(key) | |||
| 107 | - | state = &sessionMaintenance{ | |||
| 108 | - | lock: r.retainSessionState(key), | |||
| 109 | - | token: token, | |||
| 110 | - | epoch: r.storageEpoch(key), | |||
| 111 | - | conservative: conservative, | |||
| 112 | - | marked: make(map[uint64]manifestMaintenance), | |||
| 113 | - | } | |||
| 114 | - | r.maintenanceSessions[key] = state | |||
| 115 | } | 83 | } | ||
| 116 | - | return state, nil | 84 | + | lock := r.lockSession(key) |
| 117 | - | } | 85 | + | defer r.unlockSession(lock) |
| 118 | 86 | ||||
| 119 | - | // maintenanceToken is the publication boundary. A missing or corrupt HEAD is | 87 | + | generationsDir := filepath.Join(r.sessionPath(id), repositoryGenerations) |
| 120 | - | // itself stable maintenance state: we retain every classified reference until a | 88 | + | entries, err := r.readGarbageCollectionDirectory(generationsDir) |
| 121 | - | // valid publication changes that state, rather than restarting the mark pass. | 89 | + | if errors.Is(err, os.ErrNotExist) { |
| 122 | - | func (r *Repository) maintenanceToken(key string) (string, bool, error) { | 90 | + | return nil |
| 123 | - | data, exists, err := r.readOptionalBounded(r.headPath(key)) | |||
| 124 | - | if err != nil { | |||
| 125 | - | return "", false, err | |||
| 126 | - | } | |||
| 127 | - | if !exists { | |||
| 128 | - | return "missing", true, nil | |||
| 129 | } | 91 | } | ||
| 130 | - | sum := sha256.Sum256(data) | 92 | + | if err != nil { |
| 131 | - | if _, _, err := r.readHead(key); err != nil { | 93 | + | return fmt.Errorf("read snapshot generations for %s: %w", id.String(), safeFilesystemError(err)) |
| 132 | - | return fmt.Sprintf("invalid:%x", sum), true, nil | |||
| 133 | } | 94 | } | ||
| 134 | - | return fmt.Sprintf("valid:%x", sum), false, nil | |||
| 135 | - | } | |||
| 136 | 95 | ||||
| 137 | - | func (r *Repository) clearSessionMaintenance(key string) { | 96 | + | kept := make(map[uint64]struct{}, 2) |
| 138 | - | if state := r.maintenanceSessions[key]; state != nil && state.lock != nil { | 97 | + | if committed.Generation > 0 { |
| 139 | - | r.releaseSessionReference(state.lock) | 98 | + | kept[committed.Generation] = struct{}{} |
| 140 | } | 99 | } | ||
| 141 | - | delete(r.maintenanceSessions, key) | 100 | + | if committed.Generation > 1 { |
| 142 | - | prefix := "\x00" + filepath.Clean(r.sessionPath(key)) | 101 | + | kept[committed.Generation-1] = struct{}{} |
| 143 | - | for id := range r.maintenanceCursors { | |||
| 144 | - | if strings.HasSuffix(id, prefix) || strings.Contains(id, ":"+key+":") || strings.Contains(id, ":"+key+"\x00") { | |||
| 145 | - | delete(r.maintenanceCursors, id) | |||
| 146 | - | } | |||
| 147 | - | } | |||
| 148 | - | } | |||
| 149 | - | ||||
| 150 | - | func (r *Repository) markSession(ctx context.Context, key string, state *sessionMaintenance) error { | |||
| 151 | - | dir := filepath.Join(r.sessionPath(key), repositoryGenerations) | |||
| 152 | - | entries, done, err := r.readMaintenanceDir(dir, maintenanceBatch, "mark-generations:"+key) | |||
| 153 | - | if errors.Is(err, os.ErrNotExist) { | |||
| 154 | - | state.markDone = true | |||
| 155 | - | return nil | |||
| 156 | } | 102 | } | ||
| 103 | + | references, err := r.referencesForGenerations(id, entries, kept, committed) | |||
| 157 | if err != nil { | 104 | if err != nil { | ||
| 158 | - | return fmt.Errorf("read snapshot generations: %w", safeFilesystemError(err)) | 105 | + | return fmt.Errorf("mark snapshot objects for %s: %w", id.String(), err) |
| 159 | } | 106 | } | ||
| 107 | + | ||||
| 108 | + | removed := 0 | |||
| 160 | for _, entry := range entries { | 109 | for _, entry := range entries { | ||
| 161 | if err := ctx.Err(); err != nil { | 110 | if err := ctx.Err(); err != nil { | ||
| 162 | return err | 111 | return err | ||
| 163 | } | 112 | } | ||
| 164 | - | if entry.isDir { | 113 | + | generation, canonical := parseGenerationFilename(entry.Name()) |
| 165 | - | continue | 114 | + | if !canonical || entry.IsDir() { |
| 166 | - | } | |||
| 167 | - | generation, ok := parseGenerationFilename(entry.name) | |||
| 168 | - | if !ok { | |||
| 169 | - | continue | |||
| 170 | - | } | |||
| 171 | - | data, err := r.readBounded(r.manifestPath(key, generation)) | |||
| 172 | - | if err != nil { | |||
| 173 | - | state.uncertain = true | |||
| 174 | - | continue | |||
| 175 | - | } | |||
| 176 | - | manifest, err := codec.UnmarshalManifest(data) | |||
| 177 | - | if err != nil || manifest.Generation != generation || sessionKey(manifest.Name) != key { | |||
| 178 | - | state.uncertain = true | |||
| 179 | continue | 115 | continue | ||
| 180 | } | 116 | } | ||
| 181 | - | refs, validRefs := maintenanceManifestRefs(manifest) | 117 | + | if _, retain := kept[generation]; retain { |
| 182 | - | if !validRefs || !withinGenerationBudget(len(data), refs) || !state.canRetainManifest(len(refs)) { | |||
| 183 | - | state.uncertain = true | |||
| 184 | continue | 118 | continue | ||
| 185 | } | 119 | } | ||
| 186 | - | // Valid but incomplete generations are marked too. This protects blobs | 120 | + | if err := r.remove(filepath.Join(generationsDir, entry.Name())); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 187 | - | // written before a failed or in-flight publication reaches HEAD. | 121 | + | return fmt.Errorf("remove snapshot generation %d for %s: %w", generation, id.String(), safeFilesystemError(err)) |
| 188 | - | _, loadErr := r.loadGeneration(ctx, manifest.Name, key, generation) | |||
| 189 | - | state.marked[generation] = manifestMaintenance{refs: refs, complete: loadErr == nil} | |||
| 190 | - | } | |||
| 191 | - | if done { | |||
| 192 | - | state.markDone = true | |||
| 193 | - | if !state.conservative { | |||
| 194 | - | state.manifestQueue = retainedManifestQueue(state.marked) | |||
| 195 | } | 122 | } | ||
| 123 | + | removed++ | |||
| 196 | } | 124 | } | ||
| 197 | - | return nil | 125 | + | if removed > 0 { |
| 198 | - | } | 126 | + | if err := r.syncDirectory(generationsDir); err != nil { |
| 199 | - | 127 | + | return fmt.Errorf("sync snapshot generations for %s: %w", id.String(), safeFilesystemError(err)) | |
| 200 | - | // maintenanceManifestRefs applies the GC reference ceiling while building its | |||
| 201 | - | // map, rather than first materializing a hostile manifest's entire reference | |||
| 202 | - | // set. Conflicting references are invalid just as they are for publication. | |||
| 203 | - | func maintenanceManifestRefs(manifest codec.Manifest) (map[ports.SnapshotDigest]codec.ObjectRef, bool) { | |||
| 204 | - | refs := make(map[ports.SnapshotDigest]codec.ObjectRef) | |||
| 205 | - | add := func(ref codec.ObjectRef) bool { | |||
| 206 | - | if old, ok := refs[ref.Digest]; ok { | |||
| 207 | - | return old == ref | |||
| 208 | } | 128 | } | ||
| 209 | - | if len(refs) == maxMaintenanceReferences { | 129 | + | r.log.Info("snapshot_garbage_collected", "incarnation", id.String(), "action", "remove-generations", "removed", removed) |
| 210 | - | return false | |||
| 211 | - | } | |||
| 212 | - | refs[ref.Digest] = ref | |||
| 213 | - | return true | |||
| 214 | } | 130 | } | ||
| 215 | - | for _, tab := range manifest.Tabs { | 131 | + | if err := r.removeHeadForDiscardedGenerations(id, kept); err != nil { |
| 216 | - | for _, pane := range tab.Panes { | 132 | + | return err |
| 217 | - | for _, ref := range pane.Sealed { | |||
| 218 | - | if !add(ref) { | |||
| 219 | - | return nil, false | |||
| 220 | - | } | |||
| 221 | - | } | |||
| 222 | - | if !add(pane.Tail) || !add(pane.Visible) { | |||
| 223 | - | return nil, false | |||
| 224 | - | } | |||
| 225 | - | } | |||
| 226 | } | 133 | } | ||
| 227 | - | return refs, true | 134 | + | return r.sweepUnreferencedObjects(ctx, id, references) |
| 228 | } | 135 | } | ||
| 229 | 136 | ||||
| 230 | - | // completeGenerationsDescending returns only complete generations in canonical | 137 | + | // removeHeadForDiscardedGenerations removes a repository HEAD only when its |
| 231 | - | // newest-first order for both manifest and object retention. | 138 | + | // generation was removed by the retention policy. In particular, an |
| 232 | - | func completeGenerationsDescending(marked map[uint64]manifestMaintenance) []uint64 { | 139 | + | // incarnation with no committed checkpoint retains no generations, so its |
| 233 | - | complete := make([]uint64, 0, len(marked)) | 140 | + | // crash-published HEAD cannot make a retry conflict with an orphan. |
| 234 | - | for generation, item := range marked { | 141 | + | func (r *Repository) removeHeadForDiscardedGenerations(id domain.IncarnationID, kept map[uint64]struct{}) error { |
| 235 | - | if item.complete { | 142 | + | removeHead := len(kept) == 0 |
| 236 | - | complete = append(complete, generation) | 143 | + | if !removeHead { |
| 144 | + | generation, _, err := r.readHead(id) | |||
| 145 | + | if errors.Is(err, os.ErrNotExist) { | |||
| 146 | + | return nil | |||
| 147 | + | } | |||
| 148 | + | if err != nil { | |||
| 149 | + | return fmt.Errorf("read snapshot HEAD for %s: %w", id.String(), safeFilesystemError(err)) | |||
| 237 | } | 150 | } | ||
| 151 | + | _, retained := kept[generation] | |||
| 152 | + | removeHead = !retained | |||
| 238 | } | 153 | } | ||
| 239 | - | sort.Slice(complete, func(i, j int) bool { return complete[i] > complete[j] }) | 154 | + | if !removeHead { |
| 240 | - | return complete | |||
| 241 | - | } | |||
| 242 | - | ||||
| 243 | - | func retainedManifestQueue(marked map[uint64]manifestMaintenance) []uint64 { | |||
| 244 | - | complete := completeGenerationsDescending(marked) | |||
| 245 | - | if len(complete) <= 2 { | |||
| 246 | return nil | 155 | return nil | ||
| 247 | } | 156 | } | ||
| 248 | - | return complete[2:] | 157 | + | if err := r.remove(r.headPath(id)); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 249 | - | } | 158 | + | return fmt.Errorf("remove snapshot HEAD for %s: %w", id.String(), safeFilesystemError(err)) |
| 250 | - | 159 | + | } | |
| 251 | - | func (r *Repository) removeObsoleteManifests(ctx context.Context, key string, state *sessionMaintenance, budget *int) error { | 160 | + | if err := r.syncDirectory(r.sessionPath(id)); err != nil { |
| 252 | - | for len(state.manifestQueue) != 0 && *budget > 0 { | 161 | + | return fmt.Errorf("sync snapshot incarnation for %s: %w", id.String(), safeFilesystemError(err)) |
| 253 | - | if err := ctx.Err(); err != nil { | |||
| 254 | - | return err | |||
| 255 | - | } | |||
| 256 | - | generation := state.manifestQueue[0] | |||
| 257 | - | path := r.manifestPath(key, generation) | |||
| 258 | - | if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { | |||
| 259 | - | return fmt.Errorf("remove snapshot manifest %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err)) | |||
| 260 | - | } | |||
| 261 | - | if err := ctx.Err(); err != nil { | |||
| 262 | - | return err | |||
| 263 | - | } | |||
| 264 | - | if err := r.syncDirectory(filepath.Dir(path)); err != nil { | |||
| 265 | - | return fmt.Errorf("sync snapshot maintenance directory for %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err)) | |||
| 266 | - | } | |||
| 267 | - | state.manifestQueue = state.manifestQueue[1:] | |||
| 268 | - | *budget-- | |||
| 269 | } | 162 | } | ||
| 270 | return nil | 163 | return nil | ||
| 271 | } | 164 | } | ||
| 272 | 165 | ||||
| 273 | - | // removeObjectTemps performs at most one shard traversal or one root-entry | 166 | + | func (r *Repository) referencesForGenerations(id domain.IncarnationID, entries []os.DirEntry, keep map[uint64]struct{}, committed domain.CheckpointRef) (map[ports.SnapshotDigest]struct{}, error) { |
| 274 | - | // discovery per Maintain call. Empty attacker-controlled shards consume no | 167 | + | references := make(map[ports.SnapshotDigest]struct{}) |
| 275 | - | // removal budget, so this fixed step ceiling prevents an unbounded scan of | 168 | + | committedFound := committed.Generation == 0 |
| 276 | - | // them while retaining only the root cursor and one active shard cursor. | 169 | + | for _, entry := range entries { |
| 277 | - | func (r *Repository) removeObjectTemps(ctx context.Context, key string, state *sessionMaintenance, budget *int) (bool, error) { | 170 | + | generation, canonical := parseGenerationFilename(entry.Name()) |
| 278 | - | root := filepath.Join(r.sessionPath(key), repositoryObjectsDir) | 171 | + | if !canonical || entry.IsDir() { |
| 279 | - | if *budget == 0 { | 172 | + | continue |
| 280 | - | return false, nil | |||
| 281 | - | } | |||
| 282 | - | if state.objectTempShard != "" { | |||
| 283 | - | done, err := r.removeTemps(ctx, filepath.Join(root, state.objectTempShard), budget, "object-temps:"+key+":"+state.objectTempShard) | |||
| 284 | - | if err != nil || !done { | |||
| 285 | - | return false, err | |||
| 286 | } | 173 | } | ||
| 287 | - | state.objectTempShard = "" | 174 | + | if _, retained := keep[generation]; !retained { |
| 288 | - | return state.objectTempsDone, nil | 175 | + | continue |
| 289 | - | } | 176 | + | } |
| 290 | - | if state.objectTempsDone { | 177 | + | data, err := r.readBounded(r.manifestPath(id, generation)) |
| 291 | - | return true, nil | |||
| 292 | - | } | |||
| 293 | - | entries, done, err := r.readMaintenanceDir(root, 1, "object-temps-shards:"+key) | |||
| 294 | - | if errors.Is(err, os.ErrNotExist) { | |||
| 295 | - | state.objectTempsDone = true | |||
| 296 | - | return true, nil | |||
| 297 | - | } | |||
| 298 | - | if err != nil { | |||
| 299 | - | return false, fmt.Errorf("read snapshot object shards: %w", safeFilesystemError(err)) | |||
| 300 | - | } | |||
| 301 | - | state.objectTempsDone = done | |||
| 302 | - | if len(entries) != 0 && entries[0].isDir { | |||
| 303 | - | state.objectTempShard = entries[0].name | |||
| 304 | - | } | |||
| 305 | - | return state.objectTempsDone && state.objectTempShard == "", nil | |||
| 306 | - | } | |||
| 307 | - | ||||
| 308 | - | func (r *Repository) sweepSession(ctx context.Context, key string, state *sessionMaintenance, budget *int) error { | |||
| 309 | - | // Every bounded sweep batch is tied to the mark's storage epoch. Do not use | |||
| 310 | - | // stale references after any publication or replacement. | |||
| 311 | - | if state.epoch != r.storageEpoch(key) { | |||
| 312 | - | r.clearSessionMaintenance(key) | |||
| 313 | - | return nil | |||
| 314 | - | } | |||
| 315 | - | referenced := retainedReferences(state.marked, state.conservative) | |||
| 316 | - | root := filepath.Join(r.sessionPath(key), repositoryObjectsDir) | |||
| 317 | - | if state.sweepShard != "" && *budget > 0 { | |||
| 318 | - | shard := state.sweepShard | |||
| 319 | - | done, err := r.sweepShard(ctx, filepath.Join(root, shard), referenced, budget, key, shard) | |||
| 320 | if err != nil { | 178 | if err != nil { | ||
| 321 | - | return err | 179 | + | return nil, err |
| 322 | } | 180 | } | ||
| 323 | - | if !done { | 181 | + | manifest, err := codec.UnmarshalManifest(data) |
| 324 | - | return nil | 182 | + | if err != nil || manifest.IncarnationID != id || manifest.Generation != generation { |
| 183 | + | return nil, fmt.Errorf("invalid retained manifest generation %d", generation) | |||
| 325 | } | 184 | } | ||
| 326 | - | state.sweepShard = "" | 185 | + | if generation == committed.Generation { |
| 327 | - | // Release this shard's cursor before discovering another shard. | 186 | + | committedFound = true |
| 328 | - | return nil | 187 | + | if sha256.Sum256(data) != committed.ManifestDigest { |
| 329 | - | } | 188 | + | return nil, fmt.Errorf("committed manifest digest mismatch") |
| 330 | - | if !state.sweepRootDone { | |||
| 331 | - | entries, done, err := r.readMaintenanceDir(root, 1, "sweep-shards:"+key) | |||
| 332 | - | if errors.Is(err, os.ErrNotExist) { | |||
| 333 | - | state.sweepRootDone = true | |||
| 334 | - | } else if err != nil { | |||
| 335 | - | return fmt.Errorf("read snapshot objects: %w", safeFilesystemError(err)) | |||
| 336 | - | } else { | |||
| 337 | - | state.sweepRootDone = done | |||
| 338 | - | if len(entries) != 0 && entries[0].isDir { | |||
| 339 | - | state.sweepShard = entries[0].name | |||
| 340 | } | 189 | } | ||
| 341 | } | 190 | } | ||
| 191 | + | refs := manifestRefs(manifest) | |||
| 192 | + | if refs == nil || !withinGenerationBudget(len(data), refs) { | |||
| 193 | + | return nil, fmt.Errorf("invalid retained manifest references generation %d", generation) | |||
| 194 | + | } | |||
| 195 | + | for digest := range refs { | |||
| 196 | + | references[digest] = struct{}{} | |||
| 197 | + | } | |||
| 342 | } | 198 | } | ||
| 343 | - | if state.sweepRootDone && state.sweepShard == "" { | 199 | + | if !committedFound { |
| 344 | - | r.clearSessionMaintenance(key) | 200 | + | return nil, fmt.Errorf("committed manifest generation %d is missing", committed.Generation) |
| 345 | } | 201 | } | ||
| 346 | - | return nil | 202 | + | return references, nil |
| 347 | } | 203 | } | ||
| 348 | 204 | ||||
| 349 | - | func retainedReferences(marked map[uint64]manifestMaintenance, conservative bool) map[ports.SnapshotDigest]struct{} { | 205 | + | type safePathError struct { |
| 350 | - | if conservative { | 206 | + | op string |
| 351 | - | references := make(map[ports.SnapshotDigest]struct{}) | 207 | + | cause error |
| 352 | - | for _, item := range marked { | 208 | + | } |
| 353 | - | for digest := range item.refs { | 209 | + | |
| 354 | - | references[digest] = struct{}{} | 210 | + | func (e safePathError) Error() string { return e.op + ": " + e.cause.Error() } |
| 355 | - | } | 211 | + | func (e safePathError) Unwrap() error { return e.cause } |
| 356 | - | } | 212 | + | |
| 357 | - | return references | 213 | + | // safeFilesystemError preserves error matching without exposing repository |
| 358 | - | } | 214 | + | // paths in terminal-facing errors. |
| 359 | - | complete := completeGenerationsDescending(marked) | 215 | + | func safeFilesystemError(err error) error { |
| 360 | - | keep := make(map[uint64]bool, 2) | 216 | + | var linkError *os.LinkError |
| 361 | - | for _, generation := range complete { | 217 | + | if errors.As(err, &linkError) { |
| 362 | - | if len(keep) == 2 { | 218 | + | return safePathError{op: linkError.Op, cause: linkError.Err} |
| 363 | - | break | 219 | + | } |
| 364 | - | } | 220 | + | var pathError *os.PathError |
| 365 | - | keep[generation] = true | 221 | + | if errors.As(err, &pathError) { |
| 222 | + | return safePathError{op: pathError.Op, cause: pathError.Err} | |||
| 223 | + | } | |||
| 224 | + | return err | |||
| 225 | + | } | |||
| 226 | + | ||||
| 227 | + | func (r *Repository) readGarbageCollectionDirectory(path string) (entries []os.DirEntry, err error) { | |||
| 228 | + | directory, err := r.openDirectory(path) | |||
| 229 | + | if err != nil { | |||
| 230 | + | return nil, err | |||
| 366 | } | 231 | } | ||
| 367 | - | references := make(map[ports.SnapshotDigest]struct{}) | 232 | + | defer func() { |
| 368 | - | for generation, item := range marked { | 233 | + | if closeErr := directory.Close(); closeErr != nil { |
| 369 | - | if !item.complete || keep[generation] { | 234 | + | err = errors.Join(err, closeErr) |
| 370 | - | for digest := range item.refs { | |||
| 371 | - | references[digest] = struct{}{} | |||
| 372 | - | } | |||
| 373 | } | 235 | } | ||
| 374 | - | } | 236 | + | }() |
| 375 | - | return references | 237 | + | return directory.ReadDir(-1) |
| 376 | } | 238 | } | ||
| 377 | 239 | ||||
| 378 | - | func (r *Repository) sweepShard(ctx context.Context, dir string, referenced map[ports.SnapshotDigest]struct{}, budget *int, key, shard string) (bool, error) { | 240 | + | func (r *Repository) sweepUnreferencedObjects(ctx context.Context, id domain.IncarnationID, references map[ports.SnapshotDigest]struct{}) error { |
| 379 | - | entries, done, err := r.readMaintenanceDir(dir, maintenanceBatch, "sweep-objects:"+key+":"+shard) | 241 | + | objectsRoot := filepath.Join(r.sessionPath(id), repositoryObjectsDir) |
| 242 | + | shards, err := r.readGarbageCollectionDirectory(objectsRoot) | |||
| 380 | if errors.Is(err, os.ErrNotExist) { | 243 | if errors.Is(err, os.ErrNotExist) { | ||
| 381 | - | return true, nil | 244 | + | return nil |
| 382 | } | 245 | } | ||
| 383 | if err != nil { | 246 | if err != nil { | ||
| 384 | - | return false, fmt.Errorf("read snapshot object shard: %w", safeFilesystemError(err)) | 247 | + | return fmt.Errorf("read snapshot objects for %s: %w", id.String(), safeFilesystemError(err)) |
| 385 | } | 248 | } | ||
| 386 | - | for i, object := range entries { | 249 | + | removed := 0 |
| 387 | - | if *budget == 0 { | 250 | + | for _, shard := range shards { |
| 388 | - | r.requeueMaintenanceEntries(dir, "sweep-objects:"+key+":"+shard, entries[i:]) | |||
| 389 | - | return false, nil | |||
| 390 | - | } | |||
| 391 | if err := ctx.Err(); err != nil { | 251 | if err := ctx.Err(); err != nil { | ||
| 392 | - | return false, err | 252 | + | return err |
| 393 | - | } | |||
| 394 | - | digest, ok := parseObjectDigest(object.name) | |||
| 395 | - | if !ok || object.isDir { | |||
| 396 | - | continue | |||
| 397 | } | 253 | } | ||
| 398 | - | if _, used := referenced[digest]; used { | 254 | + | if !shard.IsDir() || len(shard.Name()) != 2 { |
| 399 | continue | 255 | continue | ||
| 400 | } | 256 | } | ||
| 401 | - | path := filepath.Join(dir, object.name) | 257 | + | dir := filepath.Join(objectsRoot, shard.Name()) |
| 402 | - | if err := r.remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { | 258 | + | objects, err := r.readGarbageCollectionDirectory(dir) |
| 403 | - | return false, fmt.Errorf("remove snapshot object %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err)) | 259 | + | if err != nil { |
| 260 | + | return fmt.Errorf("read snapshot object shard for %s: %w", id.String(), safeFilesystemError(err)) | |||
| 404 | } | 261 | } | ||
| 405 | - | if err := ctx.Err(); err != nil { | 262 | + | shardRemoved := false |
| 406 | - | return false, err | 263 | + | for _, object := range objects { |
| 264 | + | digest, canonical := parseObjectDigest(object.Name()) | |||
| 265 | + | if !canonical || object.IsDir() { | |||
| 266 | + | continue | |||
| 267 | + | } | |||
| 268 | + | if _, retained := references[digest]; retained { | |||
| 269 | + | continue | |||
| 270 | + | } | |||
| 271 | + | if err := r.remove(filepath.Join(dir, object.Name())); err != nil && !errors.Is(err, os.ErrNotExist) { | |||
| 272 | + | return fmt.Errorf("remove snapshot object for %s: %w", id.String(), safeFilesystemError(err)) | |||
| 273 | + | } | |||
| 274 | + | removed++ | |||
| 275 | + | shardRemoved = true | |||
| 407 | } | 276 | } | ||
| 408 | - | if err := r.syncDirectory(dir); err != nil { | 277 | + | if shardRemoved { |
| 409 | - | return false, fmt.Errorf("sync snapshot maintenance directory for %q: %w", maintenancePath(r.dir, path), safeFilesystemError(err)) | 278 | + | if err := r.syncDirectory(dir); err != nil { |
| 279 | + | return fmt.Errorf("sync snapshot object shard for %s: %w", id.String(), safeFilesystemError(err)) | |||
| 280 | + | } | |||
| 410 | } | 281 | } | ||
| 411 | - | *budget-- | |||
| 412 | } | 282 | } | ||
| 413 | - | return done, nil | 283 | + | if removed > 0 { |
| 284 | + | r.log.Info("snapshot_garbage_collected", "incarnation", id.String(), "action", "remove-objects", "removed", removed) | |||
| 285 | + | } | |||
| 286 | + | return nil | |||
| 414 | } | 287 | } | ||
A internal/adapters/snapshot/maintenance_gc_test.go +157 -0
| @@ -0,0 +1,157 @@ | |||||
| 1 | + | package snapshot | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "crypto/sha256" | |||
| 6 | + | "os" | |||
| 7 | + | "path/filepath" | |||
| 8 | + | "slices" | |||
| 9 | + | "testing" | |||
| 10 | + | ||||
| 11 | + | "github.com/bnema/vev/internal/domain" | |||
| 12 | + | "github.com/bnema/vev/internal/ports" | |||
| 13 | + | "github.com/stretchr/testify/require" | |||
| 14 | + | ) | |||
| 15 | + | ||||
| 16 | + | func TestCollectGarbageClearsUncommittedCheckpointHead(t *testing.T) { | |||
| 17 | + | t.Parallel() | |||
| 18 | + | repository := NewRepository(privateDir(t)) | |||
| 19 | + | publication := repositoryPublication(t, "work", 1, []byte("first checkpoint")) | |||
| 20 | + | require.NoError(t, repository.Publish(context.Background(), publication)) | |||
| 21 | + | ||||
| 22 | + | // Repository publication completed, but the catalogue commit did not. | |||
| 23 | + | // Startup GC knows the incarnation but has no committed checkpoint for it. | |||
| 24 | + | keep := map[domain.IncarnationID]domain.CheckpointRef{publication.IncarnationID: {}} | |||
| 25 | + | require.NoError(t, repository.CollectGarbage(context.Background(), keep)) | |||
| 26 | + | _, err := os.Stat(repository.headPath(publication.IncarnationID)) | |||
| 27 | + | require.ErrorIs(t, err, os.ErrNotExist) | |||
| 28 | + | require.Empty(t, garbageCollectionGenerations(t, repository, publication.IncarnationID)) | |||
| 29 | + | ||||
| 30 | + | // Retrying starts from an empty repository, then the resulting checkpoint | |||
| 31 | + | // can be committed to the catalogue and restored. | |||
| 32 | + | require.NoError(t, repository.Publish(context.Background(), publication)) | |||
| 33 | + | committed := domain.CheckpointRef{Generation: publication.Generation, ManifestDigest: sha256.Sum256(publication.Manifest)} | |||
| 34 | + | keep[publication.IncarnationID] = committed | |||
| 35 | + | require.NoError(t, repository.CollectGarbage(context.Background(), keep)) | |||
| 36 | + | _, err = repository.LoadCheckpoint(context.Background(), publication.IncarnationID, publication.Name, committed) | |||
| 37 | + | require.NoError(t, err) | |||
| 38 | + | } | |||
| 39 | + | ||||
| 40 | + | func TestCollectGarbageRejectsMalformedAuthorityWithoutMutation(t *testing.T) { | |||
| 41 | + | for _, tt := range []struct { | |||
| 42 | + | name string | |||
| 43 | + | mutate func(*testing.T, *Repository, domain.IncarnationID, domain.CheckpointRef) domain.CheckpointRef | |||
| 44 | + | }{ | |||
| 45 | + | { | |||
| 46 | + | name: "committed digest mismatch", | |||
| 47 | + | mutate: func(_ *testing.T, _ *Repository, _ domain.IncarnationID, committed domain.CheckpointRef) domain.CheckpointRef { | |||
| 48 | + | committed.ManifestDigest[0]++ | |||
| 49 | + | return committed | |||
| 50 | + | }, | |||
| 51 | + | }, | |||
| 52 | + | { | |||
| 53 | + | name: "committed manifest missing", | |||
| 54 | + | mutate: func(t *testing.T, repository *Repository, id domain.IncarnationID, committed domain.CheckpointRef) domain.CheckpointRef { | |||
| 55 | + | t.Helper() | |||
| 56 | + | require.NoError(t, os.Remove(repository.manifestPath(id, committed.Generation))) | |||
| 57 | + | return committed | |||
| 58 | + | }, | |||
| 59 | + | }, | |||
| 60 | + | { | |||
| 61 | + | name: "committed manifest corrupt", | |||
| 62 | + | mutate: func(t *testing.T, repository *Repository, id domain.IncarnationID, committed domain.CheckpointRef) domain.CheckpointRef { | |||
| 63 | + | t.Helper() | |||
| 64 | + | require.NoError(t, os.WriteFile(repository.manifestPath(id, committed.Generation), []byte("corrupt"), 0o600)) | |||
| 65 | + | return committed | |||
| 66 | + | }, | |||
| 67 | + | }, | |||
| 68 | + | } { | |||
| 69 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 70 | + | repository := NewRepository(privateDir(t)) | |||
| 71 | + | publications := publishGarbageCollectionGenerations(t, repository, "work", 3) | |||
| 72 | + | id := publications[0].IncarnationID | |||
| 73 | + | committed := domain.CheckpointRef{Generation: 3, ManifestDigest: sha256.Sum256(publications[2].Manifest)} | |||
| 74 | + | committed = tt.mutate(t, repository, id, committed) | |||
| 75 | + | before := snapshotRepositoryFiles(t, repository.dir) | |||
| 76 | + | ||||
| 77 | + | require.Error(t, repository.CollectGarbage(t.Context(), map[domain.IncarnationID]domain.CheckpointRef{id: committed})) | |||
| 78 | + | require.Equal(t, before, snapshotRepositoryFiles(t, repository.dir), "malformed catalogue authority must fail closed") | |||
| 79 | + | }) | |||
| 80 | + | } | |||
| 81 | + | } | |||
| 82 | + | ||||
| 83 | + | func TestCollectGarbage(t *testing.T) { | |||
| 84 | + | t.Parallel() | |||
| 85 | + | tests := []struct { | |||
| 86 | + | name string | |||
| 87 | + | committed uint64 | |||
| 88 | + | known bool | |||
| 89 | + | wantGenerations []uint64 | |||
| 90 | + | }{ | |||
| 91 | + | {name: "incarnation absent from catalogue is removed whole"}, | |||
| 92 | + | {name: "committed checkpoint prunes forward orphan and keeps predecessor", committed: 3, known: true, wantGenerations: []uint64{2, 3}}, | |||
| 93 | + | } | |||
| 94 | + | for _, tt := range tests { | |||
| 95 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 96 | + | t.Parallel() | |||
| 97 | + | repository := NewRepository(privateDir(t)) | |||
| 98 | + | publications := publishGarbageCollectionGenerations(t, repository, "work", 4) | |||
| 99 | + | id := publications[0].IncarnationID | |||
| 100 | + | keep := make(map[domain.IncarnationID]domain.CheckpointRef) | |||
| 101 | + | if tt.known { | |||
| 102 | + | publication := publications[tt.committed-1] | |||
| 103 | + | keep[id] = domain.CheckpointRef{Generation: tt.committed, ManifestDigest: sha256.Sum256(publication.Manifest)} | |||
| 104 | + | } | |||
| 105 | + | ||||
| 106 | + | require.NoError(t, repository.CollectGarbage(context.Background(), keep)) | |||
| 107 | + | if !tt.known { | |||
| 108 | + | _, err := os.Stat(repository.sessionPath(id)) | |||
| 109 | + | require.ErrorIs(t, err, os.ErrNotExist) | |||
| 110 | + | return | |||
| 111 | + | } | |||
| 112 | + | require.Equal(t, tt.wantGenerations, garbageCollectionGenerations(t, repository, id)) | |||
| 113 | + | retainedObjects := make(map[ports.SnapshotDigest]struct{}) | |||
| 114 | + | for _, publication := range publications[tt.committed-2 : tt.committed] { | |||
| 115 | + | for _, object := range publication.Objects { | |||
| 116 | + | retainedObjects[object.Digest] = struct{}{} | |||
| 117 | + | } | |||
| 118 | + | } | |||
| 119 | + | for _, publication := range publications { | |||
| 120 | + | for _, object := range publication.Objects { | |||
| 121 | + | _, retained := retainedObjects[object.Digest] | |||
| 122 | + | _, err := os.Stat(repository.objectPath(id, object.Digest)) | |||
| 123 | + | if retained { | |||
| 124 | + | require.NoError(t, err) | |||
| 125 | + | } else { | |||
| 126 | + | require.ErrorIs(t, err, os.ErrNotExist) | |||
| 127 | + | } | |||
| 128 | + | } | |||
| 129 | + | } | |||
| 130 | + | }) | |||
| 131 | + | } | |||
| 132 | + | } | |||
| 133 | + | ||||
| 134 | + | func publishGarbageCollectionGenerations(t *testing.T, repository *Repository, name string, count uint64) []ports.SnapshotPublication { | |||
| 135 | + | t.Helper() | |||
| 136 | + | publications := make([]ports.SnapshotPublication, 0, count) | |||
| 137 | + | for generation := uint64(1); generation <= count; generation++ { | |||
| 138 | + | publication := repositoryPublicationAfter(t, repository, name, generation, []byte{byte(generation)}) | |||
| 139 | + | require.NoError(t, repository.Publish(context.Background(), publication)) | |||
| 140 | + | publications = append(publications, publication) | |||
| 141 | + | } | |||
| 142 | + | return publications | |||
| 143 | + | } | |||
| 144 | + | ||||
| 145 | + | func garbageCollectionGenerations(t *testing.T, repository *Repository, id domain.IncarnationID) []uint64 { | |||
| 146 | + | t.Helper() | |||
| 147 | + | entries, err := os.ReadDir(filepath.Join(repository.sessionPath(id), repositoryGenerations)) | |||
| 148 | + | require.NoError(t, err) | |||
| 149 | + | generations := make([]uint64, 0, len(entries)) | |||
| 150 | + | for _, entry := range entries { | |||
| 151 | + | if generation, ok := parseGenerationFilename(entry.Name()); ok { | |||
| 152 | + | generations = append(generations, generation) | |||
| 153 | + | } | |||
| 154 | + | } | |||
| 155 | + | slices.Sort(generations) | |||
| 156 | + | return generations | |||
| 157 | + | } | |||
D internal/adapters/snapshot/maintenance_hardening_test.go +0 -141
| @@ -1,141 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "errors" | |||
| 6 | - | "fmt" | |||
| 7 | - | "os" | |||
| 8 | - | "path/filepath" | |||
| 9 | - | "testing" | |||
| 10 | - | ) | |||
| 11 | - | ||||
| 12 | - | func TestRepositoryMaintainBoundsCursorsAcrossHostileObjectShards(t *testing.T) { | |||
| 13 | - | repo := NewRepository(privateDir(t)) | |||
| 14 | - | key := sessionKey("named") | |||
| 15 | - | objects := filepath.Join(repo.sessionPath(key), repositoryObjectsDir) | |||
| 16 | - | const shards = maintenanceBatch * 2 | |||
| 17 | - | for i := 0; i < shards; i++ { | |||
| 18 | - | shard := filepath.Join(objects, fmt.Sprintf("%03d", i)) | |||
| 19 | - | if err := os.MkdirAll(shard, 0o700); err != nil { | |||
| 20 | - | t.Fatal(err) | |||
| 21 | - | } | |||
| 22 | - | if err := os.WriteFile(filepath.Join(shard, ".tmp-stale"), []byte("stale"), 0o600); err != nil { | |||
| 23 | - | t.Fatal(err) | |||
| 24 | - | } | |||
| 25 | - | // Force a resumable cursor in every attacker-controlled shard. | |||
| 26 | - | for j := 0; j < maintenanceBatch+1; j++ { | |||
| 27 | - | if err := os.WriteFile(filepath.Join(shard, fmt.Sprintf("live-%03d", j)), []byte("live"), 0o600); err != nil { | |||
| 28 | - | t.Fatal(err) | |||
| 29 | - | } | |||
| 30 | - | } | |||
| 31 | - | } | |||
| 32 | - | ||||
| 33 | - | for pass := 0; pass < shards*4; pass++ { | |||
| 34 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 35 | - | t.Fatal(err) | |||
| 36 | - | } | |||
| 37 | - | if got := len(repo.maintenanceCursors); got > maxMaintenanceCursors { | |||
| 38 | - | t.Fatalf("retained maintenance cursors = %d, want at most %d", got, maxMaintenanceCursors) | |||
| 39 | - | } | |||
| 40 | - | } | |||
| 41 | - | for i := 0; i < shards; i++ { | |||
| 42 | - | _, err := os.Lstat(filepath.Join(objects, fmt.Sprintf("%03d", i), ".tmp-stale")) | |||
| 43 | - | if !errors.Is(err, os.ErrNotExist) { | |||
| 44 | - | t.Fatalf("stale temporary object in shard %d = %v, want removed", i, err) | |||
| 45 | - | } | |||
| 46 | - | } | |||
| 47 | - | } | |||
| 48 | - | ||||
| 49 | - | func TestRepositoryMaintainBoundsEmptyObjectTempShards(t *testing.T) { | |||
| 50 | - | repo := NewRepository(privateDir(t)) | |||
| 51 | - | key := sessionKey("named") | |||
| 52 | - | objects := filepath.Join(repo.sessionPath(key), repositoryObjectsDir) | |||
| 53 | - | const shards = maintenanceBatch * 2 | |||
| 54 | - | for i := 0; i < shards; i++ { | |||
| 55 | - | if err := os.MkdirAll(filepath.Join(objects, fmt.Sprintf("%03d", i)), 0o700); err != nil { | |||
| 56 | - | t.Fatal(err) | |||
| 57 | - | } | |||
| 58 | - | } | |||
| 59 | - | ||||
| 60 | - | openedShards := make(map[string]struct{}, shards) | |||
| 61 | - | opensThisPass := 0 | |||
| 62 | - | tempPass := false | |||
| 63 | - | repo.hooks.openMaintenanceDirectory = func(dir string) (maintenanceDirectory, error) { | |||
| 64 | - | if tempPass && filepath.Dir(dir) == objects { | |||
| 65 | - | opensThisPass++ | |||
| 66 | - | openedShards[dir] = struct{}{} | |||
| 67 | - | } | |||
| 68 | - | file, err := os.Open(dir) | |||
| 69 | - | if err != nil { | |||
| 70 | - | return nil, err | |||
| 71 | - | } | |||
| 72 | - | return osMaintenanceDirectory{file: file}, nil | |||
| 73 | - | } | |||
| 74 | - | ||||
| 75 | - | for pass := 0; pass < shards*3; pass++ { | |||
| 76 | - | opensThisPass = 0 | |||
| 77 | - | tempPass = true | |||
| 78 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 79 | - | t.Fatal(err) | |||
| 80 | - | } | |||
| 81 | - | tempPass = false | |||
| 82 | - | if opensThisPass > 1 { | |||
| 83 | - | t.Fatalf("empty object shards opened in maintenance pass %d = %d, want at most 1", pass, opensThisPass) | |||
| 84 | - | } | |||
| 85 | - | state := repo.maintenanceSessions[key] | |||
| 86 | - | if state != nil && state.objectTempsDone { | |||
| 87 | - | if got := len(openedShards); got != shards { | |||
| 88 | - | t.Fatalf("empty object shards opened = %d, want %d", got, shards) | |||
| 89 | - | } | |||
| 90 | - | return | |||
| 91 | - | } | |||
| 92 | - | } | |||
| 93 | - | t.Fatal("empty object temp shards were not eventually traversed") | |||
| 94 | - | } | |||
| 95 | - | ||||
| 96 | - | func TestRepositoryMaintainBoundsAndResumesDeepWideQuarantine(t *testing.T) { | |||
| 97 | - | repo := NewRepository(privateDir(t)) | |||
| 98 | - | quarantine := filepath.Join(repo.dir, repositorySessionsDir, ".deleting-hostile") | |||
| 99 | - | deep := quarantine | |||
| 100 | - | for i := 0; i < maintenanceBatch*2; i++ { | |||
| 101 | - | deep = filepath.Join(deep, "d") | |||
| 102 | - | } | |||
| 103 | - | leaf := filepath.Join(deep, "leaf") | |||
| 104 | - | if err := os.MkdirAll(filepath.Dir(leaf), 0o700); err != nil { | |||
| 105 | - | t.Fatal(err) | |||
| 106 | - | } | |||
| 107 | - | if err := os.WriteFile(leaf, []byte("stale"), 0o600); err != nil { | |||
| 108 | - | t.Fatal(err) | |||
| 109 | - | } | |||
| 110 | - | for i := 0; i < maintenanceBatch*2; i++ { | |||
| 111 | - | if err := os.WriteFile(filepath.Join(quarantine, fmt.Sprintf("wide-%03d", i)), []byte("stale"), 0o600); err != nil { | |||
| 112 | - | t.Fatal(err) | |||
| 113 | - | } | |||
| 114 | - | } | |||
| 115 | - | ||||
| 116 | - | work := 0 | |||
| 117 | - | repo.hooks.beforeMaintenanceWork = func(string) { work++ } | |||
| 118 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 119 | - | t.Fatal(err) | |||
| 120 | - | } | |||
| 121 | - | if work == 0 || work > maintenanceBatch { | |||
| 122 | - | t.Fatalf("quarantine work in one maintenance call = %d, want 1..%d", work, maintenanceBatch) | |||
| 123 | - | } | |||
| 124 | - | if _, err := os.Lstat(leaf); err != nil { | |||
| 125 | - | t.Fatalf("deep quarantine leaf removed despite bounded traversal: %v", err) | |||
| 126 | - | } | |||
| 127 | - | ||||
| 128 | - | for pass := 0; pass < 1000; pass++ { | |||
| 129 | - | work = 0 | |||
| 130 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 131 | - | t.Fatal(err) | |||
| 132 | - | } | |||
| 133 | - | if work > maintenanceBatch { | |||
| 134 | - | t.Fatalf("quarantine work in pass %d = %d, want at most %d", pass, work, maintenanceBatch) | |||
| 135 | - | } | |||
| 136 | - | if _, err := os.Lstat(quarantine); errors.Is(err, os.ErrNotExist) { | |||
| 137 | - | return | |||
| 138 | - | } | |||
| 139 | - | } | |||
| 140 | - | t.Fatal("deep and wide quarantine was not eventually removed") | |||
| 141 | - | } | |||
D internal/adapters/snapshot/maintenance_test.go +0 -1010
| @@ -1,1010 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "crypto/sha256" | |||
| 6 | - | "encoding/binary" | |||
| 7 | - | "errors" | |||
| 8 | - | "fmt" | |||
| 9 | - | "io" | |||
| 10 | - | "os" | |||
| 11 | - | "path/filepath" | |||
| 12 | - | "slices" | |||
| 13 | - | "strings" | |||
| 14 | - | "syscall" | |||
| 15 | - | "testing" | |||
| 16 | - | "unsafe" | |||
| 17 | - | ||||
| 18 | - | "github.com/bnema/vev/internal/ports" | |||
| 19 | - | ) | |||
| 20 | - | ||||
| 21 | - | func TestReadMaintenanceDirentPreservesCursorAndCloseFailuresWithoutPaths(t *testing.T) { | |||
| 22 | - | privatePath := filepath.Join(t.TempDir(), "private-maintenance-directory") | |||
| 23 | - | closeCause := errors.New("injected close failure") | |||
| 24 | - | seekCause := errors.New("injected seek failure") | |||
| 25 | - | readCause := errors.New("injected read failure") | |||
| 26 | - | ||||
| 27 | - | for _, tc := range []struct { | |||
| 28 | - | name string | |||
| 29 | - | offset int64 | |||
| 30 | - | seekErr error | |||
| 31 | - | readErr error | |||
| 32 | - | wantCause []error | |||
| 33 | - | wantOps []string | |||
| 34 | - | }{ | |||
| 35 | - | { | |||
| 36 | - | name: "close only", | |||
| 37 | - | wantCause: []error{closeCause}, | |||
| 38 | - | wantOps: []string{"close maintenance directory"}, | |||
| 39 | - | }, | |||
| 40 | - | { | |||
| 41 | - | name: "seek and close", | |||
| 42 | - | offset: 1, | |||
| 43 | - | seekErr: &os.PathError{Op: "seek", Path: privatePath, Err: seekCause}, | |||
| 44 | - | wantCause: []error{seekCause, closeCause}, | |||
| 45 | - | wantOps: []string{"seek maintenance directory", "close maintenance directory"}, | |||
| 46 | - | }, | |||
| 47 | - | { | |||
| 48 | - | name: "read and close", | |||
| 49 | - | readErr: &os.PathError{Op: "read", Path: privatePath, Err: readCause}, | |||
| 50 | - | wantCause: []error{readCause, closeCause}, | |||
| 51 | - | wantOps: []string{"read maintenance directory", "close maintenance directory"}, | |||
| 52 | - | }, | |||
| 53 | - | } { | |||
| 54 | - | t.Run(tc.name, func(t *testing.T) { | |||
| 55 | - | repo := NewRepository(privateDir(t)) | |||
| 56 | - | repo.hooks.openMaintenanceDirectory = func(string) (maintenanceDirectory, error) { | |||
| 57 | - | return fakeMaintenanceDirectory{ | |||
| 58 | - | seekErr: tc.seekErr, | |||
| 59 | - | readErr: tc.readErr, | |||
| 60 | - | closeErr: &os.PathError{Op: "close", Path: privatePath, Err: closeCause}, | |||
| 61 | - | }, nil | |||
| 62 | - | } | |||
| 63 | - | ||||
| 64 | - | _, _, err := repo.readMaintenanceDirent(privatePath, 1, &maintenanceCursor{offset: tc.offset}) | |||
| 65 | - | if err == nil { | |||
| 66 | - | t.Fatal("readMaintenanceDirent error = nil, want failure") | |||
| 67 | - | } | |||
| 68 | - | for _, cause := range tc.wantCause { | |||
| 69 | - | if !errors.Is(err, cause) { | |||
| 70 | - | t.Errorf("errors.Is(%v, %v) = false, want true", err, cause) | |||
| 71 | - | } | |||
| 72 | - | } | |||
| 73 | - | for _, operation := range tc.wantOps { | |||
| 74 | - | if !strings.Contains(err.Error(), operation) { | |||
| 75 | - | t.Errorf("error = %q, want stable operation context %q", err, operation) | |||
| 76 | - | } | |||
| 77 | - | } | |||
| 78 | - | if strings.Contains(err.Error(), privatePath) { | |||
| 79 | - | t.Errorf("error leaks private path: %q", err) | |||
| 80 | - | } | |||
| 81 | - | }) | |||
| 82 | - | } | |||
| 83 | - | } | |||
| 84 | - | ||||
| 85 | - | func TestReadMaintenanceDirentRejectsRecordShorterThanNameOffset(t *testing.T) { | |||
| 86 | - | repo := NewRepository(privateDir(t)) | |||
| 87 | - | nameOffset := int(unsafe.Offsetof(syscall.Dirent{}.Name)) | |||
| 88 | - | recordData := make([]byte, nameOffset) | |||
| 89 | - | reclenOffset := unsafe.Offsetof(syscall.Dirent{}.Reclen) | |||
| 90 | - | binary.NativeEndian.PutUint16(recordData[reclenOffset:], 1) | |||
| 91 | - | ||||
| 92 | - | repo.hooks.openMaintenanceDirectory = func(string) (maintenanceDirectory, error) { | |||
| 93 | - | return fakeMaintenanceDirectory{data: recordData}, nil | |||
| 94 | - | } | |||
| 95 | - | _, _, err := repo.readMaintenanceDirent("unused", 1, &maintenanceCursor{}) | |||
| 96 | - | if !errors.Is(err, syscall.EIO) { | |||
| 97 | - | t.Fatalf("readMaintenanceDirent error = %v, want EIO", err) | |||
| 98 | - | } | |||
| 99 | - | } | |||
| 100 | - | ||||
| 101 | - | func TestDecodeMaintenanceDirentBoundsRecords(t *testing.T) { | |||
| 102 | - | nameOffset := int(unsafe.Offsetof(syscall.Dirent{}.Name)) | |||
| 103 | - | reclenOffset := unsafe.Offsetof(syscall.Dirent{}.Reclen) | |||
| 104 | - | valid := make([]byte, nameOffset+len("entry")+1) | |||
| 105 | - | binary.NativeEndian.PutUint16(valid[reclenOffset:], uint16(len(valid))) | |||
| 106 | - | copy(valid[nameOffset:], "entry") | |||
| 107 | - | ||||
| 108 | - | tests := []struct { | |||
| 109 | - | name string | |||
| 110 | - | data []byte | |||
| 111 | - | wantName string | |||
| 112 | - | wantErr bool | |||
| 113 | - | }{ | |||
| 114 | - | {name: "truncated header", data: make([]byte, nameOffset-1), wantErr: true}, | |||
| 115 | - | {name: "short record", data: append([]byte(nil), valid...), wantErr: true}, | |||
| 116 | - | {name: "truncated record", data: valid[:nameOffset], wantErr: true}, | |||
| 117 | - | {name: "record exceeds aligned storage", data: make([]byte, int(unsafe.Sizeof(syscall.Dirent{}))+1), wantErr: true}, | |||
| 118 | - | {name: "valid", data: valid, wantName: "entry"}, | |||
| 119 | - | } | |||
| 120 | - | binary.NativeEndian.PutUint16(tests[1].data[reclenOffset:], uint16(nameOffset-1)) | |||
| 121 | - | binary.NativeEndian.PutUint16(tests[2].data[reclenOffset:], uint16(len(valid))) | |||
| 122 | - | binary.NativeEndian.PutUint16(tests[3].data[reclenOffset:], uint16(len(tests[3].data))) | |||
| 123 | - | ||||
| 124 | - | for _, tc := range tests { | |||
| 125 | - | t.Run(tc.name, func(t *testing.T) { | |||
| 126 | - | record, name, reclen, err := decodeMaintenanceDirent(tc.data) | |||
| 127 | - | if tc.wantErr { | |||
| 128 | - | if !errors.Is(err, syscall.EIO) { | |||
| 129 | - | t.Fatalf("decodeMaintenanceDirent() error = %v, want EIO", err) | |||
| 130 | - | } | |||
| 131 | - | return | |||
| 132 | - | } | |||
| 133 | - | if err != nil { | |||
| 134 | - | t.Fatal(err) | |||
| 135 | - | } | |||
| 136 | - | if record == nil { | |||
| 137 | - | t.Fatal("decodeMaintenanceDirent() record = nil") | |||
| 138 | - | } | |||
| 139 | - | if reclen != len(valid) { | |||
| 140 | - | t.Fatalf("decodeMaintenanceDirent() reclen = %d, want %d", reclen, len(valid)) | |||
| 141 | - | } | |||
| 142 | - | if got := string(name[:len(tc.wantName)]); got != tc.wantName { | |||
| 143 | - | t.Fatalf("decodeMaintenanceDirent() name = %q, want %q", got, tc.wantName) | |||
| 144 | - | } | |||
| 145 | - | }) | |||
| 146 | - | } | |||
| 147 | - | } | |||
| 148 | - | ||||
| 149 | - | type fakeMaintenanceDirectory struct { | |||
| 150 | - | data []byte | |||
| 151 | - | seekOffset int64 | |||
| 152 | - | seekErr error | |||
| 153 | - | readErr error | |||
| 154 | - | closeErr error | |||
| 155 | - | } | |||
| 156 | - | ||||
| 157 | - | func (d fakeMaintenanceDirectory) Seek(int64, int) (int64, error) { return d.seekOffset, d.seekErr } | |||
| 158 | - | func (d fakeMaintenanceDirectory) ReadDirent(buffer []byte) (int, error) { | |||
| 159 | - | if d.readErr != nil { | |||
| 160 | - | return 0, d.readErr | |||
| 161 | - | } | |||
| 162 | - | return copy(buffer, d.data), nil | |||
| 163 | - | } | |||
| 164 | - | func (d fakeMaintenanceDirectory) Close() error { return d.closeErr } | |||
| 165 | - | ||||
| 166 | - | func TestReadMaintenanceDirentDrainsDarwinBatchesWithoutStarvation(t *testing.T) { | |||
| 167 | - | dir := privateDir(t) | |||
| 168 | - | if err := os.Mkdir(dir, 0o700); err != nil { | |||
| 169 | - | t.Fatal(err) | |||
| 170 | - | } | |||
| 171 | - | batches := [][]string{ | |||
| 172 | - | {"one", "two"}, | |||
| 173 | - | {"three", "four", "five", "six"}, | |||
| 174 | - | } | |||
| 175 | - | for _, batch := range batches { | |||
| 176 | - | for _, name := range batch { | |||
| 177 | - | if err := os.WriteFile(filepath.Join(dir, name), nil, 0o600); err != nil { | |||
| 178 | - | t.Fatal(err) | |||
| 179 | - | } | |||
| 180 | - | } | |||
| 181 | - | } | |||
| 182 | - | ||||
| 183 | - | file := &fakeMultiBatchMaintenanceDirectory{batches: [][]byte{ | |||
| 184 | - | maintenanceDirentBatch(batches[0]...), | |||
| 185 | - | maintenanceDirentBatch(batches[1]...), | |||
| 186 | - | }} | |||
| 187 | - | repo := NewRepository(dir) | |||
| 188 | - | repo.hooks.openMaintenanceDirectory = func(string) (maintenanceDirectory, error) { return file, nil } | |||
| 189 | - | cursor := &maintenanceCursor{} | |||
| 190 | - | ||||
| 191 | - | entries, done, err := repo.readMaintenanceDirentWithDrain(dir, 4, cursor, true, maintenanceTestDirectoryCookie) | |||
| 192 | - | if err != nil { | |||
| 193 | - | t.Fatal(err) | |||
| 194 | - | } | |||
| 195 | - | if done { | |||
| 196 | - | t.Fatal("readMaintenanceDirentWithDrain() done = true, want false before EOF") | |||
| 197 | - | } | |||
| 198 | - | if got, want := maintenanceEntryNames(entries), []string{"one", "two", "three", "four"}; !slices.Equal(got, want) { | |||
| 199 | - | t.Fatalf("first entries = %v, want %v", got, want) | |||
| 200 | - | } | |||
| 201 | - | if got, want := maintenanceEntryNames(cursor.pending), []string{"five", "six"}; !slices.Equal(got, want) { | |||
| 202 | - | t.Fatalf("pending entries = %v, want exactly final-buffer excess %v", got, want) | |||
| 203 | - | } | |||
| 204 | - | if got := file.reads; got != 2 { | |||
| 205 | - | t.Fatalf("ReadDirent calls = %d, want 2 to fill the requested batch", got) | |||
| 206 | - | } | |||
| 207 | - | ||||
| 208 | - | id := maintenanceCursorID(dir, "test") | |||
| 209 | - | repo.maintenanceCursors = map[string]*maintenanceCursor{id: cursor} | |||
| 210 | - | entries, done, err = repo.readMaintenanceDir(dir, 4, "test") | |||
| 211 | - | if err != nil { | |||
| 212 | - | t.Fatal(err) | |||
| 213 | - | } | |||
| 214 | - | if done { | |||
| 215 | - | t.Fatal("pending batch done = true, want false before EOF") | |||
| 216 | - | } | |||
| 217 | - | if got, want := maintenanceEntryNames(entries), []string{"five", "six"}; !slices.Equal(got, want) { | |||
| 218 | - | t.Fatalf("pending entries = %v, want %v", got, want) | |||
| 219 | - | } | |||
| 220 | - | if got := file.reads; got != 2 { | |||
| 221 | - | t.Fatalf("ReadDirent calls while delivering pending entries = %d, want 2", got) | |||
| 222 | - | } | |||
| 223 | - | ||||
| 224 | - | entries, done, err = repo.readMaintenanceDir(dir, 4, "test") | |||
| 225 | - | if err != nil { | |||
| 226 | - | t.Fatal(err) | |||
| 227 | - | } | |||
| 228 | - | if !done { | |||
| 229 | - | t.Fatal("EOF batch done = false, want true") | |||
| 230 | - | } | |||
| 231 | - | if len(entries) != 0 { | |||
| 232 | - | t.Fatalf("EOF entries = %v, want none", entries) | |||
| 233 | - | } | |||
| 234 | - | if _, ok := repo.maintenanceCursors[id]; ok { | |||
| 235 | - | t.Fatal("EOF cursor retained after pending entries were delivered") | |||
| 236 | - | } | |||
| 237 | - | } | |||
| 238 | - | ||||
| 239 | - | func maintenanceEntryNames(entries []maintenanceDirEntry) []string { | |||
| 240 | - | names := make([]string, len(entries)) | |||
| 241 | - | for i, entry := range entries { | |||
| 242 | - | names[i] = entry.name | |||
| 243 | - | } | |||
| 244 | - | return names | |||
| 245 | - | } | |||
| 246 | - | ||||
| 247 | - | func maintenanceDirentBatch(names ...string) []byte { | |||
| 248 | - | nameOffset := int(unsafe.Offsetof(syscall.Dirent{}.Name)) | |||
| 249 | - | reclenOffset := unsafe.Offsetof(syscall.Dirent{}.Reclen) | |||
| 250 | - | var data []byte | |||
| 251 | - | for _, name := range names { | |||
| 252 | - | record := make([]byte, nameOffset+len(name)+1) | |||
| 253 | - | binary.NativeEndian.PutUint16(record[reclenOffset:], uint16(len(record))) | |||
| 254 | - | copy(record[nameOffset:], name) | |||
| 255 | - | data = append(data, record...) | |||
| 256 | - | } | |||
| 257 | - | return data | |||
| 258 | - | } | |||
| 259 | - | ||||
| 260 | - | type fakeMultiBatchMaintenanceDirectory struct { | |||
| 261 | - | batches [][]byte | |||
| 262 | - | index int | |||
| 263 | - | reads int | |||
| 264 | - | } | |||
| 265 | - | ||||
| 266 | - | func (d *fakeMultiBatchMaintenanceDirectory) Seek(offset int64, whence int) (int64, error) { | |||
| 267 | - | if whence == io.SeekCurrent { | |||
| 268 | - | return int64(d.index), nil | |||
| 269 | - | } | |||
| 270 | - | d.index = int(offset) | |||
| 271 | - | return offset, nil | |||
| 272 | - | } | |||
| 273 | - | ||||
| 274 | - | func (d *fakeMultiBatchMaintenanceDirectory) ReadDirent(buffer []byte) (int, error) { | |||
| 275 | - | d.reads++ | |||
| 276 | - | if d.index == len(d.batches) { | |||
| 277 | - | return 0, nil | |||
| 278 | - | } | |||
| 279 | - | batch := d.batches[d.index] | |||
| 280 | - | d.index++ | |||
| 281 | - | return copy(buffer, batch), nil | |||
| 282 | - | } | |||
| 283 | - | ||||
| 284 | - | func (d *fakeMultiBatchMaintenanceDirectory) Close() error { return nil } | |||
| 285 | - | ||||
| 286 | - | func maintenanceTestDirectoryCookie(file maintenanceDirectory, _ *syscall.Dirent) (int64, error) { | |||
| 287 | - | return file.Seek(0, io.SeekCurrent) | |||
| 288 | - | } | |||
| 289 | - | ||||
| 290 | - | func TestRepositoryDeleteQuarantinesCanonicalSessionBeforeCleanup(t *testing.T) { | |||
| 291 | - | repo := NewRepository(privateDir(t)) | |||
| 292 | - | if err := repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state"))); err != nil { | |||
| 293 | - | t.Fatal(err) | |||
| 294 | - | } | |||
| 295 | - | if err := repo.Delete(context.Background(), "named"); err != nil { | |||
| 296 | - | t.Fatal(err) | |||
| 297 | - | } | |||
| 298 | - | ||||
| 299 | - | canonical := repo.sessionPath(sessionKey("named")) | |||
| 300 | - | if _, err := os.Lstat(canonical); !errors.Is(err, os.ErrNotExist) { | |||
| 301 | - | t.Fatalf("canonical session after Delete = %v, want not exist", err) | |||
| 302 | - | } | |||
| 303 | - | entries, err := os.ReadDir(filepath.Join(repo.dir, repositorySessionsDir)) | |||
| 304 | - | if err != nil { | |||
| 305 | - | t.Fatal(err) | |||
| 306 | - | } | |||
| 307 | - | if len(entries) != 1 || !isQuarantine(entries[0].Name()) || !entries[0].IsDir() { | |||
| 308 | - | t.Fatalf("session entries after Delete = %#v, want one quarantine", entries) | |||
| 309 | - | } | |||
| 310 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 311 | - | t.Fatal(err) | |||
| 312 | - | } | |||
| 313 | - | entries, err = os.ReadDir(filepath.Join(repo.dir, repositorySessionsDir)) | |||
| 314 | - | if err != nil { | |||
| 315 | - | t.Fatal(err) | |||
| 316 | - | } | |||
| 317 | - | if len(entries) != 0 { | |||
| 318 | - | t.Fatalf("session entries after Maintain = %#v, want none", entries) | |||
| 319 | - | } | |||
| 320 | - | } | |||
| 321 | - | ||||
| 322 | - | func TestRepositoryDeleteReturnsRootSyncFailureAfterQuarantine(t *testing.T) { | |||
| 323 | - | repo := NewRepository(privateDir(t)) | |||
| 324 | - | if err := repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state"))); err != nil { | |||
| 325 | - | t.Fatal(err) | |||
| 326 | - | } | |||
| 327 | - | injected := errors.New("sessions sync") | |||
| 328 | - | repo.hooks.syncDirectory = func(string) error { return injected } | |||
| 329 | - | if err := repo.Delete(context.Background(), "named"); !errors.Is(err, injected) { | |||
| 330 | - | t.Fatalf("Delete error = %v, want root sync failure", err) | |||
| 331 | - | } | |||
| 332 | - | if _, err := os.Lstat(repo.sessionPath(sessionKey("named"))); !errors.Is(err, os.ErrNotExist) { | |||
| 333 | - | t.Fatalf("canonical session after failed sync = %v, want not exist", err) | |||
| 334 | - | } | |||
| 335 | - | } | |||
| 336 | - | ||||
| 337 | - | func TestRepositoryDeleteWaitsForPublication(t *testing.T) { | |||
| 338 | - | repo := NewRepository(privateDir(t)) | |||
| 339 | - | if err := repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("first"))); err != nil { | |||
| 340 | - | t.Fatal(err) | |||
| 341 | - | } | |||
| 342 | - | entered := make(chan struct{}) | |||
| 343 | - | release := make(chan struct{}) | |||
| 344 | - | repo.hooks.beforeHeadWrite = func(string) error { | |||
| 345 | - | close(entered) | |||
| 346 | - | <-release | |||
| 347 | - | return nil | |||
| 348 | - | } | |||
| 349 | - | publishDone := make(chan error, 1) | |||
| 350 | - | go func() { | |||
| 351 | - | publishDone <- repo.Publish(context.Background(), repositoryPublication(t, "named", 2, []byte("second"))) | |||
| 352 | - | }() | |||
| 353 | - | <-entered | |||
| 354 | - | deleteReached := make(chan struct{}) | |||
| 355 | - | deleteRelease := make(chan struct{}) | |||
| 356 | - | repo.hooks.beforeSessionLock = waitAtSessionLock(deleteReached, deleteRelease) | |||
| 357 | - | deleteDone := make(chan error, 1) | |||
| 358 | - | go func() { deleteDone <- repo.Delete(context.Background(), "named") }() | |||
| 359 | - | <-deleteReached | |||
| 360 | - | // Delete is parked at the session-lock boundary while Publish owns it. | |||
| 361 | - | close(deleteRelease) | |||
| 362 | - | close(release) | |||
| 363 | - | if err := <-publishDone; err != nil { | |||
| 364 | - | t.Fatal(err) | |||
| 365 | - | } | |||
| 366 | - | if err := <-deleteDone; err != nil { | |||
| 367 | - | t.Fatal(err) | |||
| 368 | - | } | |||
| 369 | - | if _, err := os.Lstat(repo.sessionPath(sessionKey("named"))); !errors.Is(err, os.ErrNotExist) { | |||
| 370 | - | t.Fatalf("canonical session after concurrent Delete = %v, want not exist", err) | |||
| 371 | - | } | |||
| 372 | - | } | |||
| 373 | - | ||||
| 374 | - | func TestRepositoryMaintainRetainsNewestTwoCompleteGenerations(t *testing.T) { | |||
| 375 | - | repo := NewRepository(privateDir(t)) | |||
| 376 | - | for generation := uint64(1); generation <= 3; generation++ { | |||
| 377 | - | if err := repo.Publish(context.Background(), repositoryPublication(t, "named", generation, []byte{byte(generation)})); err != nil { | |||
| 378 | - | t.Fatal(err) | |||
| 379 | - | } | |||
| 380 | - | } | |||
| 381 | - | key := sessionKey("named") | |||
| 382 | - | for pass := 0; pass < maintenanceBatch; pass++ { | |||
| 383 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 384 | - | t.Fatal(err) | |||
| 385 | - | } | |||
| 386 | - | _, err := os.Lstat(repo.manifestPath(key, 1)) | |||
| 387 | - | if errors.Is(err, os.ErrNotExist) { | |||
| 388 | - | break | |||
| 389 | - | } | |||
| 390 | - | if err != nil { | |||
| 391 | - | t.Fatalf("old manifest during Maintain = %v", err) | |||
| 392 | - | } | |||
| 393 | - | if pass == maintenanceBatch-1 { | |||
| 394 | - | t.Fatal("old manifest was not eventually collected") | |||
| 395 | - | } | |||
| 396 | - | } | |||
| 397 | - | for _, generation := range []uint64{2, 3} { | |||
| 398 | - | if _, err := os.Lstat(repo.manifestPath(key, generation)); err != nil { | |||
| 399 | - | t.Fatalf("retained manifest %d: %v", generation, err) | |||
| 400 | - | } | |||
| 401 | - | } | |||
| 402 | - | if _, err := os.Lstat(repo.manifestPath(key, 1)); !errors.Is(err, os.ErrNotExist) { | |||
| 403 | - | t.Fatalf("old manifest after Maintain = %v, want not exist", err) | |||
| 404 | - | } | |||
| 405 | - | for _, generation := range []uint64{2, 3} { | |||
| 406 | - | if _, err := repo.Load(context.Background(), "named"); err != nil { | |||
| 407 | - | t.Fatalf("Load after collecting generation %d: %v", generation, err) | |||
| 408 | - | } | |||
| 409 | - | } | |||
| 410 | - | } | |||
| 411 | - | ||||
| 412 | - | func TestRepositoryMaintainPreservesIncompleteManifestReferences(t *testing.T) { | |||
| 413 | - | repo := NewRepository(privateDir(t)) | |||
| 414 | - | for generation := uint64(1); generation <= 3; generation++ { | |||
| 415 | - | if err := repo.Publish(context.Background(), repositoryPublication(t, "named", generation, []byte{byte(generation)})); err != nil { | |||
| 416 | - | t.Fatal(err) | |||
| 417 | - | } | |||
| 418 | - | } | |||
| 419 | - | key := sessionKey("named") | |||
| 420 | - | manifest := repositoryPublication(t, "named", 3, []byte{3}) | |||
| 421 | - | if err := os.Remove(repo.objectPath(key, manifest.Objects[1].Digest)); err != nil { | |||
| 422 | - | t.Fatal(err) | |||
| 423 | - | } | |||
| 424 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 425 | - | t.Fatal(err) | |||
| 426 | - | } | |||
| 427 | - | if _, err := os.Lstat(repo.manifestPath(key, 3)); err != nil { | |||
| 428 | - | t.Fatalf("incomplete manifest removed: %v", err) | |||
| 429 | - | } | |||
| 430 | - | if _, err := os.Lstat(repo.objectPath(key, manifest.Objects[0].Digest)); err != nil { | |||
| 431 | - | t.Fatalf("object referenced by incomplete manifest removed: %v", err) | |||
| 432 | - | } | |||
| 433 | - | } | |||
| 434 | - | ||||
| 435 | - | func TestRepositoryCancellationAfterSessionLockWaitPreventsMutations(t *testing.T) { | |||
| 436 | - | t.Run("publish", func(t *testing.T) { | |||
| 437 | - | repo := NewRepository(privateDir(t)) | |||
| 438 | - | key := sessionKey("named") | |||
| 439 | - | lock := repo.lockSession(key) | |||
| 440 | - | ctx, cancel := context.WithCancel(context.Background()) | |||
| 441 | - | reached := make(chan struct{}) | |||
| 442 | - | release := make(chan struct{}) | |||
| 443 | - | repo.hooks.beforeSessionLock = waitAtSessionLock(reached, release) | |||
| 444 | - | done := make(chan error, 1) | |||
| 445 | - | go func() { done <- repo.Publish(ctx, repositoryPublication(t, "named", 1, []byte("state"))) }() | |||
| 446 | - | <-reached | |||
| 447 | - | cancel() | |||
| 448 | - | close(release) | |||
| 449 | - | repo.unlockSession(lock) | |||
| 450 | - | if err := <-done; !errors.Is(err, context.Canceled) { | |||
| 451 | - | t.Fatalf("Publish error = %v, want canceled", err) | |||
| 452 | - | } | |||
| 453 | - | if _, err := os.Lstat(repo.dir); !errors.Is(err, os.ErrNotExist) { | |||
| 454 | - | t.Fatalf("Publish mutated canceled repository: %v", err) | |||
| 455 | - | } | |||
| 456 | - | }) | |||
| 457 | - | t.Run("load and delete", func(t *testing.T) { | |||
| 458 | - | repo := NewRepository(privateDir(t)) | |||
| 459 | - | if err := repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state"))); err != nil { | |||
| 460 | - | t.Fatal(err) | |||
| 461 | - | } | |||
| 462 | - | key := sessionKey("named") | |||
| 463 | - | for _, operation := range []struct { | |||
| 464 | - | name string | |||
| 465 | - | run func(context.Context) error | |||
| 466 | - | }{ | |||
| 467 | - | {"load", func(ctx context.Context) error { _, err := repo.Load(ctx, "named"); return err }}, | |||
| 468 | - | {"delete", func(ctx context.Context) error { return repo.Delete(ctx, "named") }}, | |||
| 469 | - | } { | |||
| 470 | - | t.Run(operation.name, func(t *testing.T) { | |||
| 471 | - | lock := repo.lockSession(key) | |||
| 472 | - | ctx, cancel := context.WithCancel(context.Background()) | |||
| 473 | - | reached := make(chan struct{}) | |||
| 474 | - | release := make(chan struct{}) | |||
| 475 | - | repo.hooks.beforeSessionLock = waitAtSessionLock(reached, release) | |||
| 476 | - | done := make(chan error, 1) | |||
| 477 | - | go func() { done <- operation.run(ctx) }() | |||
| 478 | - | <-reached | |||
| 479 | - | cancel() | |||
| 480 | - | close(release) | |||
| 481 | - | repo.unlockSession(lock) | |||
| 482 | - | if err := <-done; !errors.Is(err, context.Canceled) { | |||
| 483 | - | t.Fatalf("%s error = %v, want canceled", operation.name, err) | |||
| 484 | - | } | |||
| 485 | - | repo.hooks.beforeSessionLock = nil | |||
| 486 | - | if _, err := repo.Load(context.Background(), "named"); err != nil { | |||
| 487 | - | t.Fatalf("%s mutated session: %v", operation.name, err) | |||
| 488 | - | } | |||
| 489 | - | }) | |||
| 490 | - | } | |||
| 491 | - | }) | |||
| 492 | - | t.Run("maintain", func(t *testing.T) { | |||
| 493 | - | repo := NewRepository(privateDir(t)) | |||
| 494 | - | if err := repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state"))); err != nil { | |||
| 495 | - | t.Fatal(err) | |||
| 496 | - | } | |||
| 497 | - | lock := repo.lockSession(sessionKey("named")) | |||
| 498 | - | ctx, cancel := context.WithCancel(context.Background()) | |||
| 499 | - | reached := make(chan struct{}) | |||
| 500 | - | release := make(chan struct{}) | |||
| 501 | - | repo.hooks.beforeSessionLock = waitAtSessionLock(reached, release) | |||
| 502 | - | done := make(chan error, 1) | |||
| 503 | - | go func() { done <- repo.Maintain(ctx) }() | |||
| 504 | - | <-reached | |||
| 505 | - | cancel() | |||
| 506 | - | close(release) | |||
| 507 | - | repo.unlockSession(lock) | |||
| 508 | - | if err := <-done; !errors.Is(err, context.Canceled) { | |||
| 509 | - | t.Fatalf("Maintain error = %v, want canceled", err) | |||
| 510 | - | } | |||
| 511 | - | repo.hooks.beforeSessionLock = nil | |||
| 512 | - | if _, err := repo.Load(context.Background(), "named"); err != nil { | |||
| 513 | - | t.Fatalf("Maintain mutated session: %v", err) | |||
| 514 | - | } | |||
| 515 | - | }) | |||
| 516 | - | } | |||
| 517 | - | ||||
| 518 | - | func waitAtSessionLock(reached chan<- struct{}, release <-chan struct{}) func(string) { | |||
| 519 | - | return func(string) { | |||
| 520 | - | close(reached) | |||
| 521 | - | <-release | |||
| 522 | - | } | |||
| 523 | - | } | |||
| 524 | - | ||||
| 525 | - | func TestRepositoryDeleteRetriesPendingQuarantineSyncWithoutDeletingRecreatedSession(t *testing.T) { | |||
| 526 | - | repo := NewRepository(privateDir(t)) | |||
| 527 | - | if err := repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("old"))); err != nil { | |||
| 528 | - | t.Fatal(err) | |||
| 529 | - | } | |||
| 530 | - | injected := errors.New("sessions sync") | |||
| 531 | - | repo.hooks.syncDirectory = func(string) error { return injected } | |||
| 532 | - | if err := repo.Delete(context.Background(), "named"); !errors.Is(err, injected) { | |||
| 533 | - | t.Fatalf("Delete error = %v, want sync failure", err) | |||
| 534 | - | } | |||
| 535 | - | repo.hooks.syncDirectory = nil | |||
| 536 | - | if err := repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("new"))); err != nil { | |||
| 537 | - | t.Fatal(err) | |||
| 538 | - | } | |||
| 539 | - | if err := repo.Delete(context.Background(), "named"); err != nil { | |||
| 540 | - | t.Fatalf("retry Delete: %v", err) | |||
| 541 | - | } | |||
| 542 | - | if _, err := repo.Load(context.Background(), "named"); err != nil { | |||
| 543 | - | t.Fatalf("recreated session was deleted: %v", err) | |||
| 544 | - | } | |||
| 545 | - | } | |||
| 546 | - | ||||
| 547 | - | func TestRepositoryMaintainResumesNestedQuarantineWithinBatch(t *testing.T) { | |||
| 548 | - | repo := NewRepository(privateDir(t)) | |||
| 549 | - | quarantine := filepath.Join(repo.dir, repositorySessionsDir, ".deleting-named-test") | |||
| 550 | - | for i := 0; i < maintenanceBatch+1; i++ { | |||
| 551 | - | path := filepath.Join(quarantine, "nested", string(rune('a'+i))) | |||
| 552 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |||
| 553 | - | t.Fatal(err) | |||
| 554 | - | } | |||
| 555 | - | if err := os.WriteFile(path, []byte("stale"), 0o600); err != nil { | |||
| 556 | - | t.Fatal(err) | |||
| 557 | - | } | |||
| 558 | - | } | |||
| 559 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 560 | - | t.Fatal(err) | |||
| 561 | - | } | |||
| 562 | - | if _, err := os.Lstat(quarantine); errors.Is(err, os.ErrNotExist) { | |||
| 563 | - | t.Fatalf("quarantine after one bounded call = %v, want remaining tree", err) | |||
| 564 | - | } | |||
| 565 | - | for i := 0; i < 12; i++ { | |||
| 566 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 567 | - | t.Fatal(err) | |||
| 568 | - | } | |||
| 569 | - | if _, err := os.Lstat(quarantine); errors.Is(err, os.ErrNotExist) { | |||
| 570 | - | return | |||
| 571 | - | } | |||
| 572 | - | } | |||
| 573 | - | if _, err := os.Lstat(quarantine); !errors.Is(err, os.ErrNotExist) { | |||
| 574 | - | t.Fatalf("quarantine after resumed maintenance = %v, want removed", err) | |||
| 575 | - | } | |||
| 576 | - | } | |||
| 577 | - | ||||
| 578 | - | func TestRepositoryMaintainClosesContinuationDirectories(t *testing.T) { | |||
| 579 | - | repo := NewRepository(privateDir(t)) | |||
| 580 | - | sessions := filepath.Join(repo.dir, repositorySessionsDir) | |||
| 581 | - | for i := 0; i < maintenanceBatch+1; i++ { | |||
| 582 | - | generations := filepath.Join(sessions, fmt.Sprintf("named-%03d", i), repositoryGenerations) | |||
| 583 | - | if err := os.MkdirAll(generations, 0o700); err != nil { | |||
| 584 | - | t.Fatal(err) | |||
| 585 | - | } | |||
| 586 | - | for j := 0; j < maintenanceBatch+1; j++ { | |||
| 587 | - | if err := os.WriteFile(filepath.Join(generations, fmt.Sprintf("unclassified-%03d", j)), []byte("state"), 0o600); err != nil { | |||
| 588 | - | t.Fatal(err) | |||
| 589 | - | } | |||
| 590 | - | } | |||
| 591 | - | } | |||
| 592 | - | before := openDescriptorCount(t) | |||
| 593 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 594 | - | t.Fatal(err) | |||
| 595 | - | } | |||
| 596 | - | if got := openDescriptorCount(t); got > before+4 { | |||
| 597 | - | t.Fatalf("open descriptors after bounded maintenance = %d, want at most %d", got, before+4) | |||
| 598 | - | } | |||
| 599 | - | } | |||
| 600 | - | ||||
| 601 | - | func TestRepositoryMaintainBoundsQueuedShardNames(t *testing.T) { | |||
| 602 | - | repo := NewRepository(privateDir(t)) | |||
| 603 | - | key := sessionKey("named") | |||
| 604 | - | root := filepath.Join(repo.sessionPath(key), repositoryObjectsDir) | |||
| 605 | - | for i := 0; i < maintenanceBatch*2; i++ { | |||
| 606 | - | if err := os.MkdirAll(filepath.Join(root, fmt.Sprintf("%02x", i)), 0o700); err != nil { | |||
| 607 | - | t.Fatal(err) | |||
| 608 | - | } | |||
| 609 | - | } | |||
| 610 | - | repo.maintenanceCursors = make(map[string]*maintenanceCursor) | |||
| 611 | - | state := &sessionMaintenance{marked: make(map[uint64]manifestMaintenance), markDone: true} | |||
| 612 | - | for i := 0; i < maintenanceBatch+2; i++ { | |||
| 613 | - | budget := maintenanceBatch | |||
| 614 | - | if err := repo.sweepSession(context.Background(), key, state, &budget); err != nil { | |||
| 615 | - | t.Fatal(err) | |||
| 616 | - | } | |||
| 617 | - | if state.sweepShard != "" && len(repo.maintenanceCursors) > maxMaintenanceCursors { | |||
| 618 | - | t.Fatalf("retained maintenance cursors = %d, want at most %d", len(repo.maintenanceCursors), maxMaintenanceCursors) | |||
| 619 | - | } | |||
| 620 | - | } | |||
| 621 | - | } | |||
| 622 | - | ||||
| 623 | - | func TestRepositoryMaintainResumesCursorAfterDirectoryMutation(t *testing.T) { | |||
| 624 | - | repo := NewRepository(privateDir(t)) | |||
| 625 | - | if err := os.Mkdir(repo.dir, 0o700); err != nil { | |||
| 626 | - | t.Fatal(err) | |||
| 627 | - | } | |||
| 628 | - | for i := 0; i < maintenanceBatch+1; i++ { | |||
| 629 | - | if err := os.WriteFile(filepath.Join(repo.dir, fmt.Sprintf("live-%03d", i)), []byte("live"), 0o600); err != nil { | |||
| 630 | - | t.Fatal(err) | |||
| 631 | - | } | |||
| 632 | - | } | |||
| 633 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 634 | - | t.Fatal(err) | |||
| 635 | - | } | |||
| 636 | - | // Mutating entries ahead of the saved cookie must not reset traversal to | |||
| 637 | - | // the prefix or lose a stale entry appended after that cookie. | |||
| 638 | - | if err := os.Remove(filepath.Join(repo.dir, "live-000")); err != nil { | |||
| 639 | - | t.Fatal(err) | |||
| 640 | - | } | |||
| 641 | - | stale := filepath.Join(repo.dir, ".tmp-appended") | |||
| 642 | - | if err := os.WriteFile(stale, []byte("stale"), 0o600); err != nil { | |||
| 643 | - | t.Fatal(err) | |||
| 644 | - | } | |||
| 645 | - | for i := 0; i < 4; i++ { | |||
| 646 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 647 | - | t.Fatal(err) | |||
| 648 | - | } | |||
| 649 | - | if _, err := os.Lstat(stale); errors.Is(err, os.ErrNotExist) { | |||
| 650 | - | return | |||
| 651 | - | } | |||
| 652 | - | } | |||
| 653 | - | t.Fatal("stale entry appended after cursor mutation was not eventually removed") | |||
| 654 | - | } | |||
| 655 | - | ||||
| 656 | - | func TestRepositoryMaintainDoesNotStarveLaterTemporaryEntries(t *testing.T) { | |||
| 657 | - | repo := NewRepository(privateDir(t)) | |||
| 658 | - | if err := os.Mkdir(repo.dir, 0o700); err != nil { | |||
| 659 | - | t.Fatal(err) | |||
| 660 | - | } | |||
| 661 | - | // The live entries precede the stale entries in directory insertion order. | |||
| 662 | - | // A fresh ReadDir from the start on every call would never reach the latter. | |||
| 663 | - | for i := 0; i < maintenanceBatch+1; i++ { | |||
| 664 | - | if err := os.WriteFile(filepath.Join(repo.dir, "live-"+string(rune('a'+i))), []byte("live"), 0o600); err != nil { | |||
| 665 | - | t.Fatal(err) | |||
| 666 | - | } | |||
| 667 | - | } | |||
| 668 | - | for i := 0; i < maintenanceBatch+1; i++ { | |||
| 669 | - | if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-later-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil { | |||
| 670 | - | t.Fatal(err) | |||
| 671 | - | } | |||
| 672 | - | } | |||
| 673 | - | for i := 0; i < 4; i++ { | |||
| 674 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 675 | - | t.Fatal(err) | |||
| 676 | - | } | |||
| 677 | - | } | |||
| 678 | - | entries, err := os.ReadDir(repo.dir) | |||
| 679 | - | if err != nil { | |||
| 680 | - | t.Fatal(err) | |||
| 681 | - | } | |||
| 682 | - | for _, entry := range entries { | |||
| 683 | - | if strings.HasPrefix(entry.Name(), ".tmp-") { | |||
| 684 | - | t.Fatalf("stale entry %q was starved", entry.Name()) | |||
| 685 | - | } | |||
| 686 | - | } | |||
| 687 | - | } | |||
| 688 | - | ||||
| 689 | - | func TestRepositoryMaintainMarksLargeGenerationSetBeforeSweeping(t *testing.T) { | |||
| 690 | - | repo := NewRepository(privateDir(t)) | |||
| 691 | - | var newest ports.SnapshotPublication | |||
| 692 | - | for generation := uint64(1); generation <= maintenanceBatch+2; generation++ { | |||
| 693 | - | publication := repositoryPublication(t, "named", generation, []byte(fmt.Sprintf("state-%d", generation))) | |||
| 694 | - | if err := repo.Publish(context.Background(), publication); err != nil { | |||
| 695 | - | t.Fatal(err) | |||
| 696 | - | } | |||
| 697 | - | newest = publication | |||
| 698 | - | } | |||
| 699 | - | key := sessionKey("named") | |||
| 700 | - | stale := sha256.Sum256([]byte("stale object")) | |||
| 701 | - | stalePath := repo.objectPath(key, stale) | |||
| 702 | - | if err := os.MkdirAll(filepath.Dir(stalePath), 0o700); err != nil { | |||
| 703 | - | t.Fatal(err) | |||
| 704 | - | } | |||
| 705 | - | if err := os.WriteFile(stalePath, []byte("stale object"), 0o600); err != nil { | |||
| 706 | - | t.Fatal(err) | |||
| 707 | - | } | |||
| 708 | - | ||||
| 709 | - | // The first call has classified only one manifest batch. It must not sweep | |||
| 710 | - | // even an otherwise stale object, nor a blob owned by a later manifest. | |||
| 711 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 712 | - | t.Fatal(err) | |||
| 713 | - | } | |||
| 714 | - | if _, err := os.Lstat(stalePath); err != nil { | |||
| 715 | - | t.Fatalf("stale object swept before complete mark pass: %v", err) | |||
| 716 | - | } | |||
| 717 | - | for _, object := range newest.Objects { | |||
| 718 | - | if _, err := os.Lstat(repo.objectPath(key, object.Digest)); err != nil { | |||
| 719 | - | t.Fatalf("object referenced by not-yet-classified manifest removed: %v", err) | |||
| 720 | - | } | |||
| 721 | - | } | |||
| 722 | - | ||||
| 723 | - | for i := 0; i < 600; i++ { | |||
| 724 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 725 | - | t.Fatal(err) | |||
| 726 | - | } | |||
| 727 | - | if _, err := os.Lstat(stalePath); errors.Is(err, os.ErrNotExist) { | |||
| 728 | - | break | |||
| 729 | - | } | |||
| 730 | - | if i == 599 { | |||
| 731 | - | t.Fatal("stale object was not eventually swept") | |||
| 732 | - | } | |||
| 733 | - | } | |||
| 734 | - | for _, generation := range []uint64{maintenanceBatch + 1, maintenanceBatch + 2} { | |||
| 735 | - | if _, err := os.Lstat(repo.manifestPath(key, generation)); err != nil { | |||
| 736 | - | t.Fatalf("newest complete manifest %d removed: %v", generation, err) | |||
| 737 | - | } | |||
| 738 | - | } | |||
| 739 | - | } | |||
| 740 | - | ||||
| 741 | - | func TestRepositoryMaintainUsesFixedRemovalBatch(t *testing.T) { | |||
| 742 | - | repo := NewRepository(privateDir(t)) | |||
| 743 | - | if err := os.Mkdir(repo.dir, 0o700); err != nil { | |||
| 744 | - | t.Fatal(err) | |||
| 745 | - | } | |||
| 746 | - | for i := 0; i < maintenanceBatch+1; i++ { | |||
| 747 | - | if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-"+string(rune('a'+i))), []byte("stale"), 0o600); err != nil { | |||
| 748 | - | t.Fatal(err) | |||
| 749 | - | } | |||
| 750 | - | } | |||
| 751 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 752 | - | t.Fatal(err) | |||
| 753 | - | } | |||
| 754 | - | entries, err := os.ReadDir(repo.dir) | |||
| 755 | - | if err != nil { | |||
| 756 | - | t.Fatal(err) | |||
| 757 | - | } | |||
| 758 | - | if len(entries) != 1 { | |||
| 759 | - | t.Fatalf("remaining stale entries = %d, want 1 after one batch", len(entries)) | |||
| 760 | - | } | |||
| 761 | - | } | |||
| 762 | - | ||||
| 763 | - | func TestRepositoryMaintainQueuesFetchedSessionsPastWorkBudget(t *testing.T) { | |||
| 764 | - | repo := NewRepository(privateDir(t)) | |||
| 765 | - | if err := os.MkdirAll(filepath.Join(repo.dir, repositorySessionsDir), 0o700); err != nil { | |||
| 766 | - | t.Fatal(err) | |||
| 767 | - | } | |||
| 768 | - | if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-consume-budget"), []byte("stale"), 0o600); err != nil { | |||
| 769 | - | t.Fatal(err) | |||
| 770 | - | } | |||
| 771 | - | ||||
| 772 | - | removed := make(map[string]int) | |||
| 773 | - | for i := 0; i < maintenanceBatch+1; i++ { | |||
| 774 | - | path := filepath.Join(repo.dir, repositorySessionsDir, fmt.Sprintf(".deleting-%03d", i)) | |||
| 775 | - | if err := os.Mkdir(path, 0o700); err != nil { | |||
| 776 | - | t.Fatal(err) | |||
| 777 | - | } | |||
| 778 | - | } | |||
| 779 | - | repo.hooks.remove = func(path string) error { | |||
| 780 | - | if strings.HasPrefix(filepath.Base(path), ".deleting-") { | |||
| 781 | - | removed[path]++ | |||
| 782 | - | } | |||
| 783 | - | return nil | |||
| 784 | - | } | |||
| 785 | - | ||||
| 786 | - | sessions := filepath.Join(repo.dir, repositorySessionsDir) | |||
| 787 | - | f, err := os.Open(sessions) | |||
| 788 | - | if err != nil { | |||
| 789 | - | t.Fatal(err) | |||
| 790 | - | } | |||
| 791 | - | fetched, err := f.ReadDir(maintenanceBatch) | |||
| 792 | - | if err != nil { | |||
| 793 | - | t.Fatal(err) | |||
| 794 | - | } | |||
| 795 | - | unread, err := f.ReadDir(1) | |||
| 796 | - | if err != nil { | |||
| 797 | - | t.Fatal(err) | |||
| 798 | - | } | |||
| 799 | - | if err := f.Close(); err != nil { | |||
| 800 | - | t.Fatal(err) | |||
| 801 | - | } | |||
| 802 | - | queued := fetched[len(fetched)-1].Name() | |||
| 803 | - | for i := 0; i < maintenanceBatch+4; i++ { | |||
| 804 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 805 | - | t.Fatal(err) | |||
| 806 | - | } | |||
| 807 | - | if _, err := os.Lstat(filepath.Join(sessions, queued)); errors.Is(err, os.ErrNotExist) { | |||
| 808 | - | break | |||
| 809 | - | } else if i == maintenanceBatch+3 { | |||
| 810 | - | t.Fatalf("fetched but unprocessed session %q = %v, want eventually removed", queued, err) | |||
| 811 | - | } | |||
| 812 | - | } | |||
| 813 | - | if _, err := os.Lstat(filepath.Join(sessions, unread[0].Name())); err != nil { | |||
| 814 | - | t.Fatalf("unread session %q = %v, want queued entry to run first", unread[0].Name(), err) | |||
| 815 | - | } | |||
| 816 | - | ||||
| 817 | - | for i := 0; i < maintenanceBatch+4; i++ { | |||
| 818 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 819 | - | t.Fatal(err) | |||
| 820 | - | } | |||
| 821 | - | entries, err := os.ReadDir(sessions) | |||
| 822 | - | if err != nil { | |||
| 823 | - | t.Fatal(err) | |||
| 824 | - | } | |||
| 825 | - | if len(entries) == 0 { | |||
| 826 | - | break | |||
| 827 | - | } | |||
| 828 | - | } | |||
| 829 | - | entries, err := os.ReadDir(sessions) | |||
| 830 | - | if err != nil { | |||
| 831 | - | t.Fatal(err) | |||
| 832 | - | } | |||
| 833 | - | if len(entries) != 0 { | |||
| 834 | - | t.Fatalf("queued sessions after resumed maintenance = %d, want none", len(entries)) | |||
| 835 | - | } | |||
| 836 | - | if len(removed) != maintenanceBatch+1 { | |||
| 837 | - | t.Fatalf("removed sessions = %d, want %d", len(removed), maintenanceBatch+1) | |||
| 838 | - | } | |||
| 839 | - | for path, calls := range removed { | |||
| 840 | - | if calls != 1 { | |||
| 841 | - | t.Fatalf("remove calls for %q = %d, want one", path, calls) | |||
| 842 | - | } | |||
| 843 | - | } | |||
| 844 | - | } | |||
| 845 | - | ||||
| 846 | - | func TestRepositoryMaintainClassifiesUnpublishedGenerationsBeforeSweep(t *testing.T) { | |||
| 847 | - | for _, head := range []struct { | |||
| 848 | - | name string | |||
| 849 | - | set func(t *testing.T, repo *Repository, key string) | |||
| 850 | - | }{ | |||
| 851 | - | { | |||
| 852 | - | name: "missing", | |||
| 853 | - | set: func(t *testing.T, repo *Repository, key string) { | |||
| 854 | - | t.Helper() | |||
| 855 | - | if err := os.Remove(repo.headPath(key)); err != nil { | |||
| 856 | - | t.Fatal(err) | |||
| 857 | - | } | |||
| 858 | - | }, | |||
| 859 | - | }, | |||
| 860 | - | { | |||
| 861 | - | name: "corrupt", | |||
| 862 | - | set: func(t *testing.T, repo *Repository, key string) { | |||
| 863 | - | t.Helper() | |||
| 864 | - | if err := os.WriteFile(repo.headPath(key), []byte("corrupt"), 0o600); err != nil { | |||
| 865 | - | t.Fatal(err) | |||
| 866 | - | } | |||
| 867 | - | }, | |||
| 868 | - | }, | |||
| 869 | - | } { | |||
| 870 | - | t.Run(head.name, func(t *testing.T) { | |||
| 871 | - | repo := NewRepository(privateDir(t)) | |||
| 872 | - | key := sessionKey("named") | |||
| 873 | - | publications := make([]ports.SnapshotPublication, 0, maintenanceBatch+2) | |||
| 874 | - | for generation := uint64(1); generation <= maintenanceBatch+2; generation++ { | |||
| 875 | - | publication := repositoryPublication(t, "named", generation, []byte(fmt.Sprintf("state-%d", generation))) | |||
| 876 | - | if err := repo.Publish(context.Background(), publication); err != nil { | |||
| 877 | - | t.Fatal(err) | |||
| 878 | - | } | |||
| 879 | - | publications = append(publications, publication) | |||
| 880 | - | } | |||
| 881 | - | head.set(t, repo, key) | |||
| 882 | - | stale := sha256.Sum256([]byte("unpublished stale object")) | |||
| 883 | - | stalePath := repo.objectPath(key, stale) | |||
| 884 | - | if err := os.MkdirAll(filepath.Dir(stalePath), 0o700); err != nil { | |||
| 885 | - | t.Fatal(err) | |||
| 886 | - | } | |||
| 887 | - | if err := os.WriteFile(stalePath, []byte("unpublished stale object"), 0o600); err != nil { | |||
| 888 | - | t.Fatal(err) | |||
| 889 | - | } | |||
| 890 | - | ||||
| 891 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 892 | - | t.Fatal(err) | |||
| 893 | - | } | |||
| 894 | - | if _, err := os.Lstat(stalePath); err != nil { | |||
| 895 | - | t.Fatalf("stale object swept before full manifest classification: %v", err) | |||
| 896 | - | } | |||
| 897 | - | for _, publication := range publications { | |||
| 898 | - | if _, err := os.Lstat(repo.objectPath(key, publication.Objects[0].Digest)); err != nil { | |||
| 899 | - | t.Fatalf("object for unpublished generation %d removed early: %v", publication.Generation, err) | |||
| 900 | - | } | |||
| 901 | - | } | |||
| 902 | - | ||||
| 903 | - | for i := 0; i < 600; i++ { | |||
| 904 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 905 | - | t.Fatal(err) | |||
| 906 | - | } | |||
| 907 | - | if _, err := os.Lstat(stalePath); errors.Is(err, os.ErrNotExist) { | |||
| 908 | - | break | |||
| 909 | - | } else if i == 599 { | |||
| 910 | - | t.Fatal("stale object was not swept after complete classification") | |||
| 911 | - | } | |||
| 912 | - | } | |||
| 913 | - | for _, publication := range publications { | |||
| 914 | - | if _, err := os.Lstat(repo.objectPath(key, publication.Objects[0].Digest)); err != nil { | |||
| 915 | - | t.Fatalf("object for unpublished generation %d removed: %v", publication.Generation, err) | |||
| 916 | - | } | |||
| 917 | - | } | |||
| 918 | - | }) | |||
| 919 | - | } | |||
| 920 | - | } | |||
| 921 | - | ||||
| 922 | - | func TestRepositoryMaintainRestartsMarkAfterFailedPublication(t *testing.T) { | |||
| 923 | - | repo := NewRepository(privateDir(t)) | |||
| 924 | - | key := sessionKey("named") | |||
| 925 | - | first := publicationWithTailShard(t, "named", 1, "ff") | |||
| 926 | - | stalePaths := make([]string, 0, maintenanceBatch+1) | |||
| 927 | - | for i := 0; i < maintenanceBatch+1; i++ { | |||
| 928 | - | digest := digestInShard(t, "00", i) | |||
| 929 | - | path := repo.objectPath(key, digest) | |||
| 930 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | |||
| 931 | - | t.Fatal(err) | |||
| 932 | - | } | |||
| 933 | - | if err := os.WriteFile(path, []byte("stale"), 0o600); err != nil { | |||
| 934 | - | t.Fatal(err) | |||
| 935 | - | } | |||
| 936 | - | stalePaths = append(stalePaths, path) | |||
| 937 | - | } | |||
| 938 | - | if err := repo.Publish(context.Background(), first); err != nil { | |||
| 939 | - | t.Fatal(err) | |||
| 940 | - | } | |||
| 941 | - | ||||
| 942 | - | // This completes marking but exhausts the sweep budget in the stale shard, | |||
| 943 | - | // leaving the next sweep batch pending. | |||
| 944 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 945 | - | t.Fatal(err) | |||
| 946 | - | } | |||
| 947 | - | staleRemaining := false | |||
| 948 | - | for _, path := range stalePaths { | |||
| 949 | - | if _, err := os.Lstat(path); err == nil { | |||
| 950 | - | staleRemaining = true | |||
| 951 | - | break | |||
| 952 | - | } | |||
| 953 | - | } | |||
| 954 | - | if !staleRemaining { | |||
| 955 | - | t.Fatal("sweep did not leave a pending stale object") | |||
| 956 | - | } | |||
| 957 | - | ||||
| 958 | - | second := publicationWithTailShard(t, "named", 2, "ff") | |||
| 959 | - | injected := errors.New("before HEAD") | |||
| 960 | - | repo.hooks.beforeHeadWrite = func(string) error { return injected } | |||
| 961 | - | if err := repo.Publish(context.Background(), second); !errors.Is(err, injected) { | |||
| 962 | - | t.Fatalf("Publish error = %v, want injected failure", err) | |||
| 963 | - | } | |||
| 964 | - | repo.hooks.beforeHeadWrite = nil | |||
| 965 | - | ||||
| 966 | - | for i := 0; i < 20; i++ { | |||
| 967 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 968 | - | t.Fatal(err) | |||
| 969 | - | } | |||
| 970 | - | for _, object := range second.Objects { | |||
| 971 | - | if _, err := os.Lstat(repo.objectPath(key, object.Digest)); err != nil { | |||
| 972 | - | t.Fatalf("object referenced by failed publication removed on pass %d: %v", i, err) | |||
| 973 | - | } | |||
| 974 | - | } | |||
| 975 | - | staleRemaining = false | |||
| 976 | - | for _, path := range stalePaths { | |||
| 977 | - | if _, err := os.Lstat(path); err == nil { | |||
| 978 | - | staleRemaining = true | |||
| 979 | - | break | |||
| 980 | - | } | |||
| 981 | - | } | |||
| 982 | - | } | |||
| 983 | - | if staleRemaining { | |||
| 984 | - | t.Fatal("stale object was not eventually swept") | |||
| 985 | - | } | |||
| 986 | - | } | |||
| 987 | - | ||||
| 988 | - | func publicationWithTailShard(t *testing.T, name string, generation uint64, shard string) ports.SnapshotPublication { | |||
| 989 | - | t.Helper() | |||
| 990 | - | for i := 0; i < 4096; i++ { | |||
| 991 | - | publication := repositoryPublication(t, name, generation, []byte(fmt.Sprintf("state-%d-%d", generation, i))) | |||
| 992 | - | if fmt.Sprintf("%02x", publication.Objects[0].Digest[0]) == shard { | |||
| 993 | - | return publication | |||
| 994 | - | } | |||
| 995 | - | } | |||
| 996 | - | t.Fatalf("did not find publication tail in shard %q", shard) | |||
| 997 | - | return ports.SnapshotPublication{} | |||
| 998 | - | } | |||
| 999 | - | ||||
| 1000 | - | func digestInShard(t *testing.T, shard string, n int) ports.SnapshotDigest { | |||
| 1001 | - | t.Helper() | |||
| 1002 | - | for i := 0; i < 4096; i++ { | |||
| 1003 | - | digest := sha256.Sum256([]byte(fmt.Sprintf("stale-%d-%d", n, i))) | |||
| 1004 | - | if fmt.Sprintf("%02x", digest[0]) == shard { | |||
| 1005 | - | return digest | |||
| 1006 | - | } | |||
| 1007 | - | } | |||
| 1008 | - | t.Fatalf("did not find stale digest in shard %q", shard) | |||
| 1009 | - | return ports.SnapshotDigest{} | |||
| 1010 | - | } | |||
D internal/adapters/snapshot/offline_kill_tombstone_test.go +0 -53
| @@ -1,53 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "os" | |||
| 6 | - | "path/filepath" | |||
| 7 | - | "testing" | |||
| 8 | - | ||||
| 9 | - | "github.com/stretchr/testify/require" | |||
| 10 | - | ) | |||
| 11 | - | ||||
| 12 | - | func TestRepositoryTombstoneBlocksIncrementalRestoreAndLegacyImport(t *testing.T) { | |||
| 13 | - | repo := NewRepository(privateDir(t)) | |||
| 14 | - | require.NoError(t, repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state")))) | |||
| 15 | - | require.NoError(t, os.WriteFile(filepath.Join(repo.dir, filenameForName("named")), []byte("legacy"), 0o600)) | |||
| 16 | - | ||||
| 17 | - | require.NoError(t, repo.Tombstone(context.Background(), "named")) | |||
| 18 | - | ||||
| 19 | - | names, err := repo.List(context.Background()) | |||
| 20 | - | require.NoError(t, err) | |||
| 21 | - | require.NotContains(t, names, "named") | |||
| 22 | - | legacy, err := repo.LoadLegacy(context.Background()) | |||
| 23 | - | require.NoError(t, err) | |||
| 24 | - | for _, snapshot := range legacy { | |||
| 25 | - | require.NotEqual(t, "named", snapshot.Name) | |||
| 26 | - | } | |||
| 27 | - | } | |||
| 28 | - | ||||
| 29 | - | func TestRepositoryDeleteLegacyRetriesDurabilityAfterTombstone(t *testing.T) { | |||
| 30 | - | repo := NewRepository(privateDir(t)) | |||
| 31 | - | require.NoError(t, os.Mkdir(repo.dir, 0o700)) | |||
| 32 | - | require.NoError(t, os.WriteFile(filepath.Join(repo.dir, filenameForName("named")), []byte("legacy"), 0o600)) | |||
| 33 | - | require.NoError(t, repo.Tombstone(context.Background(), "named")) | |||
| 34 | - | ||||
| 35 | - | calls := 0 | |||
| 36 | - | repo.hooks.syncDirectory = func(string) error { | |||
| 37 | - | calls++ | |||
| 38 | - | if calls == 1 { | |||
| 39 | - | return os.ErrPermission | |||
| 40 | - | } | |||
| 41 | - | return nil | |||
| 42 | - | } | |||
| 43 | - | require.ErrorIs(t, repo.DeleteLegacy(context.Background(), "named"), os.ErrPermission) | |||
| 44 | - | ||||
| 45 | - | // A retry after process restart has no in-memory pending-sync state. | |||
| 46 | - | repo = NewRepository(repo.dir) | |||
| 47 | - | repo.hooks.syncDirectory = func(string) error { | |||
| 48 | - | calls++ | |||
| 49 | - | return nil | |||
| 50 | - | } | |||
| 51 | - | require.NoError(t, repo.DeleteLegacy(context.Background(), "named")) | |||
| 52 | - | require.Equal(t, 2, calls, "explicit purge retries only the pending source-directory sync") | |||
| 53 | - | } | |||
M internal/adapters/snapshot/path.go +14 -36
| @@ -1,61 +1,39 @@ | |||||
| 1 | package snapshot | 1 | package snapshot | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | - | "crypto/sha256" | |||
| 5 | "encoding/hex" | 4 | "encoding/hex" | ||
| 6 | "fmt" | 5 | "fmt" | ||
| 7 | "path/filepath" | 6 | "path/filepath" | ||
| 8 | - | "regexp" | |||
| 9 | "strconv" | 7 | "strconv" | ||
| 10 | "strings" | 8 | "strings" | ||
| 11 | 9 | ||||
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 12 | "github.com/bnema/vev/internal/ports" | 11 | "github.com/bnema/vev/internal/ports" | ||
| 13 | ) | 12 | ) | ||
| 14 | 13 | ||||
| 15 | - | var safeNameRE = regexp.MustCompile(`^[A-Za-z0-9._-]{1,200}$`) | |||
| 16 | - | ||||
| 17 | const maxSnapshotFileSize = 16 + (256 << 20) | 14 | const maxSnapshotFileSize = 16 + (256 << 20) | ||
| 18 | 15 | ||||
| 19 | - | // filenameForName maps a session name to its deterministic legacy filename. | 16 | + | func incarnationKey(id domain.IncarnationID) (string, error) { |
| 20 | - | // It remains only for the one-way v3 import path. | 17 | + | if id == (domain.IncarnationID{}) { |
| 21 | - | func filenameForName(name string) string { | 18 | + | return "", fmt.Errorf("snapshot: zero incarnation ID") |
| 22 | - | if safeNameRE.MatchString(name) { | |||
| 23 | - | return name + ".snap" | |||
| 24 | } | 19 | } | ||
| 25 | - | sum := sha256.Sum256([]byte(name)) | 20 | + | return id.String(), nil |
| 26 | - | return "@" + hex.EncodeToString(sum[:])[:40] + ".snap" | |||
| 27 | } | 21 | } | ||
| 28 | 22 | ||||
| 29 | - | func (r *Repository) sessionPath(key string) string { | 23 | + | func (r *Repository) sessionPath(id domain.IncarnationID) string { |
| 30 | - | return filepath.Join(r.dir, repositorySessionsDir, key) | 24 | + | return filepath.Join(r.dir, repositorySessionsDir, id.String()) |
| 31 | } | 25 | } | ||
| 32 | - | func (r *Repository) objectPath(key string, digest ports.SnapshotDigest) string { | 26 | + | func (r *Repository) objectPath(id domain.IncarnationID, digest ports.SnapshotDigest) string { |
| 33 | hexDigest := hex.EncodeToString(digest[:]) | 27 | hexDigest := hex.EncodeToString(digest[:]) | ||
| 34 | - | return filepath.Join(r.sessionPath(key), repositoryObjectsDir, hexDigest[:2], hexDigest) | 28 | + | return filepath.Join(r.sessionPath(id), repositoryObjectsDir, hexDigest[:2], hexDigest) |
| 35 | - | } | |||
| 36 | - | func (r *Repository) manifestPath(key string, generation uint64) string { | |||
| 37 | - | return filepath.Join(r.sessionPath(key), repositoryGenerations, generationFilename(generation)) | |||
| 38 | } | 29 | } | ||
| 39 | - | func (r *Repository) headPath(key string) string { | 30 | + | func (r *Repository) manifestPath(id domain.IncarnationID, generation uint64) string { |
| 40 | - | return filepath.Join(r.sessionPath(key), repositoryHead) | 31 | + | return filepath.Join(r.sessionPath(id), repositoryGenerations, generationFilename(generation)) |
| 41 | } | 32 | } | ||
| 42 | - | func sessionKey(name string) string { | 33 | + | func (r *Repository) headPath(id domain.IncarnationID) string { |
| 43 | - | if safeNameRE.MatchString(name) && name != "." && name != ".." { | 34 | + | return filepath.Join(r.sessionPath(id), repositoryHead) |
| 44 | - | return name | |||
| 45 | - | } | |||
| 46 | - | sum := sha256.Sum256([]byte(name)) | |||
| 47 | - | return "@" + hex.EncodeToString(sum[:]) | |||
| 48 | - | } | |||
| 49 | - | func canonicalSessionKey(key string) bool { | |||
| 50 | - | if safeNameRE.MatchString(key) && key != "." && key != ".." { | |||
| 51 | - | return true | |||
| 52 | - | } | |||
| 53 | - | if len(key) != 65 || key[0] != '@' { | |||
| 54 | - | return false | |||
| 55 | - | } | |||
| 56 | - | _, err := hex.DecodeString(key[1:]) | |||
| 57 | - | return err == nil && strings.ToLower(key[1:]) == key[1:] | |||
| 58 | } | 35 | } | ||
| 36 | + | ||||
| 59 | func generationFilename(generation uint64) string { | 37 | func generationFilename(generation uint64) string { | ||
| 60 | return fmt.Sprintf("%0*d.manifest", generationWidth, generation) | 38 | return fmt.Sprintf("%0*d.manifest", generationWidth, generation) | ||
| 61 | } | 39 | } | ||
M internal/adapters/snapshot/path_security_test.go +15 -105
| @@ -3,12 +3,14 @@ package snapshot | |||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | "errors" | 5 | "errors" | ||
| 6 | - | "fmt" | |||
| 7 | "os" | 6 | "os" | ||
| 8 | "path/filepath" | 7 | "path/filepath" | ||
| 9 | "strings" | 8 | "strings" | ||
| 10 | "syscall" | 9 | "syscall" | ||
| 11 | "testing" | 10 | "testing" | ||
| 11 | + | ||||
| 12 | + | "github.com/bnema/vev/internal/domain" | |||
| 13 | + | codec "github.com/bnema/vev/internal/usecase/snapshot" | |||
| 12 | ) | 14 | ) | ||
| 13 | 15 | ||||
| 14 | func TestRepositoryRejectsSymlinkedRoot(t *testing.T) { | 16 | func TestRepositoryRejectsSymlinkedRoot(t *testing.T) { | ||
| @@ -39,13 +41,14 @@ func TestRepositoryRejectsSymlinkedRoot(t *testing.T) { | |||||
| 39 | t.Fatal(err) | 41 | t.Fatal(err) | ||
| 40 | } | 42 | } | ||
| 41 | 43 | ||||
| 42 | - | if _, err := repo.Load(context.Background(), publication.Name); err == nil { | 44 | + | if _, err := loadPublication(context.Background(), repo, publication); err == nil { |
| 43 | - | t.Fatal("Load succeeded through replaced configured-root symlink") | 45 | + | t.Fatal("LoadCheckpoint succeeded through replaced configured-root symlink") |
| 44 | } | 46 | } | ||
| 45 | - | if err := repo.Maintain(context.Background()); err == nil { | 47 | + | keep := map[domain.IncarnationID]domain.CheckpointRef{publication.IncarnationID: {Generation: publication.Generation, ManifestDigest: codec.ManifestDigest(publication.Manifest)}} |
| 46 | - | t.Fatal("Maintain succeeded through replaced configured-root symlink") | 48 | + | if err := repo.CollectGarbage(context.Background(), keep); err == nil { |
| 49 | + | t.Fatal("CollectGarbage succeeded through replaced configured-root symlink") | |||
| 47 | } | 50 | } | ||
| 48 | - | loaded, err := external.Load(context.Background(), publication.Name) | 51 | + | loaded, err := loadPublication(context.Background(), external, externalPublication) |
| 49 | if err != nil { | 52 | if err != nil { | ||
| 50 | t.Fatal(err) | 53 | t.Fatal(err) | ||
| 51 | } | 54 | } | ||
| @@ -67,84 +70,6 @@ func TestOpenRootRejectsUnsafePinnedRoot(t *testing.T) { | |||||
| 67 | } | 70 | } | ||
| 68 | } | 71 | } | ||
| 69 | 72 | ||||
| 70 | - | func TestOpenRootDetectsReplacementAfterOpen(t *testing.T) { | |||
| 71 | - | dir := privateDir(t) | |||
| 72 | - | if err := os.Mkdir(dir, 0o700); err != nil { | |||
| 73 | - | t.Fatal(err) | |||
| 74 | - | } | |||
| 75 | - | old := dir + "-old" | |||
| 76 | - | replacementMarker := filepath.Join(dir, "replacement-marker") | |||
| 77 | - | repo := NewRepository(dir) | |||
| 78 | - | repo.hooks.afterOpenRoot = func() { | |||
| 79 | - | repo.hooks.afterOpenRoot = nil | |||
| 80 | - | if err := os.Rename(dir, old); err != nil { | |||
| 81 | - | t.Fatal(err) | |||
| 82 | - | } | |||
| 83 | - | if err := os.Mkdir(dir, 0o700); err != nil { | |||
| 84 | - | t.Fatal(err) | |||
| 85 | - | } | |||
| 86 | - | if err := os.WriteFile(replacementMarker, []byte("unchanged"), 0o600); err != nil { | |||
| 87 | - | t.Fatal(err) | |||
| 88 | - | } | |||
| 89 | - | } | |||
| 90 | - | if _, err := repo.openRoot(); !errors.Is(err, syscall.ESTALE) { | |||
| 91 | - | t.Fatalf("openRoot error = %v, want ESTALE", err) | |||
| 92 | - | } | |||
| 93 | - | if got, err := os.ReadFile(replacementMarker); err != nil || string(got) != "unchanged" { | |||
| 94 | - | t.Fatalf("replacement marker = %q, %v", got, err) | |||
| 95 | - | } | |||
| 96 | - | if _, err := os.Stat(filepath.Join(old, "replacement-marker")); !errors.Is(err, os.ErrNotExist) { | |||
| 97 | - | t.Fatalf("pinned tree was operated on: %v", err) | |||
| 98 | - | } | |||
| 99 | - | } | |||
| 100 | - | ||||
| 101 | - | func TestOpenDirectoryClosesChildWhenRootCloseFails(t *testing.T) { | |||
| 102 | - | repo := NewRepository(privateDir(t)) | |||
| 103 | - | if err := os.Mkdir(repo.dir, 0o700); err != nil { | |||
| 104 | - | t.Fatal(err) | |||
| 105 | - | } | |||
| 106 | - | child := filepath.Join(repo.dir, "child") | |||
| 107 | - | if err := os.Mkdir(child, 0o700); err != nil { | |||
| 108 | - | t.Fatal(err) | |||
| 109 | - | } | |||
| 110 | - | closeErr := errors.New("injected root close") | |||
| 111 | - | repo.hooks.closeRoot = func() error { return closeErr } | |||
| 112 | - | before := openDescriptorCount(t) | |||
| 113 | - | for range 20 { | |||
| 114 | - | file, err := repo.openDirectory(child) | |||
| 115 | - | if file != nil || !errors.Is(err, closeErr) { | |||
| 116 | - | t.Fatalf("openDirectory = (%v, %v), want nil and close error", file, err) | |||
| 117 | - | } | |||
| 118 | - | } | |||
| 119 | - | if got := openDescriptorCount(t); got > before+4 { | |||
| 120 | - | t.Fatalf("open descriptors after failed openDirectory = %d, before = %d", got, before) | |||
| 121 | - | } | |||
| 122 | - | } | |||
| 123 | - | ||||
| 124 | - | func TestRootCloseErrorsJoinAndCloseRealRoot(t *testing.T) { | |||
| 125 | - | repo := NewRepository(privateDir(t)) | |||
| 126 | - | if err := os.Mkdir(repo.dir, 0o700); err != nil { | |||
| 127 | - | t.Fatal(err) | |||
| 128 | - | } | |||
| 129 | - | closeErr := errors.New("injected root close") | |||
| 130 | - | repo.hooks.closeRoot = func() error { return closeErr } | |||
| 131 | - | root, err := repo.openRoot() | |||
| 132 | - | if err != nil { | |||
| 133 | - | t.Fatal(err) | |||
| 134 | - | } | |||
| 135 | - | if err := repo.closeRoot(root); !errors.Is(err, closeErr) { | |||
| 136 | - | t.Fatalf("closeRoot error = %v, want injected error", err) | |||
| 137 | - | } | |||
| 138 | - | if _, err := root.Stat("."); err == nil { | |||
| 139 | - | t.Fatal("real root remained open after injected close error") | |||
| 140 | - | } | |||
| 141 | - | ||||
| 142 | - | _, err = repo.stat(filepath.Join(repo.dir, "missing")) | |||
| 143 | - | if !errors.Is(err, closeErr) || !errors.Is(err, os.ErrNotExist) { | |||
| 144 | - | t.Fatalf("stat error = %v, want primary and close errors", err) | |||
| 145 | - | } | |||
| 146 | - | } | |||
| 147 | - | ||||
| 148 | func TestRepositoryRejectsSymlinkedGenerationAndObjectShards(t *testing.T) { | 73 | func TestRepositoryRejectsSymlinkedGenerationAndObjectShards(t *testing.T) { | ||
| 149 | for _, target := range []string{repositoryGenerations, repositoryObjectsDir} { | 74 | for _, target := range []string{repositoryGenerations, repositoryObjectsDir} { | ||
| 150 | t.Run(target, func(t *testing.T) { | 75 | t.Run(target, func(t *testing.T) { | ||
| @@ -153,8 +78,7 @@ func TestRepositoryRejectsSymlinkedGenerationAndObjectShards(t *testing.T) { | |||||
| 153 | if err := repo.Publish(context.Background(), publication); err != nil { | 78 | if err := repo.Publish(context.Background(), publication); err != nil { | ||
| 154 | t.Fatal(err) | 79 | t.Fatal(err) | ||
| 155 | } | 80 | } | ||
| 156 | - | key := sessionKey(publication.Name) | 81 | + | inside := filepath.Join(repo.sessionPath(publication.IncarnationID), target) |
| 157 | - | inside := filepath.Join(repo.sessionPath(key), target) | |||
| 158 | outside := t.TempDir() | 82 | outside := t.TempDir() | ||
| 159 | guard := filepath.Join(outside, "must-not-change") | 83 | guard := filepath.Join(outside, "must-not-change") | ||
| 160 | if err := os.WriteFile(guard, []byte("guard"), 0o600); err != nil { | 84 | if err := os.WriteFile(guard, []byte("guard"), 0o600); err != nil { | ||
| @@ -166,11 +90,12 @@ func TestRepositoryRejectsSymlinkedGenerationAndObjectShards(t *testing.T) { | |||||
| 166 | if err := os.Symlink(outside, inside); err != nil { | 90 | if err := os.Symlink(outside, inside); err != nil { | ||
| 167 | t.Fatal(err) | 91 | t.Fatal(err) | ||
| 168 | } | 92 | } | ||
| 169 | - | if _, err := repo.Load(context.Background(), publication.Name); err == nil { | 93 | + | if _, err := loadPublication(context.Background(), repo, publication); err == nil { |
| 170 | - | t.Fatal("Load succeeded through symlinked repository component") | 94 | + | t.Fatal("LoadCheckpoint succeeded through symlinked repository component") |
| 171 | } | 95 | } | ||
| 172 | - | if err := repo.Maintain(context.Background()); err == nil { | 96 | + | keep := map[domain.IncarnationID]domain.CheckpointRef{publication.IncarnationID: {Generation: publication.Generation, ManifestDigest: codec.ManifestDigest(publication.Manifest)}} |
| 173 | - | t.Fatal("Maintain succeeded through symlinked repository component") | 97 | + | if err := repo.CollectGarbage(context.Background(), keep); err == nil { |
| 98 | + | t.Fatal("CollectGarbage succeeded through symlinked repository component") | |||
| 174 | } | 99 | } | ||
| 175 | got, err := os.ReadFile(guard) | 100 | got, err := os.ReadFile(guard) | ||
| 176 | if err != nil || string(got) != "guard" { | 101 | if err != nil || string(got) != "guard" { | ||
| @@ -200,18 +125,3 @@ func TestFinalSymlinksRejectedByRootOperations(t *testing.T) { | |||||
| 200 | t.Fatalf("readBounded error = %v, want sanitized ELOOP", err) | 125 | t.Fatalf("readBounded error = %v, want sanitized ELOOP", err) | ||
| 201 | } | 126 | } | ||
| 202 | } | 127 | } | ||
| 203 | - | ||||
| 204 | - | func TestRepositoryLoadLegacyChargesUnrelatedRootEntries(t *testing.T) { | |||
| 205 | - | repo := NewRepository(privateDir(t)) | |||
| 206 | - | if err := os.MkdirAll(repo.dir, 0o700); err != nil { | |||
| 207 | - | t.Fatal(err) | |||
| 208 | - | } | |||
| 209 | - | for i := 0; i <= maxDirectoryTraversalEntries; i++ { | |||
| 210 | - | if err := os.WriteFile(filepath.Join(repo.dir, fmt.Sprintf("unrelated-%05d", i)), nil, 0o600); err != nil { | |||
| 211 | - | t.Fatal(err) | |||
| 212 | - | } | |||
| 213 | - | } | |||
| 214 | - | if _, err := repo.LoadLegacy(context.Background()); !errors.Is(err, ErrDirectoryTraversalBudget) { | |||
| 215 | - | t.Fatalf("LoadLegacy error = %v, want traversal budget error", err) | |||
| 216 | - | } | |||
| 217 | - | } | |||
M internal/adapters/snapshot/publish.go +83 -81
| @@ -4,8 +4,11 @@ import ( | |||||
| 4 | "bytes" | 4 | "bytes" | ||
| 5 | "context" | 5 | "context" | ||
| 6 | "crypto/sha256" | 6 | "crypto/sha256" | ||
| 7 | + | "errors" | |||
| 7 | "fmt" | 8 | "fmt" | ||
| 9 | + | "os" | |||
| 8 | 10 | ||||
| 11 | + | "github.com/bnema/vev/internal/domain" | |||
| 9 | "github.com/bnema/vev/internal/ports" | 12 | "github.com/bnema/vev/internal/ports" | ||
| 10 | codec "github.com/bnema/vev/internal/usecase/snapshot" | 13 | codec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 11 | ) | 14 | ) | ||
| @@ -14,7 +17,10 @@ func (r *Repository) Publish(ctx context.Context, publication ports.SnapshotPubl | |||||
| 14 | if err := ctx.Err(); err != nil { | 17 | if err := ctx.Err(); err != nil { | ||
| 15 | return err | 18 | return err | ||
| 16 | } | 19 | } | ||
| 17 | - | key := sessionKey(publication.Name) | 20 | + | key, err := incarnationKey(publication.IncarnationID) |
| 21 | + | if err != nil { | |||
| 22 | + | return err | |||
| 23 | + | } | |||
| 18 | lock := r.lockSession(key) | 24 | lock := r.lockSession(key) | ||
| 19 | defer r.unlockSession(lock) | 25 | defer r.unlockSession(lock) | ||
| 20 | if err := ctx.Err(); err != nil { | 26 | if err := ctx.Err(); err != nil { | ||
| @@ -28,20 +34,16 @@ func (r *Repository) Publish(ctx context.Context, publication ports.SnapshotPubl | |||||
| 28 | if err := ctx.Err(); err != nil { | 34 | if err := ctx.Err(); err != nil { | ||
| 29 | return err | 35 | return err | ||
| 30 | } | 36 | } | ||
| 31 | - | // A failed publication can still leave immutable blobs or a manifest behind. | 37 | + | // A replay may find a forward orphan left after publication but before its |
| 32 | - | // Invalidate an in-progress GC mark before creating any publication storage. | 38 | + | // catalogue commit. Accept it only after the canonical checkpoint loader has |
| 33 | - | r.invalidateStorageEpoch(key) | 39 | + | // revalidated the pointer digest, manifest, and every referenced object. |
| 34 | - | if err := r.ensureSession(key); err != nil { | 40 | + | if unchanged, err := r.unchangedPublication(ctx, publication); err != nil { |
| 35 | return err | 41 | return err | ||
| 36 | - | } | 42 | + | } else if unchanged { |
| 37 | - | // The common replay path needs neither a manifest decode nor a reference | |||
| 38 | - | // map. Checking the immutable bytes directly also keeps its descriptor | |||
| 39 | - | // reads beneath the pinned repository root. | |||
| 40 | - | if unchanged, err := r.unchangedPublication(key, publication); err == nil && unchanged { | |||
| 41 | return nil | 43 | return nil | ||
| 42 | } | 44 | } | ||
| 43 | 45 | ||||
| 44 | - | current, currentManifest, currentRefs, err := r.currentPublication(ctx, publication.Name, key) | 46 | + | current, currentManifest, currentRefs, err := r.currentIncarnationPublication(ctx, publication) |
| 45 | if err != nil { | 47 | if err != nil { | ||
| 46 | return err | 48 | return err | ||
| 47 | } | 49 | } | ||
| @@ -55,6 +57,13 @@ func (r *Repository) Publish(ctx context.Context, publication ports.SnapshotPubl | |||||
| 55 | return nil | 57 | return nil | ||
| 56 | } | 58 | } | ||
| 57 | 59 | ||||
| 60 | + | // Parent validation and all other publication checks above happen before | |||
| 61 | + | // creating repository state. A rejected child must leave the authoritative | |||
| 62 | + | // checkpoint and its storage byte-for-byte unchanged. | |||
| 63 | + | if err := r.ensureSession(publication.IncarnationID); err != nil { | |||
| 64 | + | return err | |||
| 65 | + | } | |||
| 66 | + | ||||
| 58 | // Retained refs from the authoritative generation were verified when their | 67 | // Retained refs from the authoritative generation were verified when their | ||
| 59 | // immutable files were installed. They are intentionally not re-read when a | 68 | // immutable files were installed. They are intentionally not re-read when a | ||
| 60 | // producer supplies them again. Every other referenced object is checked | 69 | // producer supplies them again. Every other referenced object is checked | ||
| @@ -70,7 +79,7 @@ func (r *Repository) Publish(ctx context.Context, publication ports.SnapshotPubl | |||||
| 70 | if err := ctx.Err(); err != nil { | 79 | if err := ctx.Err(); err != nil { | ||
| 71 | return err | 80 | return err | ||
| 72 | } | 81 | } | ||
| 73 | - | path := r.objectPath(key, digest) | 82 | + | path := r.objectPath(publication.IncarnationID, digest) |
| 74 | exists, err := r.verifyObjectFile(path, digest, ref) | 83 | exists, err := r.verifyObjectFile(path, digest, ref) | ||
| 75 | if err != nil { | 84 | if err != nil { | ||
| 76 | return fmt.Errorf("verify object %x: %w", digest, err) | 85 | return fmt.Errorf("verify object %x: %w", digest, err) | ||
| @@ -86,11 +95,6 @@ func (r *Repository) Publish(ctx context.Context, publication ports.SnapshotPubl | |||||
| 86 | if err != nil { | 95 | if err != nil { | ||
| 87 | return err | 96 | return err | ||
| 88 | } | 97 | } | ||
| 89 | - | if r.hooks.beforeBlobWrite != nil { | |||
| 90 | - | if err := r.hooks.beforeBlobWrite(path); err != nil { | |||
| 91 | - | return err | |||
| 92 | - | } | |||
| 93 | - | } | |||
| 94 | if err := ctx.Err(); err != nil { | 98 | if err := ctx.Err(); err != nil { | ||
| 95 | return err | 99 | return err | ||
| 96 | } | 100 | } | ||
| @@ -104,7 +108,7 @@ func (r *Repository) Publish(ctx context.Context, publication ports.SnapshotPubl | |||||
| 104 | } | 108 | } | ||
| 105 | } | 109 | } | ||
| 106 | 110 | ||||
| 107 | - | manifestPath := r.manifestPath(key, publication.Generation) | 111 | + | manifestPath := r.manifestPath(publication.IncarnationID, publication.Generation) |
| 108 | existing, exists, err := r.readOptionalBounded(manifestPath) | 112 | existing, exists, err := r.readOptionalBounded(manifestPath) | ||
| 109 | if err != nil { | 113 | if err != nil { | ||
| 110 | return err | 114 | return err | ||
| @@ -114,11 +118,6 @@ func (r *Repository) Publish(ctx context.Context, publication ports.SnapshotPubl | |||||
| 114 | return fmt.Errorf("manifest generation %d: immutable conflict", publication.Generation) | 118 | return fmt.Errorf("manifest generation %d: immutable conflict", publication.Generation) | ||
| 115 | } | 119 | } | ||
| 116 | } else { | 120 | } else { | ||
| 117 | - | if r.hooks.beforeManifestWrite != nil { | |||
| 118 | - | if err := r.hooks.beforeManifestWrite(manifestPath); err != nil { | |||
| 119 | - | return err | |||
| 120 | - | } | |||
| 121 | - | } | |||
| 122 | if err := ctx.Err(); err != nil { | 121 | if err := ctx.Err(); err != nil { | ||
| 123 | return err | 122 | return err | ||
| 124 | } | 123 | } | ||
| @@ -131,63 +130,78 @@ func (r *Repository) Publish(ctx context.Context, publication ports.SnapshotPubl | |||||
| 131 | return fmt.Errorf("write manifest: %w", err) | 130 | return fmt.Errorf("write manifest: %w", err) | ||
| 132 | } | 131 | } | ||
| 133 | } | 132 | } | ||
| 134 | - | if r.hooks.beforeHeadWrite != nil { | |||
| 135 | - | if err := r.hooks.beforeHeadWrite(r.headPath(key)); err != nil { | |||
| 136 | - | return err | |||
| 137 | - | } | |||
| 138 | - | } | |||
| 139 | if err := ctx.Err(); err != nil { | 133 | if err := ctx.Err(); err != nil { | ||
| 140 | return err | 134 | return err | ||
| 141 | } | 135 | } | ||
| 142 | - | if err := r.writeMutable(r.headPath(key), marshalHead(publication.Generation, sha256.Sum256(publication.Manifest))); err != nil { | 136 | + | if err := r.writeMutable(r.headPath(publication.IncarnationID), marshalHead(publication.Generation, sha256.Sum256(publication.Manifest))); err != nil { |
| 143 | return fmt.Errorf("write HEAD: %w", err) | 137 | return fmt.Errorf("write HEAD: %w", err) | ||
| 144 | } | 138 | } | ||
| 145 | return nil | 139 | return nil | ||
| 146 | } | 140 | } | ||
| 147 | 141 | ||||
| 148 | - | // unchangedPublication uses one root for the two reads in the replay fast | 142 | + | // unchangedPublication validates an existing forward orphan through the same |
| 149 | - | // path. The root is confined to this operation and is closed before return; | 143 | + | // loader used for recovery. A corrupt replay must not be accepted merely |
| 150 | - | // all other repository operations retain their open-per-operation roots. | 144 | + | // because its manifest bytes happen to match the requested publication. |
| 151 | - | func (r *Repository) unchangedPublication(key string, publication ports.SnapshotPublication) (unchanged bool, err error) { | 145 | + | func (r *Repository) unchangedPublication(ctx context.Context, publication ports.SnapshotPublication) (bool, error) { |
| 152 | - | root, err := r.openRoot() | 146 | + | generation, digest, err := r.readHead(publication.IncarnationID) |
| 153 | - | if err != nil { | 147 | + | _ = err // A missing or unreadable HEAD is not treated as a replay. |
| 154 | - | return false, err | |||
| 155 | - | } | |||
| 156 | - | defer func() { joinCloseError(&err, "close snapshot root", r.closeRoot(root)) }() | |||
| 157 | - | ||||
| 158 | - | generation, _, err := r.readHeadWithRoot(root, key) | |||
| 159 | if err != nil || generation != publication.Generation { | 148 | if err != nil || generation != publication.Generation { | ||
| 160 | return false, nil | 149 | return false, nil | ||
| 161 | } | 150 | } | ||
| 162 | - | current, err := r.readBoundedRoot(root, r.manifestPath(key, generation)) | 151 | + | current, _, err := r.loadCheckpointLocked(ctx, publication.IncarnationID, publication.Name, ports.CheckpointRef{ |
| 163 | - | return err == nil && bytes.Equal(current, publication.Manifest), nil | 152 | + | Generation: generation, |
| 153 | + | ManifestDigest: digest, | |||
| 154 | + | }) | |||
| 155 | + | if err != nil { | |||
| 156 | + | return false, fmt.Errorf("validate replayed checkpoint: %w", err) | |||
| 157 | + | } | |||
| 158 | + | return bytes.Equal(current.Manifest, publication.Manifest), nil | |||
| 164 | } | 159 | } | ||
| 165 | 160 | ||||
| 166 | - | // currentPublication takes the normal HEAD path without loading every object | 161 | + | func (r *Repository) currentIncarnationPublication(ctx context.Context, publication ports.SnapshotPublication) (uint64, []byte, map[ports.SnapshotDigest]codec.ObjectRef, error) { |
| 167 | - | // in the current generation. A damaged HEAD or manifest still uses the full | 162 | + | generation, digest, err := r.readHead(publication.IncarnationID) |
| 168 | - | // fallback scan, because only Load's object validation can establish a safe | 163 | + | if errors.Is(err, os.ErrNotExist) { |
| 169 | - | // replacement authoritative generation in that recovery case. | 164 | + | if publication.Generation != 1 || publication.ParentCheckpoint != nil { |
| 170 | - | func (r *Repository) currentPublication(ctx context.Context, name, key string) (uint64, []byte, map[ports.SnapshotDigest]codec.ObjectRef, error) { | 165 | + | return 0, nil, nil, fmt.Errorf("first snapshot generation must be 1 with no parent") |
| 171 | - | generation, digest, err := r.readHead(key) | |||
| 172 | - | if err == nil { | |||
| 173 | - | data, readErr := r.readBounded(r.manifestPath(key, generation)) | |||
| 174 | - | if readErr == nil && sha256.Sum256(data) == digest { | |||
| 175 | - | refs, validateErr := validateManifest(data, name, generation) | |||
| 176 | - | if validateErr == nil { | |||
| 177 | - | return generation, data, refs, nil | |||
| 178 | - | } | |||
| 179 | } | 166 | } | ||
| 167 | + | return 0, nil, nil, nil | |||
| 180 | } | 168 | } | ||
| 181 | - | 169 | + | if err != nil { | |
| 182 | - | generation, data, err := r.currentGeneration(ctx, name, key) | 170 | + | return 0, nil, nil, fmt.Errorf("read snapshot HEAD: %w", err) |
| 183 | - | if err != nil || generation == 0 { | 171 | + | } |
| 184 | - | return generation, data, nil, err | 172 | + | data, err := r.readBounded(r.manifestPath(publication.IncarnationID, generation)) |
| 173 | + | if err != nil { | |||
| 174 | + | return 0, nil, nil, fmt.Errorf("validate current checkpoint: %w", err) | |||
| 185 | } | 175 | } | ||
| 186 | - | refs, err := validateManifest(data, name, generation) | 176 | + | if sha256.Sum256(data) != digest { |
| 177 | + | return 0, nil, nil, fmt.Errorf("validate current checkpoint: manifest digest mismatch") | |||
| 178 | + | } | |||
| 179 | + | manifest, err := codec.UnmarshalManifest(data) | |||
| 187 | if err != nil { | 180 | if err != nil { | ||
| 188 | - | return 0, nil, nil, err | 181 | + | return 0, nil, nil, fmt.Errorf("validate current manifest: %w", err) |
| 182 | + | } | |||
| 183 | + | if manifest.IncarnationID != publication.IncarnationID || manifest.Generation != generation { | |||
| 184 | + | return 0, nil, nil, fmt.Errorf("current manifest identity mismatch") | |||
| 185 | + | } | |||
| 186 | + | // Every child checkpoint is bound to the exact authoritative HEAD. | |||
| 187 | + | if publication.Generation == generation+1 { | |||
| 188 | + | wantParent := &domain.CheckpointRef{Generation: generation, ManifestDigest: digest} | |||
| 189 | + | if !checkpointRefEqual(publication.ParentCheckpoint, wantParent) { | |||
| 190 | + | return 0, nil, nil, fmt.Errorf("publication parent does not match current checkpoint") | |||
| 191 | + | } | |||
| 189 | } | 192 | } | ||
| 190 | - | return generation, data, refs, nil | 193 | + | refs := manifestRefs(manifest) |
| 194 | + | if refs == nil || !withinGenerationBudget(len(data), refs) { | |||
| 195 | + | return 0, nil, nil, fmt.Errorf("invalid current generation") | |||
| 196 | + | } | |||
| 197 | + | return generation, data, refs, ctx.Err() | |||
| 198 | + | } | |||
| 199 | + | ||||
| 200 | + | func checkpointRefEqual(left, right *domain.CheckpointRef) bool { | |||
| 201 | + | if left == nil || right == nil { | |||
| 202 | + | return left == nil && right == nil | |||
| 203 | + | } | |||
| 204 | + | return *left == *right | |||
| 191 | } | 205 | } | ||
| 192 | 206 | ||||
| 193 | func validatePublication(p ports.SnapshotPublication) (map[ports.SnapshotDigest]codec.ObjectRef, map[ports.SnapshotDigest]ports.SnapshotObject, error) { | 207 | func validatePublication(p ports.SnapshotPublication) (map[ports.SnapshotDigest]codec.ObjectRef, map[ports.SnapshotDigest]ports.SnapshotObject, error) { | ||
| @@ -201,7 +215,7 @@ func validatePublication(p ports.SnapshotPublication) (map[ports.SnapshotDigest] | |||||
| 201 | } | 215 | } | ||
| 202 | objects := make(map[ports.SnapshotDigest]ports.SnapshotObject, len(supplied)) | 216 | objects := make(map[ports.SnapshotDigest]ports.SnapshotObject, len(supplied)) | ||
| 203 | for digest, entries := range supplied { | 217 | for digest, entries := range supplied { | ||
| 204 | - | object, err := validateSuppliedObject(entries, digest, refs[digest], nil) | 218 | + | object, err := validateSuppliedObject(entries, digest, refs[digest]) |
| 205 | if err != nil { | 219 | if err != nil { | ||
| 206 | return nil, nil, err | 220 | return nil, nil, err | ||
| 207 | } | 221 | } | ||
| @@ -211,16 +225,16 @@ func validatePublication(p ports.SnapshotPublication) (map[ports.SnapshotDigest] | |||||
| 211 | } | 225 | } | ||
| 212 | 226 | ||||
| 213 | func validatePublicationManifest(p ports.SnapshotPublication) (map[ports.SnapshotDigest]codec.ObjectRef, error) { | 227 | func validatePublicationManifest(p ports.SnapshotPublication) (map[ports.SnapshotDigest]codec.ObjectRef, error) { | ||
| 214 | - | return validateManifest(p.Manifest, p.Name, p.Generation) | 228 | + | return validateManifest(p.Manifest, p.IncarnationID, p.Name, p.Generation, p.ParentCheckpoint) |
| 215 | } | 229 | } | ||
| 216 | 230 | ||||
| 217 | - | func validateManifest(data []byte, name string, generation uint64) (map[ports.SnapshotDigest]codec.ObjectRef, error) { | 231 | + | func validateManifest(data []byte, incarnationID domain.IncarnationID, name string, generation uint64, parent *domain.CheckpointRef) (map[ports.SnapshotDigest]codec.ObjectRef, error) { |
| 218 | manifest, err := codec.UnmarshalManifest(data) | 232 | manifest, err := codec.UnmarshalManifest(data) | ||
| 219 | if err != nil { | 233 | if err != nil { | ||
| 220 | return nil, fmt.Errorf("invalid manifest: %w", err) | 234 | return nil, fmt.Errorf("invalid manifest: %w", err) | ||
| 221 | } | 235 | } | ||
| 222 | - | if manifest.Name != name || manifest.Generation != generation || generation == 0 { | 236 | + | if manifest.IncarnationID != incarnationID || manifest.Name != name || manifest.Generation != generation || generation == 0 || !checkpointRefEqual(manifest.ParentCheckpoint, parent) { |
| 223 | - | return nil, fmt.Errorf("manifest name or generation does not match publication") | 237 | + | return nil, fmt.Errorf("manifest identity, name, generation, or parent does not match publication") |
| 224 | } | 238 | } | ||
| 225 | refs := manifestRefs(manifest) | 239 | refs := manifestRefs(manifest) | ||
| 226 | if refs == nil { | 240 | if refs == nil { | ||
| @@ -255,16 +269,13 @@ func retainedObject(currentRefs map[ports.SnapshotDigest]codec.ObjectRef, digest | |||||
| 255 | } | 269 | } | ||
| 256 | 270 | ||||
| 257 | func (r *Repository) validateSuppliedObject(objects []ports.SnapshotObject, digest ports.SnapshotDigest, ref codec.ObjectRef) (ports.SnapshotObject, error) { | 271 | func (r *Repository) validateSuppliedObject(objects []ports.SnapshotObject, digest ports.SnapshotDigest, ref codec.ObjectRef) (ports.SnapshotObject, error) { | ||
| 258 | - | return validateSuppliedObject(objects, digest, ref, r) | 272 | + | return validateSuppliedObject(objects, digest, ref) |
| 259 | } | 273 | } | ||
| 260 | 274 | ||||
| 261 | - | func validateSuppliedObject(objects []ports.SnapshotObject, digest ports.SnapshotDigest, ref codec.ObjectRef, repository *Repository) (ports.SnapshotObject, error) { | 275 | + | func validateSuppliedObject(objects []ports.SnapshotObject, digest ports.SnapshotDigest, ref codec.ObjectRef) (ports.SnapshotObject, error) { |
| 262 | object := objects[0] | 276 | object := objects[0] | ||
| 263 | for _, candidate := range objects { | 277 | for _, candidate := range objects { | ||
| 264 | actual := sha256.Sum256(candidate.Data) | 278 | actual := sha256.Sum256(candidate.Data) | ||
| 265 | - | if repository != nil { | |||
| 266 | - | actual = repository.objectDigest(candidate.Data) | |||
| 267 | - | } | |||
| 268 | if actual != candidate.Digest || candidate.Digest != digest { | 279 | if actual != candidate.Digest || candidate.Digest != digest { | ||
| 269 | return ports.SnapshotObject{}, fmt.Errorf("object digest mismatch") | 280 | return ports.SnapshotObject{}, fmt.Errorf("object digest mismatch") | ||
| 270 | } | 281 | } | ||
| @@ -276,9 +287,6 @@ func validateSuppliedObject(objects []ports.SnapshotObject, digest ports.Snapsho | |||||
| 276 | if err != nil || kind != ref.Kind || len(object.Data) != int(ref.Size) || len(payload) == 0 { | 287 | if err != nil || kind != ref.Kind || len(object.Data) != int(ref.Size) || len(payload) == 0 { | ||
| 277 | return ports.SnapshotObject{}, fmt.Errorf("invalid object envelope") | 288 | return ports.SnapshotObject{}, fmt.Errorf("invalid object envelope") | ||
| 278 | } | 289 | } | ||
| 279 | - | if repository != nil && repository.hooks.beforeObjectCopy != nil { | |||
| 280 | - | repository.hooks.beforeObjectCopy(object.Data) | |||
| 281 | - | } | |||
| 282 | return ports.SnapshotObject{Digest: digest, Data: append([]byte(nil), object.Data...)}, nil | 290 | return ports.SnapshotObject{Digest: digest, Data: append([]byte(nil), object.Data...)}, nil | ||
| 283 | } | 291 | } | ||
| 284 | 292 | ||||
| @@ -326,16 +334,10 @@ func validObject(data []byte, ref codec.ObjectRef) bool { | |||||
| 326 | } | 334 | } | ||
| 327 | 335 | ||||
| 328 | func (r *Repository) objectDigest(data []byte) ports.SnapshotDigest { | 336 | func (r *Repository) objectDigest(data []byte) ports.SnapshotDigest { | ||
| 329 | - | if r.hooks.beforeObjectHash != nil { | |||
| 330 | - | r.hooks.beforeObjectHash(data) | |||
| 331 | - | } | |||
| 332 | return sha256.Sum256(data) | 337 | return sha256.Sum256(data) | ||
| 333 | } | 338 | } | ||
| 334 | 339 | ||||
| 335 | func (r *Repository) verifyObjectFile(path string, digest ports.SnapshotDigest, ref codec.ObjectRef) (bool, error) { | 340 | func (r *Repository) verifyObjectFile(path string, digest ports.SnapshotDigest, ref codec.ObjectRef) (bool, error) { | ||
| 336 | - | if r.hooks.beforeObjectRead != nil { | |||
| 337 | - | r.hooks.beforeObjectRead(path) | |||
| 338 | - | } | |||
| 339 | data, exists, err := r.readOptionalBounded(path) | 341 | data, exists, err := r.readOptionalBounded(path) | ||
| 340 | if err != nil || !exists { | 342 | if err != nil || !exists { | ||
| 341 | return exists, err | 343 | return exists, err | ||
M internal/adapters/snapshot/publish_efficiency_test.go +22 -106
| @@ -8,47 +8,31 @@ import ( | |||||
| 8 | "path/filepath" | 8 | "path/filepath" | ||
| 9 | "testing" | 9 | "testing" | ||
| 10 | 10 | ||||
| 11 | + | "github.com/bnema/vev/internal/domain" | |||
| 11 | "github.com/bnema/vev/internal/ports" | 12 | "github.com/bnema/vev/internal/ports" | ||
| 12 | codec "github.com/bnema/vev/internal/usecase/snapshot" | 13 | codec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 13 | ) | 14 | ) | ||
| 14 | 15 | ||||
| 15 | - | func TestRepositoryPublishSkipsUnchangedTenThousandObjectHistory(t *testing.T) { | 16 | + | func TestRepositoryPublishRevalidatesUnchangedTenThousandObjectHistory(t *testing.T) { |
| 16 | const historyObjects = 10_000 | 17 | const historyObjects = 10_000 | ||
| 17 | 18 | ||||
| 18 | repo := NewRepository(privateDir(t)) | 19 | repo := NewRepository(privateDir(t)) | ||
| 19 | first, second := largeIncrementalPublications(t, "named", historyObjects) | 20 | first, second := largeIncrementalPublications(t, "named", historyObjects) | ||
| 20 | seedCompletePublication(t, repo, first) | 21 | seedCompletePublication(t, repo, first) | ||
| 21 | 22 | ||||
| 22 | - | accounting := installFilesystemPublishAccounting(repo) | |||
| 23 | if err := repo.Publish(context.Background(), second); err != nil { | 23 | if err := repo.Publish(context.Background(), second); err != nil { | ||
| 24 | t.Fatal(err) | 24 | t.Fatal(err) | ||
| 25 | } | 25 | } | ||
| 26 | - | // Only the new mutable tail and visible objects need work. The 10k sealed | |||
| 27 | - | // objects are retained history supplied again by the producer, not input | |||
| 28 | - | // that Publish needs to read, hash, copy, or write. | |||
| 29 | - | requireFilesystemPublishAccounting(t, accounting, filesystemPublishAccounting{ | |||
| 30 | - | objectReads: 2, objectHashes: 2, objectCopies: 2, | |||
| 31 | - | tempCreates: 4, tempWrites: 4, fileSyncs: 4, | |||
| 32 | - | objectWrites: 2, manifestWrites: 1, headWrites: 1, | |||
| 33 | - | immutableInstalls: 3, mutableRenames: 1, tempRemoves: 3, directorySyncs: 4, | |||
| 34 | - | }) | |||
| 35 | 26 | ||||
| 36 | - | // Replaying the now-current generation still parses the 10k references, but | 27 | + | // Replaying the current generation revalidates every referenced immutable |
| 37 | - | // must not allocate one object-sized buffer per retained history entry. | 28 | + | // object before accepting it. That is required for a forward orphan retry: |
| 38 | - | // The limit is 450, not lower: the os.Root-based fast path pays a constant | 29 | + | // the catalogue must not advance to a checkpoint whose files were damaged |
| 39 | - | // allocation cost for guarded reads and root identity validation, and -race | 30 | + | // after its original publication. |
| 40 | - | // adds further overhead. A per-retained-object regression would exceed this | 31 | + | if err := repo.Publish(context.Background(), second); err != nil { |
| 41 | - | // by orders of magnitude. | 32 | + | t.Fatal(err) |
| 42 | - | allocations := testing.AllocsPerRun(3, func() { | |||
| 43 | - | if err := repo.Publish(context.Background(), second); err != nil { | |||
| 44 | - | panic(err) | |||
| 45 | - | } | |||
| 46 | - | }) | |||
| 47 | - | if allocations > 450 { | |||
| 48 | - | t.Fatalf("unchanged 10k history Publish allocations = %.0f, want <= 450; retained objects were likely copied", allocations) | |||
| 49 | } | 33 | } | ||
| 50 | 34 | ||||
| 51 | - | got, err := repo.Load(context.Background(), second.Name) | 35 | + | got, err := loadPublication(context.Background(), repo, second) |
| 52 | if err != nil { | 36 | if err != nil { | ||
| 53 | t.Fatal(err) | 37 | t.Fatal(err) | ||
| 54 | } | 38 | } | ||
| @@ -57,44 +41,10 @@ func TestRepositoryPublishSkipsUnchangedTenThousandObjectHistory(t *testing.T) { | |||||
| 57 | } | 41 | } | ||
| 58 | } | 42 | } | ||
| 59 | 43 | ||||
| 60 | - | type filesystemPublishAccounting struct { | |||
| 61 | - | objectReads, objectHashes, objectCopies int | |||
| 62 | - | tempCreates, tempWrites, fileSyncs int | |||
| 63 | - | objectWrites, manifestWrites, headWrites int | |||
| 64 | - | immutableInstalls, mutableRenames, tempRemoves, directorySyncs int | |||
| 65 | - | } | |||
| 66 | - | ||||
| 67 | - | func installFilesystemPublishAccounting(repo *Repository) *filesystemPublishAccounting { | |||
| 68 | - | accounting := &filesystemPublishAccounting{} | |||
| 69 | - | repo.hooks.beforeObjectRead = func(string) { accounting.objectReads++ } | |||
| 70 | - | repo.hooks.beforeObjectHash = func([]byte) { accounting.objectHashes++ } | |||
| 71 | - | repo.hooks.beforeObjectCopy = func([]byte) { accounting.objectCopies++ } | |||
| 72 | - | repo.hooks.createTemp = func(string) error { accounting.tempCreates++; return nil } | |||
| 73 | - | repo.hooks.writeTemp = func(string) error { accounting.tempWrites++; return nil } | |||
| 74 | - | repo.hooks.syncFile = func(string) error { accounting.fileSyncs++; return nil } | |||
| 75 | - | repo.hooks.beforeBlobWrite = func(string) error { accounting.objectWrites++; return nil } | |||
| 76 | - | repo.hooks.beforeManifestWrite = func(string) error { accounting.manifestWrites++; return nil } | |||
| 77 | - | repo.hooks.beforeHeadWrite = func(string) error { accounting.headWrites++; return nil } | |||
| 78 | - | repo.hooks.installImmutable = func(string) error { accounting.immutableInstalls++; return nil } | |||
| 79 | - | repo.hooks.rename = func(string) error { accounting.mutableRenames++; return nil } | |||
| 80 | - | repo.hooks.remove = func(string) error { accounting.tempRemoves++; return nil } | |||
| 81 | - | repo.hooks.syncDirectory = func(string) error { accounting.directorySyncs++; return nil } | |||
| 82 | - | return accounting | |||
| 83 | - | } | |||
| 84 | - | ||||
| 85 | - | func requireFilesystemPublishAccounting(t *testing.T, got *filesystemPublishAccounting, want filesystemPublishAccounting) { | |||
| 86 | - | t.Helper() | |||
| 87 | - | if *got != want { | |||
| 88 | - | t.Fatalf("filesystem Publish accounting = %+v, want %+v", *got, want) | |||
| 89 | - | } | |||
| 90 | - | } | |||
| 91 | - | ||||
| 92 | func BenchmarkRepositoryPublishUnchangedTenThousandObjectHistory(b *testing.B) { | 44 | func BenchmarkRepositoryPublishUnchangedTenThousandObjectHistory(b *testing.B) { | ||
| 93 | first, second := largeIncrementalPublications(b, "named", 10_000) | 45 | first, second := largeIncrementalPublications(b, "named", 10_000) | ||
| 94 | repo := NewRepository(filepath.Join(b.TempDir(), "vev")) | 46 | repo := NewRepository(filepath.Join(b.TempDir(), "vev")) | ||
| 95 | seedCompletePublication(b, repo, first) | 47 | seedCompletePublication(b, repo, first) | ||
| 96 | - | accounting := installFilesystemPublishAccounting(repo) | |||
| 97 | - | ||||
| 98 | b.ReportAllocs() | 48 | b.ReportAllocs() | ||
| 99 | b.ResetTimer() | 49 | b.ResetTimer() | ||
| 100 | for b.Loop() { | 50 | for b.Loop() { | ||
| @@ -103,57 +53,22 @@ func BenchmarkRepositoryPublishUnchangedTenThousandObjectHistory(b *testing.B) { | |||||
| 103 | } | 53 | } | ||
| 104 | } | 54 | } | ||
| 105 | b.StopTimer() | 55 | b.StopTimer() | ||
| 106 | - | // The benchmark deliberately replays the current generation so it can run | |||
| 107 | - | // repeatedly without fixture writes. Its bounds catch a regression that | |||
| 108 | - | // reads, hashes, or copies all 10k retained objects on the fast path. | |||
| 109 | - | if accounting.objectReads > 2 || accounting.objectHashes > 2 || accounting.objectCopies > 2 { | |||
| 110 | - | b.Fatalf("unchanged 10k history per-run object work = %d/%d/%d, want <= 2/2/2", accounting.objectReads, accounting.objectHashes, accounting.objectCopies) | |||
| 111 | - | } | |||
| 112 | - | } | |||
| 113 | - | ||||
| 114 | - | func TestRepositoryPublishVerifiesNecessaryExistingObjectOnce(t *testing.T) { | |||
| 115 | - | repo := NewRepository(privateDir(t)) | |||
| 116 | - | first := repositoryPublication(t, "named", 1, []byte("first")) | |||
| 117 | - | if err := repo.Publish(context.Background(), first); err != nil { | |||
| 118 | - | t.Fatal(err) | |||
| 119 | - | } | |||
| 120 | - | second := repositoryPublication(t, "named", 2, []byte("second")) | |||
| 121 | - | newTail := second.Objects[0] | |||
| 122 | - | key := sessionKey(second.Name) | |||
| 123 | - | if err := os.MkdirAll(filepath.Dir(repo.objectPath(key, newTail.Digest)), 0o700); err != nil { | |||
| 124 | - | t.Fatal(err) | |||
| 125 | - | } | |||
| 126 | - | if err := os.WriteFile(repo.objectPath(key, newTail.Digest), newTail.Data, 0o600); err != nil { | |||
| 127 | - | t.Fatal(err) | |||
| 128 | - | } | |||
| 129 | - | ||||
| 130 | - | var reads, hashes, copies int | |||
| 131 | - | repo.hooks.beforeObjectRead = func(string) { reads++ } | |||
| 132 | - | repo.hooks.beforeObjectHash = func([]byte) { hashes++ } | |||
| 133 | - | repo.hooks.beforeObjectCopy = func([]byte) { copies++ } | |||
| 134 | - | if err := repo.Publish(context.Background(), second); err != nil { | |||
| 135 | - | t.Fatal(err) | |||
| 136 | - | } | |||
| 137 | - | // The existing new tail is checked from disk once. The unchanged visible | |||
| 138 | - | // object is retained from generation one and needs no further work. | |||
| 139 | - | if reads != 1 || hashes != 1 || copies != 0 { | |||
| 140 | - | t.Fatalf("necessary object reads/hashes/copies = %d/%d/%d, want 1/1/0", reads, hashes, copies) | |||
| 141 | - | } | |||
| 142 | } | 56 | } | ||
| 143 | 57 | ||||
| 144 | func largeIncrementalPublications(t testing.TB, name string, count int) (ports.SnapshotPublication, ports.SnapshotPublication) { | 58 | func largeIncrementalPublications(t testing.TB, name string, count int) (ports.SnapshotPublication, ports.SnapshotPublication) { | ||
| 145 | t.Helper() | 59 | t.Helper() | ||
| 60 | + | id := testIncarnationID(name) | |||
| 146 | sealed := make([]codec.ObjectRef, 0, count) | 61 | sealed := make([]codec.ObjectRef, 0, count) | ||
| 147 | objects := make([]ports.SnapshotObject, 0, count+2) | 62 | objects := make([]ports.SnapshotObject, 0, count+2) | ||
| 148 | for i := range count { | 63 | for i := range count { | ||
| 149 | - | object, err := codec.MarshalObject(codec.HistoryChunk, []byte(fmt.Sprintf("history-%d", i))) | 64 | + | object, err := codec.MarshalObject(codec.HistoryChunk, fmt.Appendf(nil, "history-%d", i)) |
| 150 | if err != nil { | 65 | if err != nil { | ||
| 151 | t.Fatal(err) | 66 | t.Fatal(err) | ||
| 152 | } | 67 | } | ||
| 153 | sealed = append(sealed, codec.ObjectRef{Kind: codec.HistoryChunk, Digest: object.Digest, Size: uint32(len(object.Data))}) | 68 | sealed = append(sealed, codec.ObjectRef{Kind: codec.HistoryChunk, Digest: object.Digest, Size: uint32(len(object.Data))}) | ||
| 154 | objects = append(objects, object) | 69 | objects = append(objects, object) | ||
| 155 | } | 70 | } | ||
| 156 | - | makePublication := func(generation uint64, tailPayload, visiblePayload string) ports.SnapshotPublication { | 71 | + | makePublication := func(generation uint64, parent *domain.CheckpointRef, tailPayload, visiblePayload string) ports.SnapshotPublication { |
| 157 | t.Helper() | 72 | t.Helper() | ||
| 158 | tail, err := codec.MarshalObject(codec.HistoryTail, []byte(tailPayload)) | 73 | tail, err := codec.MarshalObject(codec.HistoryTail, []byte(tailPayload)) | ||
| 159 | if err != nil { | 74 | if err != nil { | ||
| @@ -163,19 +78,20 @@ func largeIncrementalPublications(t testing.TB, name string, count int) (ports.S | |||||
| 163 | if err != nil { | 78 | if err != nil { | ||
| 164 | t.Fatal(err) | 79 | t.Fatal(err) | ||
| 165 | } | 80 | } | ||
| 166 | - | manifest, err := codec.MarshalManifest(codec.Manifest{Generation: generation, Name: name, Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Sealed: sealed, Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}}) | 81 | + | manifest, err := codec.MarshalManifest(codec.Manifest{Generation: generation, IncarnationID: id, ParentCheckpoint: parent, Name: name, Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Sealed: sealed, Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}}) |
| 167 | if err != nil { | 82 | if err != nil { | ||
| 168 | t.Fatal(err) | 83 | t.Fatal(err) | ||
| 169 | } | 84 | } | ||
| 170 | - | return ports.SnapshotPublication{Name: name, Generation: generation, Manifest: manifest, Objects: append(append([]ports.SnapshotObject(nil), objects...), tail, visible)} | 85 | + | return ports.SnapshotPublication{IncarnationID: id, Name: name, Generation: generation, ParentCheckpoint: parent, Manifest: manifest, Objects: append(append([]ports.SnapshotObject(nil), objects...), tail, visible)} |
| 171 | } | 86 | } | ||
| 172 | - | return makePublication(1, "tail-1", "visible-1"), makePublication(2, "tail-2", "visible-2") | 87 | + | first := makePublication(1, nil, "tail-1", "visible-1") |
| 88 | + | parent := &domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(first.Manifest)} | |||
| 89 | + | return first, makePublication(2, parent, "tail-2", "visible-2") | |||
| 173 | } | 90 | } | ||
| 174 | 91 | ||||
| 175 | func seedCompletePublication(t testing.TB, repo *Repository, publication ports.SnapshotPublication) { | 92 | func seedCompletePublication(t testing.TB, repo *Repository, publication ports.SnapshotPublication) { | ||
| 176 | t.Helper() | 93 | t.Helper() | ||
| 177 | - | key := sessionKey(publication.Name) | 94 | + | if err := repo.ensureSession(publication.IncarnationID); err != nil { |
| 178 | - | if err := repo.ensureSession(key); err != nil { | |||
| 179 | t.Fatal(err) | 95 | t.Fatal(err) | ||
| 180 | } | 96 | } | ||
| 181 | manifest, err := codec.UnmarshalManifest(publication.Manifest) | 97 | manifest, err := codec.UnmarshalManifest(publication.Manifest) | ||
| @@ -183,17 +99,17 @@ func seedCompletePublication(t testing.TB, repo *Repository, publication ports.S | |||||
| 183 | t.Fatal(err) | 99 | t.Fatal(err) | ||
| 184 | } | 100 | } | ||
| 185 | for _, object := range publication.Objects { | 101 | for _, object := range publication.Objects { | ||
| 186 | - | if err := os.MkdirAll(filepath.Dir(repo.objectPath(key, object.Digest)), 0o700); err != nil { | 102 | + | if err := os.MkdirAll(filepath.Dir(repo.objectPath(publication.IncarnationID, object.Digest)), 0o700); err != nil { |
| 187 | t.Fatal(err) | 103 | t.Fatal(err) | ||
| 188 | } | 104 | } | ||
| 189 | - | if err := os.WriteFile(repo.objectPath(key, object.Digest), object.Data, 0o600); err != nil { | 105 | + | if err := os.WriteFile(repo.objectPath(publication.IncarnationID, object.Digest), object.Data, 0o600); err != nil { |
| 190 | t.Fatal(err) | 106 | t.Fatal(err) | ||
| 191 | } | 107 | } | ||
| 192 | } | 108 | } | ||
| 193 | - | if err := os.WriteFile(repo.manifestPath(key, publication.Generation), publication.Manifest, 0o600); err != nil { | 109 | + | if err := os.WriteFile(repo.manifestPath(publication.IncarnationID, publication.Generation), publication.Manifest, 0o600); err != nil { |
| 194 | t.Fatal(err) | 110 | t.Fatal(err) | ||
| 195 | } | 111 | } | ||
| 196 | - | if err := os.WriteFile(repo.headPath(key), marshalHead(publication.Generation, sha256.Sum256(publication.Manifest)), 0o600); err != nil { | 112 | + | if err := os.WriteFile(repo.headPath(publication.IncarnationID), marshalHead(publication.Generation, sha256.Sum256(publication.Manifest)), 0o600); err != nil { |
| 197 | t.Fatal(err) | 113 | t.Fatal(err) | ||
| 198 | } | 114 | } | ||
| 199 | if refs := manifestRefs(manifest); refs == nil { | 115 | if refs := manifestRefs(manifest); refs == nil { | ||
A internal/adapters/snapshot/publish_parent_test.go +109 -0
| @@ -0,0 +1,109 @@ | |||||
| 1 | + | package snapshot | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "crypto/sha256" | |||
| 6 | + | "os" | |||
| 7 | + | "path/filepath" | |||
| 8 | + | "testing" | |||
| 9 | + | ||||
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 11 | + | ) | |||
| 12 | + | ||||
| 13 | + | func TestRepositoryPublishAcceptsGenerationOneWithoutParent(t *testing.T) { | |||
| 14 | + | repo := NewRepository(privateDir(t)) | |||
| 15 | + | publication := repositoryPublication(t, "named", 1, []byte("one")) | |||
| 16 | + | ||||
| 17 | + | if err := repo.Publish(context.Background(), publication); err != nil { | |||
| 18 | + | t.Fatalf("Publish generation 1 without parent: %v", err) | |||
| 19 | + | } | |||
| 20 | + | } | |||
| 21 | + | ||||
| 22 | + | func TestRepositoryPublishRejectsInvalidLaterParentWithoutMutation(t *testing.T) { | |||
| 23 | + | tests := []struct { | |||
| 24 | + | name string | |||
| 25 | + | parent func(domain.CheckpointRef) *domain.CheckpointRef | |||
| 26 | + | }{ | |||
| 27 | + | {name: "nil", parent: func(domain.CheckpointRef) *domain.CheckpointRef { return nil }}, | |||
| 28 | + | {name: "stale generation", parent: func(current domain.CheckpointRef) *domain.CheckpointRef { | |||
| 29 | + | current.Generation-- | |||
| 30 | + | return ¤t | |||
| 31 | + | }}, | |||
| 32 | + | {name: "mismatched digest", parent: func(current domain.CheckpointRef) *domain.CheckpointRef { | |||
| 33 | + | current.ManifestDigest[0]++ | |||
| 34 | + | return ¤t | |||
| 35 | + | }}, | |||
| 36 | + | } | |||
| 37 | + | ||||
| 38 | + | for _, test := range tests { | |||
| 39 | + | t.Run(test.name, func(t *testing.T) { | |||
| 40 | + | repo := NewRepository(privateDir(t)) | |||
| 41 | + | first := repositoryPublication(t, "named", 1, []byte("one")) | |||
| 42 | + | if err := repo.Publish(context.Background(), first); err != nil { | |||
| 43 | + | t.Fatalf("Publish generation 1: %v", err) | |||
| 44 | + | } | |||
| 45 | + | firstRef := domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(first.Manifest)} | |||
| 46 | + | second := incarnationPublication(t, first.IncarnationID, first.Name, 2, &firstRef) | |||
| 47 | + | if err := repo.Publish(context.Background(), second); err != nil { | |||
| 48 | + | t.Fatalf("Publish generation 2: %v", err) | |||
| 49 | + | } | |||
| 50 | + | current := domain.CheckpointRef{Generation: 2, ManifestDigest: sha256.Sum256(second.Manifest)} | |||
| 51 | + | third := incarnationPublication(t, first.IncarnationID, first.Name, 3, test.parent(current)) | |||
| 52 | + | before := snapshotRepositoryFiles(t, repo.dir) | |||
| 53 | + | ||||
| 54 | + | if err := repo.Publish(context.Background(), third); err == nil { | |||
| 55 | + | t.Fatal("Publish accepted invalid parent") | |||
| 56 | + | } | |||
| 57 | + | after := snapshotRepositoryFiles(t, repo.dir) | |||
| 58 | + | if len(after) != len(before) { | |||
| 59 | + | t.Fatalf("repository file count changed: before=%d after=%d", len(before), len(after)) | |||
| 60 | + | } | |||
| 61 | + | for path, data := range before { | |||
| 62 | + | if string(after[path]) != string(data) { | |||
| 63 | + | t.Fatalf("repository file %q mutated", path) | |||
| 64 | + | } | |||
| 65 | + | } | |||
| 66 | + | }) | |||
| 67 | + | } | |||
| 68 | + | } | |||
| 69 | + | ||||
| 70 | + | func TestRepositoryPublishAcceptsExactCurrentParent(t *testing.T) { | |||
| 71 | + | repo := NewRepository(privateDir(t)) | |||
| 72 | + | first := repositoryPublication(t, "named", 1, []byte("one")) | |||
| 73 | + | if err := repo.Publish(context.Background(), first); err != nil { | |||
| 74 | + | t.Fatalf("Publish generation 1: %v", err) | |||
| 75 | + | } | |||
| 76 | + | parent := &domain.CheckpointRef{Generation: 1, ManifestDigest: sha256.Sum256(first.Manifest)} | |||
| 77 | + | second := incarnationPublication(t, first.IncarnationID, first.Name, 2, parent) | |||
| 78 | + | ||||
| 79 | + | if err := repo.Publish(context.Background(), second); err != nil { | |||
| 80 | + | t.Fatalf("Publish generation 2 with exact parent: %v", err) | |||
| 81 | + | } | |||
| 82 | + | } | |||
| 83 | + | ||||
| 84 | + | func snapshotRepositoryFiles(t *testing.T, root string) map[string][]byte { | |||
| 85 | + | t.Helper() | |||
| 86 | + | files := make(map[string][]byte) | |||
| 87 | + | err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { | |||
| 88 | + | if err != nil { | |||
| 89 | + | return err | |||
| 90 | + | } | |||
| 91 | + | if entry.IsDir() { | |||
| 92 | + | return nil | |||
| 93 | + | } | |||
| 94 | + | data, err := os.ReadFile(path) | |||
| 95 | + | if err != nil { | |||
| 96 | + | return err | |||
| 97 | + | } | |||
| 98 | + | relative, err := filepath.Rel(root, path) | |||
| 99 | + | if err != nil { | |||
| 100 | + | return err | |||
| 101 | + | } | |||
| 102 | + | files[relative] = data | |||
| 103 | + | return nil | |||
| 104 | + | }) | |||
| 105 | + | if err != nil { | |||
| 106 | + | t.Fatalf("snapshot repository files: %v", err) | |||
| 107 | + | } | |||
| 108 | + | return files | |||
| 109 | + | } | |||
M internal/adapters/snapshot/repository.go +22 -110
| @@ -1,6 +1,7 @@ | |||||
| 1 | package snapshot | 1 | package snapshot | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | + | "log/slog" | |||
| 4 | "sync" | 5 | "sync" | ||
| 5 | 6 | ||||
| 6 | "github.com/bnema/vev/internal/ports" | 7 | "github.com/bnema/vev/internal/ports" | ||
| @@ -13,109 +14,43 @@ const ( | |||||
| 13 | repositoryHead = "HEAD" | 14 | repositoryHead = "HEAD" | ||
| 14 | generationWidth = 20 | 15 | generationWidth = 20 | ||
| 15 | maxRepositoryRead = maxSnapshotFileSize | 16 | maxRepositoryRead = maxSnapshotFileSize | ||
| 16 | - | // Legacy import is deliberately smaller than the per-file parser ceiling: | |||
| 17 | - | // importing old snapshots must not accumulate an unbounded startup payload. | |||
| 18 | - | maxLegacySnapshotFiles = 64 | |||
| 19 | - | maxLegacySnapshotBytes = 8 << 20 | |||
| 20 | ) | 17 | ) | ||
| 21 | 18 | ||||
| 22 | // Repository is the crash-safe, content-addressed session snapshot store. | 19 | // Repository is the crash-safe, content-addressed session snapshot store. | ||
| 23 | - | // Store remains available separately for the one-way legacy bridge. | |||
| 24 | type Repository struct { | 20 | type Repository struct { | ||
| 25 | - | dir string | 21 | + | dir string |
| 26 | - | hooks repositoryHooks | 22 | + | log *slog.Logger |
| 27 | 23 | ||||
| 28 | - | // sessionStateMu owns both maps. A caller retains a reference before it | 24 | + | // sessionStateMu owns locks. A caller retains a reference before it waits |
| 29 | - | // waits on a session mutex, so an idle entry can never be removed while a | 25 | + | // on a session mutex, so an idle entry can never be removed while a waiter |
| 30 | - | // waiter still holds its old mutex. Epoch entries share that lifetime. | 26 | + | // still holds its old mutex. |
| 31 | sessionStateMu sync.Mutex | 27 | sessionStateMu sync.Mutex | ||
| 32 | locks map[string]*sessionMutex | 28 | locks map[string]*sessionMutex | ||
| 33 | - | storageEpochs map[string]uint64 | |||
| 34 | - | nextEpoch uint64 | |||
| 35 | - | ||||
| 36 | - | // maintenanceMu owns bounded continuation metadata: seek cookies and | |||
| 37 | - | // pending directory entries. Directory descriptors are opened and closed | |||
| 38 | - | // for each maintenance call. | |||
| 39 | - | maintenanceMu sync.Mutex | |||
| 40 | - | maintenanceCursors map[string]*maintenanceCursor | |||
| 41 | - | maintenanceSessions map[string]*sessionMaintenance | |||
| 42 | - | maintenanceQuarantine *quarantineMaintenance | |||
| 43 | - | ||||
| 44 | - | // pendingLegacySync records an unlink whose root-directory sync failed. | |||
| 45 | - | // It is keyed by the deterministic legacy filename and shares a per-file | |||
| 46 | - | // lock with DeleteLegacy so a retry cannot acknowledge a recreated file. | |||
| 47 | - | pendingLegacySync sync.Map // map[string]struct{} | |||
| 48 | - | } | |||
| 49 | - | ||||
| 50 | - | // repositoryHooks makes each persistence boundary fault-injectable. Hooks run | |||
| 51 | - | // immediately before their respective syscall and are intentionally package | |||
| 52 | - | // private so production callers cannot weaken repository guarantees. | |||
| 53 | - | type repositoryHooks struct { | |||
| 54 | - | beforeBlobWrite func(string) error | |||
| 55 | - | beforeManifestWrite func(string) error | |||
| 56 | - | // Object hooks instrument the publication path in package tests. They keep | |||
| 57 | - | // the steady-state cost of retained history observable without exposing a | |||
| 58 | - | // production metrics surface. | |||
| 59 | - | beforeObjectRead func(string) | |||
| 60 | - | beforeObjectHash func([]byte) | |||
| 61 | - | beforeObjectCopy func([]byte) | |||
| 62 | - | beforeHeadRead func(string) error | |||
| 63 | - | beforeHeadWrite func(string) error | |||
| 64 | - | beforeTombstoneCheck func(string) | |||
| 65 | - | beforePendingQuarantineCheck func(string) | |||
| 66 | - | createTemp func(string) error | |||
| 67 | - | writeTemp func(string) error | |||
| 68 | - | syncFile func(string) error | |||
| 69 | - | closeFile func(string) error | |||
| 70 | - | installImmutable func(string) error | |||
| 71 | - | rename func(string) error | |||
| 72 | - | syncDirectory func(string) error | |||
| 73 | - | remove func(string) error | |||
| 74 | - | // afterOpenRoot and closeRoot make descriptor race and close-error paths | |||
| 75 | - | // deterministic in package tests. closeRoot never replaces the real close. | |||
| 76 | - | afterOpenRoot func() | |||
| 77 | - | closeRoot func() error | |||
| 78 | - | openMaintenanceDirectory func(string) (maintenanceDirectory, error) | |||
| 79 | - | // beforeMaintenanceWork observes each budgeted quarantine filesystem step. | |||
| 80 | - | // It is test-only instrumentation for hostile traversal bounds. | |||
| 81 | - | beforeMaintenanceWork func(string) | |||
| 82 | - | openLegacyDirectory func(string) (legacyDirectory, error) | |||
| 83 | - | beforeSessionLock func(string) | |||
| 84 | } | 29 | } | ||
| 85 | 30 | ||||
| 86 | var _ ports.SnapshotRepository = (*Repository)(nil) | 31 | var _ ports.SnapshotRepository = (*Repository)(nil) | ||
| 87 | - | var _ ports.LegacySnapshotSource = (*Repository)(nil) | |||
| 88 | 32 | ||||
| 89 | - | // NewRepository creates a repository rooted at dir. It does not create files | 33 | + | // NewRepository creates a repository rooted at dir with the default logger. It |
| 90 | - | // until the first publication, so merely constructing it is side-effect free. | 34 | + | // does not create files until the first publication, so construction is side-effect free. |
| 91 | func NewRepository(dir string) *Repository { | 35 | func NewRepository(dir string) *Repository { | ||
| 92 | - | return &Repository{ | 36 | + | return NewRepositoryWithLogger(dir, slog.Default()) |
| 93 | - | dir: dir, | |||
| 94 | - | locks: make(map[string]*sessionMutex), | |||
| 95 | - | storageEpochs: make(map[string]uint64), | |||
| 96 | - | } | |||
| 97 | - | } | |||
| 98 | - | ||||
| 99 | - | // invalidateStorageEpoch must be called with lockSession(key) held. The | |||
| 100 | - | // registry mutex makes accesses for different session keys race-free too. | |||
| 101 | - | func (r *Repository) invalidateStorageEpoch(key string) { | |||
| 102 | - | r.sessionStateMu.Lock() | |||
| 103 | - | r.nextEpoch++ | |||
| 104 | - | r.storageEpochs[key] = r.nextEpoch | |||
| 105 | - | r.sessionStateMu.Unlock() | |||
| 106 | } | 37 | } | ||
| 107 | 38 | ||||
| 108 | - | // storageEpoch must be called with lockSession(key) held. | 39 | + | // NewRepositoryWithLogger creates a repository rooted at dir with log. |
| 109 | - | func (r *Repository) storageEpoch(key string) uint64 { | 40 | + | func NewRepositoryWithLogger(dir string, log *slog.Logger) *Repository { |
| 110 | - | r.sessionStateMu.Lock() | 41 | + | if log == nil { |
| 111 | - | epoch := r.storageEpochs[key] | 42 | + | log = slog.Default() |
| 112 | - | r.sessionStateMu.Unlock() | 43 | + | } |
| 113 | - | return epoch | 44 | + | return &Repository{ |
| 45 | + | dir: dir, | |||
| 46 | + | log: log, | |||
| 47 | + | locks: make(map[string]*sessionMutex), | |||
| 48 | + | } | |||
| 114 | } | 49 | } | ||
| 115 | 50 | ||||
| 116 | - | // sessionMutex is retained by each waiter and each resumable maintenance | 51 | + | // sessionMutex is retained by each waiter. That reference makes eviction |
| 117 | - | // state. That reference is what makes eviction safe: a new caller cannot lock | 52 | + | // safe: a new caller cannot lock a replacement mutex until all users of the |
| 118 | - | // a replacement mutex until all users of the prior mutex have left it. | 53 | + | // prior mutex have left it. |
| 119 | type sessionMutex struct { | 54 | type sessionMutex struct { | ||
| 120 | key string | 55 | key string | ||
| 121 | mu sync.Mutex | 56 | mu sync.Mutex | ||
| @@ -131,9 +66,6 @@ func (r *Repository) lockSession(key string) *sessionMutex { | |||||
| 131 | } | 66 | } | ||
| 132 | lock.references++ | 67 | lock.references++ | ||
| 133 | r.sessionStateMu.Unlock() | 68 | r.sessionStateMu.Unlock() | ||
| 134 | - | if hook := r.hooks.beforeSessionLock; hook != nil { | |||
| 135 | - | hook(key) | |||
| 136 | - | } | |||
| 137 | lock.mu.Lock() | 69 | lock.mu.Lock() | ||
| 138 | return lock | 70 | return lock | ||
| 139 | } | 71 | } | ||
| @@ -143,18 +75,6 @@ func (r *Repository) unlockSession(lock *sessionMutex) { | |||||
| 143 | r.releaseSessionReference(lock) | 75 | r.releaseSessionReference(lock) | ||
| 144 | } | 76 | } | ||
| 145 | 77 | ||||
| 146 | - | func (r *Repository) retainSessionState(key string) *sessionMutex { | |||
| 147 | - | r.sessionStateMu.Lock() | |||
| 148 | - | lock := r.locks[key] | |||
| 149 | - | // sessionMaintenanceState is called while the active session lock is held. | |||
| 150 | - | if lock == nil { | |||
| 151 | - | panic("snapshot: retain missing session lock") | |||
| 152 | - | } | |||
| 153 | - | lock.references++ | |||
| 154 | - | r.sessionStateMu.Unlock() | |||
| 155 | - | return lock | |||
| 156 | - | } | |||
| 157 | - | ||||
| 158 | func (r *Repository) releaseSessionReference(lock *sessionMutex) { | 78 | func (r *Repository) releaseSessionReference(lock *sessionMutex) { | ||
| 159 | r.sessionStateMu.Lock() | 79 | r.sessionStateMu.Lock() | ||
| 160 | lock.references-- | 80 | lock.references-- | ||
| @@ -168,14 +88,6 @@ func (r *Repository) releaseSessionReference(lock *sessionMutex) { | |||||
| 168 | panic("snapshot: session lock registry mismatch") | 88 | panic("snapshot: session lock registry mismatch") | ||
| 169 | } | 89 | } | ||
| 170 | delete(r.locks, lock.key) | 90 | delete(r.locks, lock.key) | ||
| 171 | - | delete(r.storageEpochs, lock.key) | |||
| 172 | } | 91 | } | ||
| 173 | r.sessionStateMu.Unlock() | 92 | r.sessionStateMu.Unlock() | ||
| 174 | } | 93 | } | ||
| 175 | - | ||||
| 176 | - | // sessionStateCounts is test evidence that idle per-session state is evicted. | |||
| 177 | - | func (r *Repository) sessionStateCounts() (locks, epochs int) { | |||
| 178 | - | r.sessionStateMu.Lock() | |||
| 179 | - | defer r.sessionStateMu.Unlock() | |||
| 180 | - | return len(r.locks), len(r.storageEpochs) | |||
| 181 | - | } | |||
D internal/adapters/snapshot/repository_durability_test.go +0 -183
| @@ -1,183 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "bytes" | |||
| 5 | - | "context" | |||
| 6 | - | "errors" | |||
| 7 | - | "os" | |||
| 8 | - | "path/filepath" | |||
| 9 | - | "testing" | |||
| 10 | - | ||||
| 11 | - | "github.com/bnema/vev/internal/ports" | |||
| 12 | - | codec "github.com/bnema/vev/internal/usecase/snapshot" | |||
| 13 | - | ) | |||
| 14 | - | ||||
| 15 | - | func TestRepositoryImmutableInstallDoesNotOverwriteRacedTarget(t *testing.T) { | |||
| 16 | - | repo := NewRepository(privateDir(t)) | |||
| 17 | - | pub := repositoryPublication(t, "named", 1, []byte("state")) | |||
| 18 | - | object := pub.Objects[0] | |||
| 19 | - | target := repo.objectPath(sessionKey(pub.Name), object.Digest) | |||
| 20 | - | repo.hooks.installImmutable = func(path string) error { | |||
| 21 | - | if path == target { | |||
| 22 | - | return os.WriteFile(path, []byte("attacker bytes"), 0o600) | |||
| 23 | - | } | |||
| 24 | - | return nil | |||
| 25 | - | } | |||
| 26 | - | if err := repo.Publish(context.Background(), pub); err == nil { | |||
| 27 | - | t.Fatal("Publish succeeded after raced immutable target") | |||
| 28 | - | } | |||
| 29 | - | got, err := os.ReadFile(target) | |||
| 30 | - | if err != nil { | |||
| 31 | - | t.Fatal(err) | |||
| 32 | - | } | |||
| 33 | - | if string(got) != "attacker bytes" { | |||
| 34 | - | t.Fatalf("raced target = %q, want attacker bytes", got) | |||
| 35 | - | } | |||
| 36 | - | } | |||
| 37 | - | ||||
| 38 | - | func prepareHeadStage(t *testing.T, repo *Repository, pub ports.SnapshotPublication) { | |||
| 39 | - | t.Helper() | |||
| 40 | - | key := sessionKey(pub.Name) | |||
| 41 | - | prepareObjects(t, repo, pub) | |||
| 42 | - | if err := repo.writeImmutable(repo.manifestPath(key, pub.Generation), pub.Manifest, func(data []byte) error { | |||
| 43 | - | if !bytes.Equal(data, pub.Manifest) { | |||
| 44 | - | return errors.New("invalid manifest") | |||
| 45 | - | } | |||
| 46 | - | return nil | |||
| 47 | - | }); err != nil { | |||
| 48 | - | t.Fatal(err) | |||
| 49 | - | } | |||
| 50 | - | } | |||
| 51 | - | ||||
| 52 | - | func manifestReference(t *testing.T, data []byte, digest ports.SnapshotDigest) codec.ObjectRef { | |||
| 53 | - | t.Helper() | |||
| 54 | - | manifest, err := codec.UnmarshalManifest(data) | |||
| 55 | - | if err != nil { | |||
| 56 | - | t.Fatal(err) | |||
| 57 | - | } | |||
| 58 | - | ref, ok := manifestRefs(manifest)[digest] | |||
| 59 | - | if !ok { | |||
| 60 | - | t.Fatal("missing object reference") | |||
| 61 | - | } | |||
| 62 | - | return ref | |||
| 63 | - | } | |||
| 64 | - | ||||
| 65 | - | func TestRepositoryFaultsAtEveryPublicationBoundary(t *testing.T) { | |||
| 66 | - | cases := []struct { | |||
| 67 | - | location string | |||
| 68 | - | operation string | |||
| 69 | - | }{ | |||
| 70 | - | {"blob", "create"}, {"blob", "write"}, {"blob", "sync-file"}, {"blob", "close"}, {"blob", "install"}, {"blob", "sync-directory"}, | |||
| 71 | - | {"manifest", "create"}, {"manifest", "write"}, {"manifest", "sync-file"}, {"manifest", "close"}, {"manifest", "install"}, {"manifest", "sync-directory"}, | |||
| 72 | - | {"HEAD", "create"}, {"HEAD", "write"}, {"HEAD", "sync-file"}, {"HEAD", "close"}, {"HEAD", "rename"}, {"HEAD", "sync-directory"}, | |||
| 73 | - | } | |||
| 74 | - | for _, tc := range cases { | |||
| 75 | - | t.Run(tc.location+"/"+tc.operation, func(t *testing.T) { | |||
| 76 | - | repo := NewRepository(privateDir(t)) | |||
| 77 | - | first := repositoryPublication(t, "named", 1, []byte("one")) | |||
| 78 | - | if err := repo.Publish(context.Background(), first); err != nil { | |||
| 79 | - | t.Fatal(err) | |||
| 80 | - | } | |||
| 81 | - | second := repositoryPublication(t, "named", 2, []byte("two")) | |||
| 82 | - | key := sessionKey(second.Name) | |||
| 83 | - | if err := repo.ensureSession(key); err != nil { | |||
| 84 | - | t.Fatal(err) | |||
| 85 | - | } | |||
| 86 | - | preparePublicationLocation(t, repo, second, tc.location) | |||
| 87 | - | injectBoundary(repo, tc.location, tc.operation, second) | |||
| 88 | - | if err := repo.Publish(context.Background(), second); err == nil { | |||
| 89 | - | t.Fatal("Publish succeeded with injected persistence failure") | |||
| 90 | - | } | |||
| 91 | - | repo.hooks = repositoryHooks{} | |||
| 92 | - | got, err := repo.Load(context.Background(), "named") | |||
| 93 | - | if err != nil { | |||
| 94 | - | t.Fatal(err) | |||
| 95 | - | } | |||
| 96 | - | if got.Generation != 1 && got.Generation != 2 { | |||
| 97 | - | t.Fatalf("authoritative generation = %d, want old or new", got.Generation) | |||
| 98 | - | } | |||
| 99 | - | }) | |||
| 100 | - | } | |||
| 101 | - | } | |||
| 102 | - | ||||
| 103 | - | // preparePublicationLocation advances setup only far enough to target the | |||
| 104 | - | // requested persistence location in the publication fault matrix. | |||
| 105 | - | func preparePublicationLocation(t *testing.T, repo *Repository, pub ports.SnapshotPublication, location string) { | |||
| 106 | - | t.Helper() | |||
| 107 | - | switch location { | |||
| 108 | - | case "manifest": | |||
| 109 | - | prepareObjects(t, repo, pub) | |||
| 110 | - | case "HEAD": | |||
| 111 | - | prepareHeadStage(t, repo, pub) | |||
| 112 | - | } | |||
| 113 | - | } | |||
| 114 | - | ||||
| 115 | - | func injectBoundary(repo *Repository, location, operation string, pub ports.SnapshotPublication) { | |||
| 116 | - | key := sessionKey(pub.Name) | |||
| 117 | - | dir := filepath.Dir(repo.objectPath(key, pub.Objects[0].Digest)) | |||
| 118 | - | if location == "manifest" { | |||
| 119 | - | dir = filepath.Dir(repo.manifestPath(key, pub.Generation)) | |||
| 120 | - | } | |||
| 121 | - | if location == "HEAD" { | |||
| 122 | - | dir = filepath.Dir(repo.headPath(key)) | |||
| 123 | - | } | |||
| 124 | - | fail := errors.New("injected persistence failure") | |||
| 125 | - | repo.hooks.createTemp = func(got string) error { | |||
| 126 | - | if operation == "create" && got == dir { | |||
| 127 | - | return fail | |||
| 128 | - | } | |||
| 129 | - | return nil | |||
| 130 | - | } | |||
| 131 | - | repo.hooks.writeTemp = func(path string) error { | |||
| 132 | - | if operation == "write" && filepath.Dir(path) == dir { | |||
| 133 | - | return fail | |||
| 134 | - | } | |||
| 135 | - | return nil | |||
| 136 | - | } | |||
| 137 | - | repo.hooks.syncFile = func(path string) error { | |||
| 138 | - | if operation == "sync-file" && filepath.Dir(path) == dir { | |||
| 139 | - | return fail | |||
| 140 | - | } | |||
| 141 | - | return nil | |||
| 142 | - | } | |||
| 143 | - | repo.hooks.closeFile = func(path string) error { | |||
| 144 | - | if operation == "close" && filepath.Dir(path) == dir { | |||
| 145 | - | return fail | |||
| 146 | - | } | |||
| 147 | - | return nil | |||
| 148 | - | } | |||
| 149 | - | repo.hooks.installImmutable = func(path string) error { | |||
| 150 | - | if operation == "install" && filepath.Dir(path) == dir { | |||
| 151 | - | return fail | |||
| 152 | - | } | |||
| 153 | - | return nil | |||
| 154 | - | } | |||
| 155 | - | repo.hooks.rename = func(path string) error { | |||
| 156 | - | if operation == "rename" && location == "HEAD" && path == repo.headPath(key) { | |||
| 157 | - | return fail | |||
| 158 | - | } | |||
| 159 | - | return nil | |||
| 160 | - | } | |||
| 161 | - | repo.hooks.syncDirectory = func(got string) error { | |||
| 162 | - | if operation == "sync-directory" && got == dir { | |||
| 163 | - | return fail | |||
| 164 | - | } | |||
| 165 | - | return nil | |||
| 166 | - | } | |||
| 167 | - | } | |||
| 168 | - | ||||
| 169 | - | func prepareObjects(t *testing.T, repo *Repository, pub ports.SnapshotPublication) { | |||
| 170 | - | t.Helper() | |||
| 171 | - | key := sessionKey(pub.Name) | |||
| 172 | - | for _, object := range pub.Objects { | |||
| 173 | - | ref := manifestReference(t, pub.Manifest, object.Digest) | |||
| 174 | - | if err := repo.writeImmutable(repo.objectPath(key, object.Digest), object.Data, func(data []byte) error { | |||
| 175 | - | if !bytes.Equal(data, object.Data) || !validObject(data, ref) { | |||
| 176 | - | return errors.New("invalid object") | |||
| 177 | - | } | |||
| 178 | - | return nil | |||
| 179 | - | }); err != nil { | |||
| 180 | - | t.Fatal(err) | |||
| 181 | - | } | |||
| 182 | - | } | |||
| 183 | - | } | |||
D internal/adapters/snapshot/repository_fault_test.go +0 -204
| @@ -1,204 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "errors" | |||
| 6 | - | "os" | |||
| 7 | - | "path/filepath" | |||
| 8 | - | "strings" | |||
| 9 | - | "testing" | |||
| 10 | - | ||||
| 11 | - | codec "github.com/bnema/vev/internal/usecase/snapshot" | |||
| 12 | - | ) | |||
| 13 | - | ||||
| 14 | - | func TestRepositoryHeadFailureKeepsOldCompleteGeneration(t *testing.T) { | |||
| 15 | - | dir := privateDir(t) | |||
| 16 | - | repo := NewRepository(dir) | |||
| 17 | - | first := repositoryPublication(t, "named", 1, []byte("one")) | |||
| 18 | - | if err := repo.Publish(context.Background(), first); err != nil { | |||
| 19 | - | t.Fatal(err) | |||
| 20 | - | } | |||
| 21 | - | second := repositoryPublication(t, "named", 2, []byte("two")) | |||
| 22 | - | repo.hooks.beforeHeadWrite = func(string) error { return errors.New("injected HEAD failure") } | |||
| 23 | - | if err := repo.Publish(context.Background(), second); err == nil { | |||
| 24 | - | t.Fatal("Publish succeeded with injected HEAD failure") | |||
| 25 | - | } | |||
| 26 | - | repo.hooks.beforeHeadWrite = nil | |||
| 27 | - | got, err := repo.Load(context.Background(), "named") | |||
| 28 | - | if err != nil { | |||
| 29 | - | t.Fatal(err) | |||
| 30 | - | } | |||
| 31 | - | if got.Generation != 1 { | |||
| 32 | - | t.Fatalf("generation after failed publish = %d, want 1", got.Generation) | |||
| 33 | - | } | |||
| 34 | - | } | |||
| 35 | - | ||||
| 36 | - | func TestRepositoryCleanupFaultsSurfaceForImmutableAndMutablePublication(t *testing.T) { | |||
| 37 | - | primary := errors.New("primary write failure") | |||
| 38 | - | cleanup := errors.New("cleanup failure") | |||
| 39 | - | cases := []struct { | |||
| 40 | - | name string | |||
| 41 | - | location string | |||
| 42 | - | cleanup string | |||
| 43 | - | }{ | |||
| 44 | - | {name: "immutable/close temporary", location: "manifest", cleanup: "close"}, | |||
| 45 | - | {name: "immutable/remove temporary", location: "manifest", cleanup: "remove"}, | |||
| 46 | - | {name: "immutable/sync after temporary removal", location: "manifest", cleanup: "sync-after-remove"}, | |||
| 47 | - | {name: "mutable/close temporary", location: "HEAD", cleanup: "close"}, | |||
| 48 | - | {name: "mutable/remove temporary", location: "HEAD", cleanup: "remove"}, | |||
| 49 | - | {name: "mutable/sync after temporary removal", location: "HEAD", cleanup: "sync-after-remove"}, | |||
| 50 | - | } | |||
| 51 | - | for _, tc := range cases { | |||
| 52 | - | t.Run(tc.name, func(t *testing.T) { | |||
| 53 | - | repo := NewRepository(privateDir(t)) | |||
| 54 | - | first := repositoryPublication(t, "named", 1, []byte("one")) | |||
| 55 | - | if err := repo.Publish(context.Background(), first); err != nil { | |||
| 56 | - | t.Fatal(err) | |||
| 57 | - | } | |||
| 58 | - | second := repositoryPublication(t, "named", 2, []byte("two")) | |||
| 59 | - | key := sessionKey(second.Name) | |||
| 60 | - | if tc.location == "manifest" { | |||
| 61 | - | prepareObjects(t, repo, second) | |||
| 62 | - | } else { | |||
| 63 | - | prepareHeadStage(t, repo, second) | |||
| 64 | - | } | |||
| 65 | - | dir := filepath.Dir(repo.manifestPath(key, second.Generation)) | |||
| 66 | - | phase := "write manifest" | |||
| 67 | - | if tc.location == "HEAD" { | |||
| 68 | - | dir = filepath.Dir(repo.headPath(key)) | |||
| 69 | - | phase = "write HEAD" | |||
| 70 | - | } | |||
| 71 | - | repo.hooks.writeTemp = func(path string) error { | |||
| 72 | - | if filepath.Dir(path) == dir { | |||
| 73 | - | return primary | |||
| 74 | - | } | |||
| 75 | - | return nil | |||
| 76 | - | } | |||
| 77 | - | repo.hooks.closeFile = func(path string) error { | |||
| 78 | - | if tc.cleanup == "close" && filepath.Dir(path) == dir { | |||
| 79 | - | return cleanup | |||
| 80 | - | } | |||
| 81 | - | return nil | |||
| 82 | - | } | |||
| 83 | - | repo.hooks.remove = func(string) error { | |||
| 84 | - | if tc.cleanup == "remove" { | |||
| 85 | - | return cleanup | |||
| 86 | - | } | |||
| 87 | - | return nil | |||
| 88 | - | } | |||
| 89 | - | repo.hooks.syncDirectory = func(path string) error { | |||
| 90 | - | if tc.cleanup == "sync-after-remove" && path == dir { | |||
| 91 | - | return cleanup | |||
| 92 | - | } | |||
| 93 | - | return nil | |||
| 94 | - | } | |||
| 95 | - | ||||
| 96 | - | err := repo.Publish(context.Background(), second) | |||
| 97 | - | if !errors.Is(err, primary) || !errors.Is(err, cleanup) || !strings.Contains(err.Error(), phase) { | |||
| 98 | - | t.Fatalf("Publish error = %v, want %q and both primary and cleanup failures", err, phase) | |||
| 99 | - | } | |||
| 100 | - | if tc.cleanup != "remove" { | |||
| 101 | - | matches, globErr := filepath.Glob(filepath.Join(dir, ".tmp-*")) | |||
| 102 | - | if globErr != nil || len(matches) != 0 { | |||
| 103 | - | t.Fatalf("temporary files after successful cleanup = %v, glob error = %v", matches, globErr) | |||
| 104 | - | } | |||
| 105 | - | } | |||
| 106 | - | repo.hooks = repositoryHooks{} | |||
| 107 | - | assertRepositoryCompleteGeneration(t, repo, "named", 1, 2) | |||
| 108 | - | }) | |||
| 109 | - | } | |||
| 110 | - | } | |||
| 111 | - | ||||
| 112 | - | func TestRepositoryNewDirectorySyncFaultsAreIndependent(t *testing.T) { | |||
| 113 | - | cases := []struct { | |||
| 114 | - | name string | |||
| 115 | - | phase string | |||
| 116 | - | occurrence int | |||
| 117 | - | directory func(*Repository, string) string | |||
| 118 | - | }{ | |||
| 119 | - | {name: "repository", phase: "repository", occurrence: 1, directory: func(_ *Repository, root string) string { return filepath.Dir(root) }}, | |||
| 120 | - | {name: "sessions", phase: "sessions", occurrence: 1, directory: func(repo *Repository, _ string) string { return repo.dir }}, | |||
| 121 | - | {name: "session", phase: "session", occurrence: 1, directory: func(repo *Repository, _ string) string { return filepath.Join(repo.dir, repositorySessionsDir) }}, | |||
| 122 | - | {name: "objects", phase: "objects", occurrence: 1, directory: func(repo *Repository, _ string) string { return repo.sessionPath(sessionKey("named")) }}, | |||
| 123 | - | {name: "generations", phase: "generations", occurrence: 2, directory: func(repo *Repository, _ string) string { return repo.sessionPath(sessionKey("named")) }}, | |||
| 124 | - | {name: "object shard", phase: "object shard", occurrence: 1, directory: func(repo *Repository, _ string) string { | |||
| 125 | - | return filepath.Join(repo.sessionPath(sessionKey("named")), repositoryObjectsDir) | |||
| 126 | - | }}, | |||
| 127 | - | } | |||
| 128 | - | for _, tc := range cases { | |||
| 129 | - | t.Run(tc.name, func(t *testing.T) { | |||
| 130 | - | parent := privateDir(t) | |||
| 131 | - | root := filepath.Join(parent, "repository") | |||
| 132 | - | repo := NewRepository(root) | |||
| 133 | - | wantDirectory := tc.directory(repo, root) | |||
| 134 | - | calls := 0 | |||
| 135 | - | injected := errors.New("new directory sync failure") | |||
| 136 | - | repo.hooks.syncDirectory = func(directory string) error { | |||
| 137 | - | if directory == wantDirectory { | |||
| 138 | - | calls++ | |||
| 139 | - | if calls == tc.occurrence { | |||
| 140 | - | return injected | |||
| 141 | - | } | |||
| 142 | - | } | |||
| 143 | - | return nil | |||
| 144 | - | } | |||
| 145 | - | ||||
| 146 | - | publication := repositoryPublication(t, "named", 1, []byte("one")) | |||
| 147 | - | err := repo.Publish(context.Background(), publication) | |||
| 148 | - | if !errors.Is(err, injected) || !strings.Contains(err.Error(), tc.phase) { | |||
| 149 | - | t.Fatalf("Publish error = %v, want %q-labelled directory sync failure", err, tc.phase) | |||
| 150 | - | } | |||
| 151 | - | repo.hooks = repositoryHooks{} | |||
| 152 | - | // A first-generation failure cannot have an older complete generation; | |||
| 153 | - | // it must not expose an incomplete one as if it were loadable. | |||
| 154 | - | if _, loadErr := repo.Load(context.Background(), "named"); loadErr == nil || !strings.Contains(loadErr.Error(), "no complete snapshot generation") { | |||
| 155 | - | t.Fatalf("Load error after incomplete first generation = %v, want no complete generation", loadErr) | |||
| 156 | - | } | |||
| 157 | - | if err := repo.Publish(context.Background(), publication); err != nil { | |||
| 158 | - | t.Fatal(err) | |||
| 159 | - | } | |||
| 160 | - | assertRepositoryCompleteGeneration(t, repo, "named", 1) | |||
| 161 | - | }) | |||
| 162 | - | } | |||
| 163 | - | } | |||
| 164 | - | ||||
| 165 | - | func assertRepositoryCompleteGeneration(t *testing.T, repo *Repository, name string, generations ...uint64) { | |||
| 166 | - | t.Helper() | |||
| 167 | - | got, err := repo.Load(context.Background(), name) | |||
| 168 | - | if err != nil { | |||
| 169 | - | t.Fatal(err) | |||
| 170 | - | } | |||
| 171 | - | for _, generation := range generations { | |||
| 172 | - | if got.Generation == generation && len(got.Manifest) != 0 && len(got.Objects) != 0 { | |||
| 173 | - | return | |||
| 174 | - | } | |||
| 175 | - | } | |||
| 176 | - | t.Fatalf("Load = incomplete or mixed generation %+v, want one of %v", got, generations) | |||
| 177 | - | } | |||
| 178 | - | ||||
| 179 | - | func TestRepositoryLoadFallsBackFromIncompleteNewestGeneration(t *testing.T) { | |||
| 180 | - | dir := privateDir(t) | |||
| 181 | - | repo := NewRepository(dir) | |||
| 182 | - | first := repositoryPublication(t, "named", 1, []byte("one")) | |||
| 183 | - | if err := repo.Publish(context.Background(), first); err != nil { | |||
| 184 | - | t.Fatal(err) | |||
| 185 | - | } | |||
| 186 | - | second := repositoryPublication(t, "named", 2, []byte("two")) | |||
| 187 | - | if err := repo.Publish(context.Background(), second); err != nil { | |||
| 188 | - | t.Fatal(err) | |||
| 189 | - | } | |||
| 190 | - | manifest, err := codec.UnmarshalManifest(second.Manifest) | |||
| 191 | - | if err != nil { | |||
| 192 | - | t.Fatal(err) | |||
| 193 | - | } | |||
| 194 | - | if err := os.Remove(repo.objectPath(sessionKey("named"), manifest.Tabs[0].Panes[0].Tail.Digest)); err != nil { | |||
| 195 | - | t.Fatal(err) | |||
| 196 | - | } | |||
| 197 | - | got, err := repo.Load(context.Background(), "named") | |||
| 198 | - | if err != nil { | |||
| 199 | - | t.Fatal(err) | |||
| 200 | - | } | |||
| 201 | - | if got.Generation != 1 || !got.Fallback { | |||
| 202 | - | t.Fatalf("Load = generation %d fallback %v, want generation 1 fallback true", got.Generation, got.Fallback) | |||
| 203 | - | } | |||
| 204 | - | } | |||
D internal/adapters/snapshot/repository_maintenance_test.go +0 -69
| @@ -1,69 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "errors" | |||
| 6 | - | "os" | |||
| 7 | - | "path/filepath" | |||
| 8 | - | "strings" | |||
| 9 | - | "testing" | |||
| 10 | - | ) | |||
| 11 | - | ||||
| 12 | - | func TestSafeFilesystemErrorDoesNotExposePath(t *testing.T) { | |||
| 13 | - | cause := errors.New("permission denied") | |||
| 14 | - | err := safeFilesystemError(&os.PathError{Op: "remove", Path: "\x1b[31munsafe", Err: cause}) | |||
| 15 | - | if !errors.Is(err, cause) || strings.Contains(err.Error(), "\x1b[31munsafe") { | |||
| 16 | - | t.Fatalf("safe filesystem error = %q", err) | |||
| 17 | - | } | |||
| 18 | - | } | |||
| 19 | - | ||||
| 20 | - | func TestSafeFilesystemErrorDoesNotExposeLinkErrorPath(t *testing.T) { | |||
| 21 | - | err := safeFilesystemError(&os.LinkError{Op: "rename", Old: "/unsafe/old", New: "/unsafe/new", Err: os.ErrExist}) | |||
| 22 | - | if !errors.Is(err, os.ErrExist) || strings.Contains(err.Error(), "/unsafe/") { | |||
| 23 | - | t.Fatalf("safe link error = %q", err) | |||
| 24 | - | } | |||
| 25 | - | } | |||
| 26 | - | ||||
| 27 | - | func TestRepositoryMaintenanceFilesystemErrorsHaveSafeContext(t *testing.T) { | |||
| 28 | - | injected := errors.New("injected filesystem failure") | |||
| 29 | - | cases := []struct { | |||
| 30 | - | name string | |||
| 31 | - | run func(*Repository) error | |||
| 32 | - | want string | |||
| 33 | - | }{ | |||
| 34 | - | { | |||
| 35 | - | name: "delete sync", | |||
| 36 | - | run: func(repo *Repository) error { | |||
| 37 | - | publication := repositoryPublication(t, "named", 1, []byte("state")) | |||
| 38 | - | if err := repo.Publish(context.Background(), publication); err != nil { | |||
| 39 | - | t.Fatal(err) | |||
| 40 | - | } | |||
| 41 | - | repo.hooks.syncDirectory = func(string) error { return injected } | |||
| 42 | - | return repo.Delete(context.Background(), "named") | |||
| 43 | - | }, | |||
| 44 | - | want: `sync deleted snapshot session directory "named"`, | |||
| 45 | - | }, | |||
| 46 | - | { | |||
| 47 | - | name: "maintain remove", | |||
| 48 | - | run: func(repo *Repository) error { | |||
| 49 | - | if err := os.Mkdir(repo.dir, 0o700); err != nil { | |||
| 50 | - | t.Fatal(err) | |||
| 51 | - | } | |||
| 52 | - | if err := os.WriteFile(filepath.Join(repo.dir, ".tmp-stale"), []byte("stale"), 0o600); err != nil { | |||
| 53 | - | t.Fatal(err) | |||
| 54 | - | } | |||
| 55 | - | repo.hooks.remove = func(string) error { return injected } | |||
| 56 | - | return repo.Maintain(context.Background()) | |||
| 57 | - | }, | |||
| 58 | - | want: `remove snapshot maintenance file ".tmp-stale"`, | |||
| 59 | - | }, | |||
| 60 | - | } | |||
| 61 | - | for _, tc := range cases { | |||
| 62 | - | t.Run(tc.name, func(t *testing.T) { | |||
| 63 | - | err := tc.run(NewRepository(privateDir(t))) | |||
| 64 | - | if !errors.Is(err, injected) || !strings.Contains(err.Error(), tc.want) { | |||
| 65 | - | t.Fatalf("error = %v, want context %q and injected error", err, tc.want) | |||
| 66 | - | } | |||
| 67 | - | }) | |||
| 68 | - | } | |||
| 69 | - | } | |||
D internal/adapters/snapshot/repository_phase6_test.go +0 -107
| @@ -1,107 +0,0 @@ | |||||
| 1 | - | package snapshot | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "errors" | |||
| 6 | - | "fmt" | |||
| 7 | - | "os" | |||
| 8 | - | "path/filepath" | |||
| 9 | - | "sync" | |||
| 10 | - | "sync/atomic" | |||
| 11 | - | "testing" | |||
| 12 | - | ||||
| 13 | - | "github.com/bnema/vev/internal/ports" | |||
| 14 | - | ) | |||
| 15 | - | ||||
| 16 | - | func TestRepositoryPublishDeleteManySessionsRaceFree(t *testing.T) { | |||
| 17 | - | repo := NewRepository(privateDir(t)) | |||
| 18 | - | const sessions = 128 | |||
| 19 | - | ||||
| 20 | - | publications := make([]ports.SnapshotPublication, sessions) | |||
| 21 | - | for i := range publications { | |||
| 22 | - | name := fmt.Sprintf("named-%03d", i) | |||
| 23 | - | publications[i] = repositoryPublication(t, name, 1, []byte(name)) | |||
| 24 | - | if err := repo.Publish(context.Background(), publications[i]); err != nil { | |||
| 25 | - | t.Fatalf("initial Publish(%q): %v", name, err) | |||
| 26 | - | } | |||
| 27 | - | } | |||
| 28 | - | ||||
| 29 | - | var wg sync.WaitGroup | |||
| 30 | - | for _, publication := range publications { | |||
| 31 | - | wg.Add(2) | |||
| 32 | - | go func() { | |||
| 33 | - | defer wg.Done() | |||
| 34 | - | if err := repo.Publish(context.Background(), publication); err != nil { | |||
| 35 | - | t.Errorf("Publish(%q): %v", publication.Name, err) | |||
| 36 | - | } | |||
| 37 | - | }() | |||
| 38 | - | go func() { | |||
| 39 | - | defer wg.Done() | |||
| 40 | - | if err := repo.Delete(context.Background(), publication.Name); err != nil { | |||
| 41 | - | t.Errorf("Delete(%q): %v", publication.Name, err) | |||
| 42 | - | } | |||
| 43 | - | }() | |||
| 44 | - | } | |||
| 45 | - | wg.Wait() | |||
| 46 | - | ||||
| 47 | - | locks, epochs := repo.sessionStateCounts() | |||
| 48 | - | if locks != 0 || epochs != 0 { | |||
| 49 | - | t.Fatalf("retained session state = %d locks, %d epochs, want none", locks, epochs) | |||
| 50 | - | } | |||
| 51 | - | } | |||
| 52 | - | ||||
| 53 | - | func TestRepositoryBoundsAndEvictsPartialMaintenanceSessionState(t *testing.T) { | |||
| 54 | - | repo := NewRepository(privateDir(t)) | |||
| 55 | - | for _, key := range []string{"named-000", "named-001"} { | |||
| 56 | - | generations := filepath.Join(repo.sessionPath(key), repositoryGenerations) | |||
| 57 | - | for i := 0; i < maintenanceBatch+1; i++ { | |||
| 58 | - | if err := os.MkdirAll(generations, 0o700); err != nil { | |||
| 59 | - | t.Fatal(err) | |||
| 60 | - | } | |||
| 61 | - | if err := os.WriteFile(filepath.Join(generations, fmt.Sprintf("unclassified-%03d", i)), []byte("state"), 0o600); err != nil { | |||
| 62 | - | t.Fatal(err) | |||
| 63 | - | } | |||
| 64 | - | } | |||
| 65 | - | } | |||
| 66 | - | if err := repo.Maintain(context.Background()); err != nil { | |||
| 67 | - | t.Fatal(err) | |||
| 68 | - | } | |||
| 69 | - | locks, epochs := repo.sessionStateCounts() | |||
| 70 | - | if locks != 1 || epochs != 0 { | |||
| 71 | - | t.Fatalf("partial maintenance state = %d locks, %d epochs, want one lock and no epochs", locks, epochs) | |||
| 72 | - | } | |||
| 73 | - | ||||
| 74 | - | ctx, cancel := context.WithCancel(context.Background()) | |||
| 75 | - | cancel() | |||
| 76 | - | if err := repo.Maintain(ctx); !errors.Is(err, context.Canceled) { | |||
| 77 | - | t.Fatalf("canceled Maintain error = %v, want context canceled", err) | |||
| 78 | - | } | |||
| 79 | - | locks, epochs = repo.sessionStateCounts() | |||
| 80 | - | if locks != 0 || epochs != 0 { | |||
| 81 | - | t.Fatalf("session state after maintenance reset = %d locks, %d epochs, want none", locks, epochs) | |||
| 82 | - | } | |||
| 83 | - | } | |||
| 84 | - | ||||
| 85 | - | func TestRepositoryDeleteUsesDeterministicPendingQuarantine(t *testing.T) { | |||
| 86 | - | repo := NewRepository(privateDir(t)) | |||
| 87 | - | if err := repo.Publish(context.Background(), repositoryPublication(t, "named", 1, []byte("state"))); err != nil { | |||
| 88 | - | t.Fatal(err) | |||
| 89 | - | } | |||
| 90 | - | sessions := filepath.Join(repo.dir, repositorySessionsDir) | |||
| 91 | - | for i := 0; i < maxDirectoryTraversalEntries*2; i++ { | |||
| 92 | - | if err := os.Mkdir(filepath.Join(sessions, fmt.Sprintf("unrelated-%05d", i)), 0o700); err != nil { | |||
| 93 | - | t.Fatal(err) | |||
| 94 | - | } | |||
| 95 | - | } | |||
| 96 | - | var checks atomic.Int32 | |||
| 97 | - | repo.hooks.beforePendingQuarantineCheck = func(string) { checks.Add(1) } | |||
| 98 | - | if err := repo.Delete(context.Background(), "named"); err != nil { | |||
| 99 | - | t.Fatal(err) | |||
| 100 | - | } | |||
| 101 | - | if got := checks.Load(); got != 1 { | |||
| 102 | - | t.Fatalf("pending quarantine checks = %d, want one deterministic lookup", got) | |||
| 103 | - | } | |||
| 104 | - | if _, err := os.Lstat(filepath.Join(sessions, deletingSessionName(sessionKey("named")))); err != nil { | |||
| 105 | - | t.Fatalf("deterministic quarantine missing: %v", err) | |||
| 106 | - | } | |||
| 107 | - | } | |||
M internal/adapters/snapshot/repository_test.go +42 -21
| @@ -2,8 +2,11 @@ package snapshot | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | + | "crypto/sha256" | |||
| 5 | "testing" | 6 | "testing" | ||
| 6 | 7 | ||||
| 8 | + | "github.com/bnema/vev/internal/domain" | |||
| 9 | + | ||||
| 7 | "github.com/bnema/vev/internal/ports" | 10 | "github.com/bnema/vev/internal/ports" | ||
| 8 | codec "github.com/bnema/vev/internal/usecase/snapshot" | 11 | codec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 9 | ) | 12 | ) | ||
| @@ -16,7 +19,7 @@ func TestRepositoryPublishesAndLoadsCompleteGeneration(t *testing.T) { | |||||
| 16 | if err := repo.Publish(context.Background(), pub); err != nil { | 19 | if err := repo.Publish(context.Background(), pub); err != nil { | ||
| 17 | t.Fatalf("Publish: %v", err) | 20 | t.Fatalf("Publish: %v", err) | ||
| 18 | } | 21 | } | ||
| 19 | - | got, err := repo.Load(context.Background(), "named") | 22 | + | got, err := loadPublication(context.Background(), repo, pub) |
| 20 | if err != nil { | 23 | if err != nil { | ||
| 21 | t.Fatalf("Load: %v", err) | 24 | t.Fatalf("Load: %v", err) | ||
| 22 | } | 25 | } | ||
| @@ -25,36 +28,47 @@ func TestRepositoryPublishesAndLoadsCompleteGeneration(t *testing.T) { | |||||
| 25 | } | 28 | } | ||
| 26 | } | 29 | } | ||
| 27 | 30 | ||||
| 28 | - | func TestRepositoryDoesNotRewriteVerifiedImmutableBlob(t *testing.T) { | 31 | + | func loadPublication(ctx context.Context, repo *Repository, publication ports.SnapshotPublication) (ports.SnapshotGeneration, error) { |
| 29 | - | t.Parallel() | 32 | + | return repo.LoadCheckpoint(ctx, publication.IncarnationID, publication.Name, ports.CheckpointRef{ |
| 30 | - | dir := privateDir(t) | 33 | + | Generation: publication.Generation, |
| 31 | - | repo := NewRepository(dir) | 34 | + | ManifestDigest: codec.ManifestDigest(publication.Manifest), |
| 32 | - | pub := repositoryPublication(t, "named", 1, []byte("state")) | 35 | + | }) |
| 33 | - | if err := repo.Publish(context.Background(), pub); err != nil { | 36 | + | } |
| 34 | - | t.Fatal(err) | 37 | + | |
| 38 | + | func repositoryPublicationAfter(t *testing.T, repo *Repository, name string, generation uint64, payload []byte) ports.SnapshotPublication { | |||
| 39 | + | t.Helper() | |||
| 40 | + | return publicationWithCurrentParent(t, repo, repositoryPublication(t, name, generation, payload)) | |||
| 41 | + | } | |||
| 42 | + | ||||
| 43 | + | // publicationWithCurrentParent requires repositories to contain HEAD for every | |||
| 44 | + | // publication after generation one. Callers such as prepareHeadStage and | |||
| 45 | + | // seedCompletePublication must seed or publish the parent generation first. | |||
| 46 | + | func publicationWithCurrentParent(t *testing.T, repo *Repository, publication ports.SnapshotPublication) ports.SnapshotPublication { | |||
| 47 | + | t.Helper() | |||
| 48 | + | if publication.Generation == 1 { | |||
| 49 | + | return publication | |||
| 35 | } | 50 | } | ||
| 36 | - | writes := 0 | 51 | + | currentGeneration, digest, err := repo.readHead(publication.IncarnationID) |
| 37 | - | repo.hooks.beforeBlobWrite = func(string) error { writes++; return nil } | |||
| 38 | - | pub.Generation = 2 | |||
| 39 | - | manifest, err := codec.UnmarshalManifest(pub.Manifest) | |||
| 40 | if err != nil { | 52 | if err != nil { | ||
| 41 | - | t.Fatal(err) | 53 | + | t.Fatalf("read parent HEAD: %v", err) |
| 42 | } | 54 | } | ||
| 43 | - | manifest.Generation = 2 | 55 | + | parent := &domain.CheckpointRef{Generation: currentGeneration, ManifestDigest: digest} |
| 44 | - | pub.Manifest, err = codec.MarshalManifest(manifest) | 56 | + | manifest, err := codec.UnmarshalManifest(publication.Manifest) |
| 45 | if err != nil { | 57 | if err != nil { | ||
| 46 | t.Fatal(err) | 58 | t.Fatal(err) | ||
| 47 | } | 59 | } | ||
| 48 | - | if err := repo.Publish(context.Background(), pub); err != nil { | 60 | + | manifest.ParentCheckpoint = parent |
| 61 | + | publication.Manifest, err = codec.MarshalManifest(manifest) | |||
| 62 | + | if err != nil { | |||
| 49 | t.Fatal(err) | 63 | t.Fatal(err) | ||
| 50 | } | 64 | } | ||
| 51 | - | if writes != 0 { | 65 | + | publication.ParentCheckpoint = parent |
| 52 | - | t.Fatalf("blob writes = %d, want 0", writes) | 66 | + | return publication |
| 53 | - | } | |||
| 54 | } | 67 | } | ||
| 55 | 68 | ||||
| 56 | func repositoryPublication(t *testing.T, name string, generation uint64, payload []byte) ports.SnapshotPublication { | 69 | func repositoryPublication(t *testing.T, name string, generation uint64, payload []byte) ports.SnapshotPublication { | ||
| 57 | t.Helper() | 70 | t.Helper() | ||
| 71 | + | id := testIncarnationID(name) | |||
| 58 | tail, err := codec.MarshalObject(codec.HistoryTail, payload) | 72 | tail, err := codec.MarshalObject(codec.HistoryTail, payload) | ||
| 59 | if err != nil { | 73 | if err != nil { | ||
| 60 | t.Fatal(err) | 74 | t.Fatal(err) | ||
| @@ -63,9 +77,16 @@ func repositoryPublication(t *testing.T, name string, generation uint64, payload | |||||
| 63 | if err != nil { | 77 | if err != nil { | ||
| 64 | t.Fatal(err) | 78 | t.Fatal(err) | ||
| 65 | } | 79 | } | ||
| 66 | - | manifest, err := codec.MarshalManifest(codec.Manifest{Generation: generation, Name: name, Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}}) | 80 | + | manifest, err := codec.MarshalManifest(codec.Manifest{Generation: generation, IncarnationID: id, Name: name, Tabs: []codec.ManifestTab{{Cols: 1, Rows: 1, Panes: []codec.ManifestPane{{ID: "p", Tail: codec.ObjectRef{Kind: codec.HistoryTail, Digest: tail.Digest, Size: uint32(len(tail.Data))}, Visible: codec.ObjectRef{Kind: codec.Visible, Digest: visible.Digest, Size: uint32(len(visible.Data))}}}}}}) |
| 67 | if err != nil { | 81 | if err != nil { | ||
| 68 | t.Fatal(err) | 82 | t.Fatal(err) | ||
| 69 | } | 83 | } | ||
| 70 | - | return ports.SnapshotPublication{Name: name, Generation: generation, Manifest: manifest, Objects: []ports.SnapshotObject{tail, visible}} | 84 | + | return ports.SnapshotPublication{IncarnationID: id, Name: name, Generation: generation, Manifest: manifest, Objects: []ports.SnapshotObject{tail, visible}} |
| 85 | + | } | |||
| 86 | + | ||||
| 87 | + | func testIncarnationID(name string) domain.IncarnationID { | |||
| 88 | + | digest := sha256.Sum256([]byte("test incarnation: " + name)) | |||
| 89 | + | var id domain.IncarnationID | |||
| 90 | + | copy(id[:], digest[:]) | |||
| 91 | + | return id | |||
| 71 | } | 92 | } | ||
M internal/adapters/snapshot/safe_path.go +1 -21
| @@ -18,9 +18,6 @@ func (r *Repository) openRoot() (*os.Root, error) { | |||||
| 18 | if err != nil { | 18 | if err != nil { | ||
| 19 | return nil, err | 19 | return nil, err | ||
| 20 | } | 20 | } | ||
| 21 | - | if hook := r.hooks.afterOpenRoot; hook != nil { | |||
| 22 | - | hook() | |||
| 23 | - | } | |||
| 24 | current, err := os.Lstat(r.dir) | 21 | current, err := os.Lstat(r.dir) | ||
| 25 | if err != nil { | 22 | if err != nil { | ||
| 26 | return nil, r.closeRootOnError(root, err) | 23 | return nil, r.closeRootOnError(root, err) | ||
| @@ -47,11 +44,7 @@ func privateDirectory(fi os.FileInfo) bool { | |||||
| 47 | } | 44 | } | ||
| 48 | 45 | ||||
| 49 | func (r *Repository) closeRoot(root *os.Root) error { | 46 | func (r *Repository) closeRoot(root *os.Root) error { | ||
| 50 | - | var injected error | 47 | + | return root.Close() |
| 51 | - | if hook := r.hooks.closeRoot; hook != nil { | |||
| 52 | - | injected = hook() | |||
| 53 | - | } | |||
| 54 | - | return errors.Join(injected, root.Close()) | |||
| 55 | } | 48 | } | ||
| 56 | 49 | ||||
| 57 | func (r *Repository) closeRootOnError(root *os.Root, err error) error { | 50 | func (r *Repository) closeRootOnError(root *os.Root, err error) error { | ||
| @@ -110,19 +103,6 @@ func (r *Repository) repositoryRelative(path string) (string, bool) { | |||||
| 110 | return rel, true | 103 | return rel, true | ||
| 111 | } | 104 | } | ||
| 112 | 105 | ||||
| 113 | - | func (r *Repository) stat(path string) (fi os.FileInfo, err error) { | |||
| 114 | - | rel, ok := r.repositoryRelative(path) | |||
| 115 | - | if !ok { | |||
| 116 | - | return nil, fmt.Errorf("snapshot path outside repository") | |||
| 117 | - | } | |||
| 118 | - | root, err := r.openRoot() | |||
| 119 | - | if err != nil { | |||
| 120 | - | return nil, err | |||
| 121 | - | } | |||
| 122 | - | defer func() { joinCloseError(&err, "close snapshot root", r.closeRoot(root)) }() | |||
| 123 | - | return root.Lstat(rel) | |||
| 124 | - | } | |||
| 125 | - | ||||
| 126 | // createTempAt creates an exclusively owned temporary file in an already | 106 | // createTempAt creates an exclusively owned temporary file in an already | ||
| 127 | // validated directory. os.CreateTemp(path) would resolve that path again. | 107 | // validated directory. os.CreateTemp(path) would resolve that path again. | ||
| 128 | func (r *Repository) createTempAt(dir string) (temp *os.File, err error) { | 108 | func (r *Repository) createTempAt(dir string) (temp *os.File, err error) { | ||
M internal/app/daemon_integration_test.go +782 -0
| @@ -4,12 +4,16 @@ package app | |||||
| 4 | 4 | ||||
| 5 | import ( | 5 | import ( | ||
| 6 | "context" | 6 | "context" | ||
| 7 | + | "crypto/rand" | |||
| 7 | "errors" | 8 | "errors" | ||
| 8 | "io" | 9 | "io" | ||
| 9 | "log/slog" | 10 | "log/slog" | ||
| 11 | + | "net" | |||
| 10 | "os" | 12 | "os" | ||
| 11 | "path/filepath" | 13 | "path/filepath" | ||
| 12 | "strings" | 14 | "strings" | ||
| 15 | + | "sync" | |||
| 16 | + | "sync/atomic" | |||
| 13 | "testing" | 17 | "testing" | ||
| 14 | "time" | 18 | "time" | ||
| 15 | 19 | ||||
| @@ -17,10 +21,14 @@ import ( | |||||
| 17 | 21 | ||||
| 18 | "github.com/bnema/vev/internal/adapters/clock" | 22 | "github.com/bnema/vev/internal/adapters/clock" | ||
| 19 | "github.com/bnema/vev/internal/adapters/ipc" | 23 | "github.com/bnema/vev/internal/adapters/ipc" | ||
| 24 | + | "github.com/bnema/vev/internal/adapters/lifecycle" | |||
| 20 | "github.com/bnema/vev/internal/adapters/pty" | 25 | "github.com/bnema/vev/internal/adapters/pty" | ||
| 26 | + | "github.com/bnema/vev/internal/adapters/snapshot" | |||
| 21 | "github.com/bnema/vev/internal/domain" | 27 | "github.com/bnema/vev/internal/domain" | ||
| 28 | + | "github.com/bnema/vev/internal/persist" | |||
| 22 | "github.com/bnema/vev/internal/ports" | 29 | "github.com/bnema/vev/internal/ports" | ||
| 23 | "github.com/bnema/vev/internal/usecase/daemon" | 30 | "github.com/bnema/vev/internal/usecase/daemon" | ||
| 31 | + | "github.com/bnema/vev/internal/usecase/recovery" | |||
| 24 | "github.com/bnema/vev/pkg/vt" | 32 | "github.com/bnema/vev/pkg/vt" | ||
| 25 | ) | 33 | ) | ||
| 26 | 34 | ||||
| @@ -422,6 +430,780 @@ func TestIntegration_NamedSurvivesReattach(t *testing.T) { | |||||
| 422 | awaitText(t, p2, sz, "MARKER") | 430 | awaitText(t, p2, sz, "MARKER") | ||
| 423 | } | 431 | } | ||
| 424 | 432 | ||||
| 433 | + | func TestMultipleClientsOneLifecycleOwner(t *testing.T) { | |||
| 434 | + | dir := filepath.Join(t.TempDir(), "runtime") | |||
| 435 | + | owner, err := lifecycle.TryAcquire(dir) | |||
| 436 | + | require.NoError(t, err) | |||
| 437 | + | ||||
| 438 | + | ready := make(chan struct{}) | |||
| 439 | + | transport := &integrationTransport{} | |||
| 440 | + | dial := func(context.Context, string) (ports.Transport, error) { | |||
| 441 | + | select { | |||
| 442 | + | case <-ready: | |||
| 443 | + | return transport, nil | |||
| 444 | + | default: | |||
| 445 | + | return nil, os.ErrNotExist | |||
| 446 | + | } | |||
| 447 | + | } | |||
| 448 | + | var spawns atomic.Int32 | |||
| 449 | + | cfg := backoffConfig{initial: time.Millisecond, max: 2 * time.Millisecond, total: time.Second} | |||
| 450 | + | ||||
| 451 | + | const clients = 4 | |||
| 452 | + | errs := make(chan error, clients) | |||
| 453 | + | var wg sync.WaitGroup | |||
| 454 | + | for range clients { | |||
| 455 | + | wg.Go(func() { | |||
| 456 | + | _, err := ensureDaemonWithLifecycle(context.Background(), dir, dial, func() error { | |||
| 457 | + | spawns.Add(1) | |||
| 458 | + | return nil | |||
| 459 | + | }, cfg) | |||
| 460 | + | errs <- err | |||
| 461 | + | }) | |||
| 462 | + | } | |||
| 463 | + | require.Never(t, func() bool { return spawns.Load() != 0 }, 50*time.Millisecond, time.Millisecond) | |||
| 464 | + | close(ready) | |||
| 465 | + | require.NoError(t, owner.Release()) | |||
| 466 | + | wg.Wait() | |||
| 467 | + | close(errs) | |||
| 468 | + | for err := range errs { | |||
| 469 | + | require.NoError(t, err) | |||
| 470 | + | } | |||
| 471 | + | require.Zero(t, spawns.Load()) | |||
| 472 | + | } | |||
| 473 | + | ||||
| 474 | + | func TestListWaitsForLifecycleOwner(t *testing.T) { | |||
| 475 | + | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | |||
| 476 | + | t.Setenv("XDG_STATE_HOME", stateRoot) | |||
| 477 | + | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | |||
| 478 | + | p := newTestPersister(t, filepath.Join(stateRoot, "vev")) | |||
| 479 | + | require.NoError(t, p.Close()) | |||
| 480 | + | owner, err := lifecycle.TryAcquire(ipc.SocketDir()) | |||
| 481 | + | require.NoError(t, err) | |||
| 482 | + | ||||
| 483 | + | done := make(chan error, 1) | |||
| 484 | + | go func() { done <- runList(context.Background()) }() | |||
| 485 | + | require.Never(t, func() bool { return len(done) != 0 }, 50*time.Millisecond, time.Millisecond) | |||
| 486 | + | require.NoError(t, owner.Release()) | |||
| 487 | + | require.NoError(t, <-done) | |||
| 488 | + | } | |||
| 489 | + | ||||
| 490 | + | func TestOfflineKillWaitsForLifecycleOwner(t *testing.T) { | |||
| 491 | + | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | |||
| 492 | + | t.Setenv("XDG_STATE_HOME", stateRoot) | |||
| 493 | + | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | |||
| 494 | + | p := newTestPersister(t, filepath.Join(stateRoot, "vev")) | |||
| 495 | + | now := time.Now().UnixNano() | |||
| 496 | + | require.NoError(t, p.Save(persist.Record{Name: "named", IncarnationID: domain.IncarnationID{1}, CreatedAt: now, UpdatedAt: now})) | |||
| 497 | + | require.NoError(t, p.Close()) | |||
| 498 | + | owner, err := lifecycle.TryAcquire(ipc.SocketDir()) | |||
| 499 | + | require.NoError(t, err) | |||
| 500 | + | ||||
| 501 | + | done := make(chan error, 1) | |||
| 502 | + | go func() { done <- runKill(context.Background(), "named", false, false) }() | |||
| 503 | + | require.Never(t, func() bool { return len(done) != 0 }, 50*time.Millisecond, time.Millisecond) | |||
| 504 | + | require.NoError(t, owner.Release()) | |||
| 505 | + | require.NoError(t, <-done) | |||
| 506 | + | } | |||
| 507 | + | ||||
| 508 | + | func TestKillDaemonWaitsForOwnershipTransfer(t *testing.T) { | |||
| 509 | + | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | |||
| 510 | + | t.Setenv("XDG_STATE_HOME", stateRoot) | |||
| 511 | + | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | |||
| 512 | + | owner, err := lifecycle.TryAcquire(ipc.SocketDir()) | |||
| 513 | + | require.NoError(t, err) | |||
| 514 | + | _, served := startDaemonInDir(t, ipc.SocketDir()) | |||
| 515 | + | ||||
| 516 | + | done := make(chan error, 1) | |||
| 517 | + | go func() { done <- requestDaemonStop(context.Background()) }() | |||
| 518 | + | select { | |||
| 519 | + | case err := <-served: | |||
| 520 | + | require.NoError(t, err) | |||
| 521 | + | case <-time.After(time.Second): | |||
| 522 | + | t.Fatal("daemon did not stop") | |||
| 523 | + | } | |||
| 524 | + | require.Never(t, func() bool { return len(done) != 0 }, 50*time.Millisecond, time.Millisecond) | |||
| 525 | + | require.NoError(t, owner.Release()) | |||
| 526 | + | require.NoError(t, <-done) | |||
| 527 | + | } | |||
| 528 | + | ||||
| 529 | + | func TestLifecycleOwnershipOutlivesMaintenanceWriter(t *testing.T) { | |||
| 530 | + | runtimeDir := filepath.Join(t.TempDir(), "runtime") | |||
| 531 | + | stateDir := filepath.Join(t.TempDir(), "state") | |||
| 532 | + | catalogue := &lifecycleObservedCatalogue{ | |||
| 533 | + | Catalogue: newTestPersister(t, stateDir), | |||
| 534 | + | closed: make(chan struct{}), | |||
| 535 | + | } | |||
| 536 | + | require.NoError(t, catalogue.Create(domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}})) | |||
| 537 | + | repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots")) | |||
| 538 | + | shutdownClock := newLifecycleShutdownClock(t) | |||
| 539 | + | maintenanceRepository := &lifecycleBlockingMaintenanceRepository{ | |||
| 540 | + | SnapshotRepository: repository, | |||
| 541 | + | entered: make(chan struct{}), | |||
| 542 | + | release: make(chan struct{}), | |||
| 543 | + | returned: make(chan struct{}), | |||
| 544 | + | } | |||
| 545 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 546 | + | t.Cleanup(cancel) | |||
| 547 | + | var releaseOnce sync.Once | |||
| 548 | + | releaseMaintenance := func() { releaseOnce.Do(func() { close(maintenanceRepository.release) }) } | |||
| 549 | + | t.Cleanup(releaseMaintenance) | |||
| 550 | + | ||||
| 551 | + | callbackReturned := make(chan struct{}) | |||
| 552 | + | wrapperReturned := make(chan error, 1) | |||
| 553 | + | go func() { | |||
| 554 | + | wrapperReturned <- runWithLifecycleOwner(ctx, runtimeDir, stateDir, func(ctx context.Context) error { | |||
| 555 | + | defer close(callbackReturned) | |||
| 556 | + | d := daemon.New( | |||
| 557 | + | pty.NewFactory(), | |||
| 558 | + | shutdownClock, | |||
| 559 | + | discardLog(), | |||
| 560 | + | daemon.WithSnapshotRepository(repository), | |||
| 561 | + | daemon.WithDurableMaintenance(catalogue, maintenanceRepository), | |||
| 562 | + | daemon.WithCatalogue(catalogue, nil), | |||
| 563 | + | ) | |||
| 564 | + | if err := d.CollectStartupGarbage(ctx); err != nil { | |||
| 565 | + | return err | |||
| 566 | + | } | |||
| 567 | + | listener, err := ipc.Listen(runtimeDir) | |||
| 568 | + | if err != nil { | |||
| 569 | + | return err | |||
| 570 | + | } | |||
| 571 | + | return d.Serve(ctx, listener) | |||
| 572 | + | }) | |||
| 573 | + | }() | |||
| 574 | + | ||||
| 575 | + | awaitLifecycleStage(t, maintenanceRepository.entered, "pre-publication maintenance repository call") | |||
| 576 | + | cancel() | |||
| 577 | + | select { | |||
| 578 | + | case <-callbackReturned: | |||
| 579 | + | t.Fatal("Serve callback returned after shutdown deadline while maintenance was blocked") | |||
| 580 | + | case <-time.After(100 * time.Millisecond): | |||
| 581 | + | } | |||
| 582 | + | assertLifecycleStagePending(t, catalogue.closed, "catalogue close while maintenance is blocked") | |||
| 583 | + | assertLifecycleResultPending(t, wrapperReturned, "lifecycle wrapper return while maintenance is blocked") | |||
| 584 | + | _, err := lifecycle.TryAcquire(runtimeDir) | |||
| 585 | + | require.ErrorIs(t, err, lifecycle.ErrBusy) | |||
| 586 | + | ||||
| 587 | + | releaseMaintenance() | |||
| 588 | + | awaitLifecycleStage(t, maintenanceRepository.returned, "maintenance repository return") | |||
| 589 | + | awaitLifecycleStage(t, catalogue.closed, "catalogue close after maintenance") | |||
| 590 | + | awaitLifecycleStage(t, callbackReturned, "Serve callback return after catalogue close") | |||
| 591 | + | require.NoError(t, awaitLifecycleResult(t, wrapperReturned, "lifecycle wrapper return")) | |||
| 592 | + | owner, err := lifecycle.TryAcquire(runtimeDir) | |||
| 593 | + | require.NoError(t, err) | |||
| 594 | + | require.NoError(t, owner.Release()) | |||
| 595 | + | } | |||
| 596 | + | ||||
| 597 | + | type lifecycleBlockingMaintenanceRepository struct { | |||
| 598 | + | ports.SnapshotRepository | |||
| 599 | + | entered chan struct{} | |||
| 600 | + | release chan struct{} | |||
| 601 | + | returned chan struct{} | |||
| 602 | + | enterOnce sync.Once | |||
| 603 | + | returnOnce sync.Once | |||
| 604 | + | } | |||
| 605 | + | ||||
| 606 | + | func (r *lifecycleBlockingMaintenanceRepository) CollectGarbage(context.Context, map[domain.IncarnationID]domain.CheckpointRef) error { | |||
| 607 | + | r.enterOnce.Do(func() { close(r.entered) }) | |||
| 608 | + | <-r.release // Intentionally ignore cancellation: lifecycle ownership must outlive this call. | |||
| 609 | + | r.returnOnce.Do(func() { close(r.returned) }) | |||
| 610 | + | return nil | |||
| 611 | + | } | |||
| 612 | + | ||||
| 613 | + | type lifecycleObservedCatalogue struct { | |||
| 614 | + | ports.Catalogue | |||
| 615 | + | closed chan struct{} | |||
| 616 | + | closeOnce sync.Once | |||
| 617 | + | } | |||
| 618 | + | ||||
| 619 | + | func (c *lifecycleObservedCatalogue) Close() error { | |||
| 620 | + | err := c.Catalogue.Close() | |||
| 621 | + | c.closeOnce.Do(func() { close(c.closed) }) | |||
| 622 | + | return err | |||
| 623 | + | } | |||
| 624 | + | ||||
| 625 | + | func TestLifecycleOwnershipOutlivesSnapshotWriter(t *testing.T) { | |||
| 626 | + | result := runBlockedSnapshotWriterShutdown(t) | |||
| 627 | + | assertLifecycleStagePending(t, result.callbackReturned, "Serve callback return while snapshot writer is blocked") | |||
| 628 | + | assertLifecycleResultPending(t, result.wrapperReturned, "lifecycle wrapper return while snapshot writer is blocked") | |||
| 629 | + | _, err := lifecycle.TryAcquire(result.runtimeDir) | |||
| 630 | + | require.ErrorIs(t, err, lifecycle.ErrBusy) | |||
| 631 | + | ||||
| 632 | + | result.releaseWriter() | |||
| 633 | + | awaitLifecycleStage(t, result.publishReturned, "snapshot writer return") | |||
| 634 | + | require.NoError(t, awaitLifecycleResult(t, result.wrapperReturned, "lifecycle wrapper return")) | |||
| 635 | + | owner, err := lifecycle.TryAcquire(result.runtimeDir) | |||
| 636 | + | require.NoError(t, err) | |||
| 637 | + | require.NoError(t, owner.Release()) | |||
| 638 | + | } | |||
| 639 | + | ||||
| 640 | + | func TestLifecycleCallbackWaitsForEveryWriter(t *testing.T) { | |||
| 641 | + | result := runBlockedSnapshotWriterShutdown(t) | |||
| 642 | + | assertLifecycleStagePending(t, result.callbackReturned, "Serve callback return while snapshot writer is blocked") | |||
| 643 | + | ||||
| 644 | + | result.releaseWriter() | |||
| 645 | + | awaitLifecycleStage(t, result.publishReturned, "snapshot writer return") | |||
| 646 | + | awaitLifecycleStage(t, result.callbackReturned, "Serve callback return after snapshot writer") | |||
| 647 | + | require.NoError(t, awaitLifecycleResult(t, result.wrapperReturned, "lifecycle wrapper return")) | |||
| 648 | + | } | |||
| 649 | + | ||||
| 650 | + | func TestFinalCheckpointTimeoutKeepsOwner(t *testing.T) { | |||
| 651 | + | result := runBlockedSnapshotWriterShutdown(t) | |||
| 652 | + | assertLifecycleResultPending(t, result.wrapperReturned, "ownership release after checkpoint timeout with writer alive") | |||
| 653 | + | _, err := lifecycle.TryAcquire(result.runtimeDir) | |||
| 654 | + | require.ErrorIs(t, err, lifecycle.ErrBusy) | |||
| 655 | + | ||||
| 656 | + | result.releaseWriter() | |||
| 657 | + | require.NoError(t, awaitLifecycleResult(t, result.wrapperReturned, "lifecycle wrapper return")) | |||
| 658 | + | } | |||
| 659 | + | ||||
| 660 | + | // Startup restoration repairs HEADs, promotes fallbacks, and replaces catalogue | |||
| 661 | + | // records. Lifecycle ownership must therefore outlive it: releasing the flock | |||
| 662 | + | // while a restoration repository call is still running would let a second | |||
| 663 | + | // daemon mutate the same durable state. | |||
| 664 | + | func TestLifecycleOwnershipOutlivesRestorationWriter(t *testing.T) { | |||
| 665 | + | stateDir := filepath.Join(t.TempDir(), "state") | |||
| 666 | + | repository := snapshot.NewRepository(filepath.Join(stateDir, "snapshots")) | |||
| 667 | + | checkpointed := publishRestorableCheckpoint(t, stateDir, repository) | |||
| 668 | + | ||||
| 669 | + | runtimeDir := filepath.Join(t.TempDir(), "runtime") | |||
| 670 | + | blocking := &lifecycleBlockingRestore{ | |||
| 671 | + | SnapshotRepository: repository, | |||
| 672 | + | entered: make(chan struct{}), | |||
| 673 | + | release: make(chan struct{}), | |||
| 674 | + | returned: make(chan struct{}), | |||
| 675 | + | } | |||
| 676 | + | var releaseOnce sync.Once | |||
| 677 | + | releaseRestore := func() { releaseOnce.Do(func() { close(blocking.release) }) } | |||
| 678 | + | t.Cleanup(releaseRestore) | |||
| 679 | + | ||||
| 680 | + | shutdownClock := newLifecycleShutdownClock(t) | |||
| 681 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 682 | + | t.Cleanup(cancel) | |||
| 683 | + | listenerReady := make(chan struct{}) | |||
| 684 | + | callbackReturned := make(chan struct{}) | |||
| 685 | + | wrapperReturned := make(chan error, 1) | |||
| 686 | + | go func() { | |||
| 687 | + | wrapperReturned <- runWithLifecycleOwner(ctx, runtimeDir, stateDir, func(ctx context.Context) error { | |||
| 688 | + | defer close(callbackReturned) | |||
| 689 | + | opened, err := persist.OpenOrCreate(stateDir) | |||
| 690 | + | if err != nil { | |||
| 691 | + | return err | |||
| 692 | + | } | |||
| 693 | + | listener, err := ipc.Listen(runtimeDir) | |||
| 694 | + | if err != nil { | |||
| 695 | + | return errors.Join(err, opened.Catalogue.Close()) | |||
| 696 | + | } | |||
| 697 | + | close(listenerReady) | |||
| 698 | + | d := daemon.New( | |||
| 699 | + | pty.NewFactory(), | |||
| 700 | + | shutdownClock, | |||
| 701 | + | discardLog(), | |||
| 702 | + | daemon.WithShell("/bin/cat", nil), | |||
| 703 | + | daemon.WithCatalogue(opened.Catalogue, opened.Records), | |||
| 704 | + | daemon.WithSnapshotRepository(blocking), | |||
| 705 | + | ) | |||
| 706 | + | return d.Serve(ctx, listener) | |||
| 707 | + | }) | |||
| 708 | + | }() | |||
| 709 | + | ||||
| 710 | + | awaitLifecycleStage(t, listenerReady, "daemon listener") | |||
| 711 | + | awaitLifecycleStage(t, blocking.entered, "restoration repository call") | |||
| 712 | + | require.Equal(t, checkpointed, blocking.repairedName(), "restoration must target the checkpointed session") | |||
| 713 | + | ||||
| 714 | + | cancel() | |||
| 715 | + | shutdownClock.nextTimer(t).fire() | |||
| 716 | + | select { | |||
| 717 | + | case <-callbackReturned: | |||
| 718 | + | t.Fatal("Serve callback returned after shutdown deadline while restoration was blocked") | |||
| 719 | + | case <-time.After(100 * time.Millisecond): | |||
| 720 | + | } | |||
| 721 | + | assertLifecycleStagePending(t, callbackReturned, "Serve callback return while restoration is blocked") | |||
| 722 | + | assertLifecycleResultPending(t, wrapperReturned, "lifecycle wrapper return while restoration is blocked") | |||
| 723 | + | _, err := lifecycle.TryAcquire(runtimeDir) | |||
| 724 | + | require.ErrorIs(t, err, lifecycle.ErrBusy) | |||
| 725 | + | ||||
| 726 | + | releaseRestore() | |||
| 727 | + | awaitLifecycleStage(t, blocking.returned, "restoration repository return") | |||
| 728 | + | awaitLifecycleStage(t, callbackReturned, "Serve callback return after restoration") | |||
| 729 | + | require.NoError(t, awaitLifecycleResult(t, wrapperReturned, "lifecycle wrapper return")) | |||
| 730 | + | owner, err := lifecycle.TryAcquire(runtimeDir) | |||
| 731 | + | require.NoError(t, err) | |||
| 732 | + | require.NoError(t, owner.Release()) | |||
| 733 | + | } | |||
| 734 | + | ||||
| 735 | + | // publishRestorableCheckpoint runs a complete daemon over the real catalogue and | |||
| 736 | + | // repository until one named session owns a committed checkpoint, then shuts it | |||
| 737 | + | // down. The returned name is restorable by any later daemon on the same state. | |||
| 738 | + | func publishRestorableCheckpoint(t *testing.T, stateDir string, repository *snapshot.Repository) string { | |||
| 739 | + | t.Helper() | |||
| 740 | + | const name = "restorable" | |||
| 741 | + | runtimeDir := filepath.Join(t.TempDir(), "seed-runtime") | |||
| 742 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 743 | + | defer cancel() | |||
| 744 | + | ||||
| 745 | + | opened, err := persist.OpenOrCreate(stateDir) | |||
| 746 | + | require.NoError(t, err) | |||
| 747 | + | coordinator := recovery.NewCoordinator(opened.Catalogue, repository, rand.Reader) | |||
| 748 | + | listener, err := ipc.Listen(runtimeDir) | |||
| 749 | + | require.NoError(t, err) | |||
| 750 | + | d := daemon.New( | |||
| 751 | + | pty.NewFactory(), | |||
| 752 | + | clock.New(), | |||
| 753 | + | discardLog(), | |||
| 754 | + | daemon.WithShell("/bin/cat", nil), | |||
| 755 | + | daemon.WithCatalogue(opened.Catalogue, opened.Records), | |||
| 756 | + | daemon.WithSnapshotRepository(repository), | |||
| 757 | + | daemon.WithRecoveryCoordinator(coordinator), | |||
| 758 | + | ) | |||
| 759 | + | served := make(chan error, 1) | |||
| 760 | + | go func() { served <- d.Serve(ctx, listener) }() | |||
| 761 | + | ||||
| 762 | + | tr, _ := attach(t, runtimeDir, ports.IntentNew, name, domain.Size{Cols: 80, Rows: 24}) | |||
| 763 | + | require.NoError(t, tr.Send(ports.Frame{Type: ports.MsgInput, Payload: ports.MarshalInput(ports.Input{Data: []byte("checkpoint me\n")})})) | |||
| 764 | + | require.Eventually(t, func() bool { | |||
| 765 | + | record, ok, _ := opened.Catalogue.Record(name) | |||
| 766 | + | return ok && record.Committed != nil && record.DegradedReason == "" | |||
| 767 | + | }, 5*time.Second, 10*time.Millisecond, "session never committed a checkpoint") | |||
| 768 | + | require.NoError(t, tr.Close()) | |||
| 769 | + | ||||
| 770 | + | cancel() | |||
| 771 | + | require.NoError(t, awaitLifecycleResult(t, served, "seed daemon shutdown")) | |||
| 772 | + | return name | |||
| 773 | + | } | |||
| 774 | + | ||||
| 775 | + | type lifecycleBlockingRestore struct { | |||
| 776 | + | ports.SnapshotRepository | |||
| 777 | + | entered chan struct{} | |||
| 778 | + | release chan struct{} | |||
| 779 | + | returned chan struct{} | |||
| 780 | + | enterOnce sync.Once | |||
| 781 | + | returnOnce sync.Once | |||
| 782 | + | mu sync.Mutex | |||
| 783 | + | repaired string | |||
| 784 | + | } | |||
| 785 | + | ||||
| 786 | + | func (r *lifecycleBlockingRestore) LoadCheckpoint(ctx context.Context, id domain.IncarnationID, name string, ref domain.CheckpointRef) (ports.SnapshotGeneration, error) { | |||
| 787 | + | r.mu.Lock() | |||
| 788 | + | r.repaired = name | |||
| 789 | + | r.mu.Unlock() | |||
| 790 | + | return r.SnapshotRepository.LoadCheckpoint(ctx, id, name, ref) | |||
| 791 | + | } | |||
| 792 | + | ||||
| 793 | + | func (r *lifecycleBlockingRestore) ReconcileCheckpoint(ctx context.Context, id domain.IncarnationID, ref domain.CheckpointRef) error { | |||
| 794 | + | r.enterOnce.Do(func() { close(r.entered) }) | |||
| 795 | + | <-r.release // Intentionally ignore cancellation: lifecycle ownership must outlive this call. | |||
| 796 | + | r.returnOnce.Do(func() { close(r.returned) }) | |||
| 797 | + | return r.SnapshotRepository.ReconcileCheckpoint(ctx, id, ref) | |||
| 798 | + | } | |||
| 799 | + | ||||
| 800 | + | func (r *lifecycleBlockingRestore) repairedName() string { | |||
| 801 | + | r.mu.Lock() | |||
| 802 | + | defer r.mu.Unlock() | |||
| 803 | + | return r.repaired | |||
| 804 | + | } | |||
| 805 | + | ||||
| 806 | + | type blockedSnapshotWriterShutdown struct { | |||
| 807 | + | runtimeDir string | |||
| 808 | + | callbackReturned <-chan struct{} | |||
| 809 | + | wrapperReturned <-chan error | |||
| 810 | + | publishReturned <-chan struct{} | |||
| 811 | + | releaseWriter func() | |||
| 812 | + | } | |||
| 813 | + | ||||
| 814 | + | func runBlockedSnapshotWriterShutdown(t *testing.T) blockedSnapshotWriterShutdown { | |||
| 815 | + | t.Helper() | |||
| 816 | + | runtimeDir := filepath.Join(t.TempDir(), "runtime") | |||
| 817 | + | stateDir := filepath.Join(t.TempDir(), "state") | |||
| 818 | + | shutdownClock := newLifecycleShutdownClock(t) | |||
| 819 | + | repository := newLifecycleBlockingRepository() | |||
| 820 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 821 | + | t.Cleanup(cancel) | |||
| 822 | + | ||||
| 823 | + | listenerReady := make(chan struct{}) | |||
| 824 | + | listenerClosed := make(chan struct{}) | |||
| 825 | + | callbackReturned := make(chan struct{}) | |||
| 826 | + | wrapperReturned := make(chan error, 1) | |||
| 827 | + | go func() { | |||
| 828 | + | wrapperReturned <- runWithLifecycleOwner(ctx, runtimeDir, stateDir, func(ctx context.Context) error { | |||
| 829 | + | defer close(callbackReturned) | |||
| 830 | + | listener, err := ipc.Listen(runtimeDir) | |||
| 831 | + | if err != nil { | |||
| 832 | + | return err | |||
| 833 | + | } | |||
| 834 | + | observed := &lifecycleObservedListener{Listener: listener, closed: listenerClosed} | |||
| 835 | + | close(listenerReady) | |||
| 836 | + | d := daemon.New( | |||
| 837 | + | pty.NewFactory(), | |||
| 838 | + | shutdownClock, | |||
| 839 | + | discardLog(), | |||
| 840 | + | daemon.WithShell("/bin/cat", nil), | |||
| 841 | + | daemon.WithSnapshotRepository(repository), | |||
| 842 | + | ) | |||
| 843 | + | return d.Serve(ctx, observed) | |||
| 844 | + | }) | |||
| 845 | + | }() | |||
| 846 | + | ||||
| 847 | + | awaitLifecycleStage(t, listenerReady, "daemon listener") | |||
| 848 | + | tr, _ := attach(t, runtimeDir, ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24}) | |||
| 849 | + | require.NoError(t, tr.Send(ports.Frame{Type: ports.MsgInput, Payload: ports.MarshalInput(ports.Input{Data: []byte("dirty state\n")})})) | |||
| 850 | + | awaitLifecycleStage(t, repository.entered, "snapshot publication") | |||
| 851 | + | require.NoError(t, tr.Close()) | |||
| 852 | + | require.Eventually(t, func() bool { | |||
| 853 | + | sessions := listRemoteSessions(t, runtimeDir) | |||
| 854 | + | return len(sessions.Sessions) == 1 && !sessions.Sessions[0].Attached | |||
| 855 | + | }, time.Second, 10*time.Millisecond) | |||
| 856 | + | cancel() | |||
| 857 | + | awaitLifecycleStage(t, listenerClosed, "listener close") | |||
| 858 | + | ||||
| 859 | + | shutdownClock.nextTimer(t).fire() | |||
| 860 | + | select { | |||
| 861 | + | case <-callbackReturned: | |||
| 862 | + | t.Fatal("Serve callback returned after checkpoint timeout while snapshot writer was blocked") | |||
| 863 | + | case <-time.After(100 * time.Millisecond): | |||
| 864 | + | } | |||
| 865 | + | assertLifecycleStagePending(t, callbackReturned, "Serve callback return after checkpoint timeout") | |||
| 866 | + | assertLifecycleResultPending(t, wrapperReturned, "lifecycle wrapper return after checkpoint timeout") | |||
| 867 | + | ||||
| 868 | + | var releaseOnce sync.Once | |||
| 869 | + | releaseWriter := func() { releaseOnce.Do(func() { close(repository.release) }) } | |||
| 870 | + | t.Cleanup(releaseWriter) | |||
| 871 | + | return blockedSnapshotWriterShutdown{ | |||
| 872 | + | runtimeDir: runtimeDir, | |||
| 873 | + | callbackReturned: callbackReturned, | |||
| 874 | + | wrapperReturned: wrapperReturned, | |||
| 875 | + | publishReturned: repository.returned, | |||
| 876 | + | releaseWriter: releaseWriter, | |||
| 877 | + | } | |||
| 878 | + | } | |||
| 879 | + | ||||
| 880 | + | type lifecycleBlockingRepository struct { | |||
| 881 | + | ports.SnapshotRepository | |||
| 882 | + | entered chan struct{} | |||
| 883 | + | release chan struct{} | |||
| 884 | + | returned chan struct{} | |||
| 885 | + | enterOnce sync.Once | |||
| 886 | + | returnOnce sync.Once | |||
| 887 | + | } | |||
| 888 | + | ||||
| 889 | + | func newLifecycleBlockingRepository() *lifecycleBlockingRepository { | |||
| 890 | + | return &lifecycleBlockingRepository{ | |||
| 891 | + | entered: make(chan struct{}), | |||
| 892 | + | release: make(chan struct{}), | |||
| 893 | + | returned: make(chan struct{}), | |||
| 894 | + | } | |||
| 895 | + | } | |||
| 896 | + | ||||
| 897 | + | func (r *lifecycleBlockingRepository) Publish(context.Context, ports.SnapshotPublication) error { | |||
| 898 | + | r.enterOnce.Do(func() { close(r.entered) }) | |||
| 899 | + | <-r.release // Intentionally ignore cancellation: lifecycle ownership must outlive this call. | |||
| 900 | + | r.returnOnce.Do(func() { close(r.returned) }) | |||
| 901 | + | return nil | |||
| 902 | + | } | |||
| 903 | + | ||||
| 904 | + | type lifecycleShutdownClock struct { | |||
| 905 | + | t *testing.T | |||
| 906 | + | base ports.Clock | |||
| 907 | + | timers chan *lifecycleManualTimer | |||
| 908 | + | } | |||
| 909 | + | ||||
| 910 | + | func newLifecycleShutdownClock(t *testing.T) *lifecycleShutdownClock { | |||
| 911 | + | t.Helper() | |||
| 912 | + | return &lifecycleShutdownClock{t: t, base: clock.New(), timers: make(chan *lifecycleManualTimer, 4)} | |||
| 913 | + | } | |||
| 914 | + | ||||
| 915 | + | func (c *lifecycleShutdownClock) Now() time.Time { return c.base.Now() } | |||
| 916 | + | func (c *lifecycleShutdownClock) NewTimer(delay time.Duration) ports.Timer { | |||
| 917 | + | if delay != daemon.SnapshotShutdownTimeout() { | |||
| 918 | + | return c.base.NewTimer(delay) | |||
| 919 | + | } | |||
| 920 | + | timer := &lifecycleManualTimer{t: c.t, ch: make(chan time.Time, 1)} | |||
| 921 | + | select { | |||
| 922 | + | case c.timers <- timer: | |||
| 923 | + | default: | |||
| 924 | + | c.t.Errorf("unexpected additional snapshot shutdown timer with delay %s", delay) | |||
| 925 | + | timer.fire() | |||
| 926 | + | } | |||
| 927 | + | return timer | |||
| 928 | + | } | |||
| 929 | + | ||||
| 930 | + | func (c *lifecycleShutdownClock) nextTimer(t *testing.T) *lifecycleManualTimer { | |||
| 931 | + | t.Helper() | |||
| 932 | + | select { | |||
| 933 | + | case timer := <-c.timers: | |||
| 934 | + | return timer | |||
| 935 | + | case <-time.After(3 * time.Second): | |||
| 936 | + | t.Fatal("timed out waiting for shutdown deadline timer") | |||
| 937 | + | return nil | |||
| 938 | + | } | |||
| 939 | + | } | |||
| 940 | + | ||||
| 941 | + | type lifecycleManualTimer struct { | |||
| 942 | + | t *testing.T | |||
| 943 | + | ch chan time.Time | |||
| 944 | + | stopped atomic.Bool | |||
| 945 | + | } | |||
| 946 | + | ||||
| 947 | + | func (t *lifecycleManualTimer) C() <-chan time.Time { return t.ch } | |||
| 948 | + | func (t *lifecycleManualTimer) Reset(delay time.Duration) bool { | |||
| 949 | + | if delay != daemon.SnapshotShutdownTimeout() { | |||
| 950 | + | t.t.Errorf("snapshot shutdown timer reset with delay %s, want %s", delay, daemon.SnapshotShutdownTimeout()) | |||
| 951 | + | } | |||
| 952 | + | t.stopped.Store(false) | |||
| 953 | + | return false | |||
| 954 | + | } | |||
| 955 | + | func (t *lifecycleManualTimer) Stop() bool { return !t.stopped.Swap(true) } | |||
| 956 | + | func (t *lifecycleManualTimer) fire() { | |||
| 957 | + | if t.stopped.Load() { | |||
| 958 | + | return | |||
| 959 | + | } | |||
| 960 | + | select { | |||
| 961 | + | case t.ch <- time.Now(): | |||
| 962 | + | default: | |||
| 963 | + | } | |||
| 964 | + | } | |||
| 965 | + | ||||
| 966 | + | func TestLifecycleManualTimerRepeatedFireIsNonBlocking(t *testing.T) { | |||
| 967 | + | timer := &lifecycleManualTimer{t: t, ch: make(chan time.Time, 1)} | |||
| 968 | + | timer.fire() | |||
| 969 | + | ||||
| 970 | + | returned := make(chan struct{}) | |||
| 971 | + | go func() { | |||
| 972 | + | timer.fire() | |||
| 973 | + | close(returned) | |||
| 974 | + | }() | |||
| 975 | + | ||||
| 976 | + | select { | |||
| 977 | + | case <-returned: | |||
| 978 | + | case <-time.After(time.Second): | |||
| 979 | + | t.Fatal("repeated manual timer fire blocked on a pending tick") | |||
| 980 | + | } | |||
| 981 | + | } | |||
| 982 | + | ||||
| 983 | + | type lifecycleObservedListener struct { | |||
| 984 | + | ports.Listener | |||
| 985 | + | closed chan struct{} | |||
| 986 | + | once sync.Once | |||
| 987 | + | } | |||
| 988 | + | ||||
| 989 | + | func (l *lifecycleObservedListener) Close() error { | |||
| 990 | + | err := l.Listener.Close() | |||
| 991 | + | l.once.Do(func() { close(l.closed) }) | |||
| 992 | + | return err | |||
| 993 | + | } | |||
| 994 | + | ||||
| 995 | + | func TestLifecycleSocketCloseCatalogueRace(t *testing.T) { | |||
| 996 | + | runtimeDir := filepath.Join(t.TempDir(), "runtime") | |||
| 997 | + | stateDir := filepath.Join(t.TempDir(), "state") | |||
| 998 | + | oldOwner, err := lifecycle.TryAcquire(runtimeDir) | |||
| 999 | + | require.NoError(t, err) | |||
| 1000 | + | oldOpened, err := persist.OpenOrCreate(stateDir) | |||
| 1001 | + | require.NoError(t, err) | |||
| 1002 | + | oldListener, err := ipc.Listen(runtimeDir) | |||
| 1003 | + | require.NoError(t, err) | |||
| 1004 | + | ||||
| 1005 | + | socketCloseEntered := make(chan struct{}) | |||
| 1006 | + | allowSocketClose := make(chan struct{}) | |||
| 1007 | + | socketClosed := make(chan struct{}) | |||
| 1008 | + | catalogueCloseEntered := make(chan struct{}) | |||
| 1009 | + | allowCatalogueClose := make(chan struct{}) | |||
| 1010 | + | catalogueClosed := make(chan struct{}) | |||
| 1011 | + | ownerReleaseEntered := make(chan struct{}) | |||
| 1012 | + | allowOwnerRelease := make(chan struct{}) | |||
| 1013 | + | ownerReleased := make(chan struct{}) | |||
| 1014 | + | teardownEvents := make(chan string, 3) | |||
| 1015 | + | ||||
| 1016 | + | controlledListener := &lifecycleControlledListener{ | |||
| 1017 | + | Listener: oldListener, | |||
| 1018 | + | entered: socketCloseEntered, | |||
| 1019 | + | proceed: allowSocketClose, | |||
| 1020 | + | closed: socketClosed, | |||
| 1021 | + | events: teardownEvents, | |||
| 1022 | + | } | |||
| 1023 | + | controlledCatalogue := &lifecycleControlledCatalogue{ | |||
| 1024 | + | Catalogue: oldOpened.Catalogue, | |||
| 1025 | + | entered: catalogueCloseEntered, | |||
| 1026 | + | proceed: allowCatalogueClose, | |||
| 1027 | + | closed: catalogueClosed, | |||
| 1028 | + | events: teardownEvents, | |||
| 1029 | + | } | |||
| 1030 | + | oldDaemon := daemon.New(pty.NewFactory(), clock.New(), discardLog(), daemon.WithCatalogue(controlledCatalogue, oldOpened.Records)) | |||
| 1031 | + | oldCtx, stopOld := context.WithCancel(context.Background()) | |||
| 1032 | + | oldDone := make(chan error, 1) | |||
| 1033 | + | go func() { | |||
| 1034 | + | serveErr := oldDaemon.Serve(oldCtx, controlledListener) | |||
| 1035 | + | teardownEvents <- "owner-release" | |||
| 1036 | + | close(ownerReleaseEntered) | |||
| 1037 | + | <-allowOwnerRelease | |||
| 1038 | + | releaseErr := oldOwner.Release() | |||
| 1039 | + | close(ownerReleased) | |||
| 1040 | + | oldDone <- errors.Join(serveErr, releaseErr) | |||
| 1041 | + | }() | |||
| 1042 | + | ||||
| 1043 | + | newAcquireAttempted := make(chan struct{}) | |||
| 1044 | + | newDurableOpen := make(chan struct{}) | |||
| 1045 | + | newListen := make(chan struct{}) | |||
| 1046 | + | startupEvents := make(chan string, 3) | |||
| 1047 | + | newDone := make(chan error, 1) | |||
| 1048 | + | go func() { | |||
| 1049 | + | newDone <- runWithLifecycleOwnerDeps(context.Background(), runtimeDir, stateDir, func(ctx context.Context) error { | |||
| 1050 | + | close(newDurableOpen) | |||
| 1051 | + | opened, err := persist.OpenOrCreate(stateDir) | |||
| 1052 | + | if err != nil { | |||
| 1053 | + | return err | |||
| 1054 | + | } | |||
| 1055 | + | startupEvents <- "durable-open" | |||
| 1056 | + | ||||
| 1057 | + | close(newListen) | |||
| 1058 | + | listener, err := ipc.Listen(runtimeDir) | |||
| 1059 | + | if err != nil { | |||
| 1060 | + | return errors.Join(err, opened.Catalogue.Close()) | |||
| 1061 | + | } | |||
| 1062 | + | startupEvents <- "listen" | |||
| 1063 | + | return errors.Join(listener.Close(), opened.Catalogue.Close()) | |||
| 1064 | + | }, lifecycleStartupDeps{ | |||
| 1065 | + | ensurePrivate: func(string) error { return nil }, | |||
| 1066 | + | acquire: func(ctx context.Context, dir string, retry time.Duration) (lifecycleOwnership, error) { | |||
| 1067 | + | close(newAcquireAttempted) | |||
| 1068 | + | owner, err := lifecycle.Acquire(ctx, dir, retry) | |||
| 1069 | + | if err != nil { | |||
| 1070 | + | return nil, err | |||
| 1071 | + | } | |||
| 1072 | + | <-ownerReleased | |||
| 1073 | + | startupEvents <- "owner-acquired" | |||
| 1074 | + | return owner, nil | |||
| 1075 | + | }, | |||
| 1076 | + | }) | |||
| 1077 | + | }() | |||
| 1078 | + | ||||
| 1079 | + | awaitLifecycleStage(t, newAcquireAttempted, "new lifecycle acquisition attempt") | |||
| 1080 | + | assertLifecycleStagePending(t, newDurableOpen, "new durable open before old teardown") | |||
| 1081 | + | assertLifecycleStagePending(t, newListen, "new listen before old teardown") | |||
| 1082 | + | ||||
| 1083 | + | stopOld() | |||
| 1084 | + | awaitLifecycleStage(t, socketCloseEntered, "old socket close") | |||
| 1085 | + | assertNewDaemonStartupPending(t, newDurableOpen, newListen, "socket close") | |||
| 1086 | + | close(allowSocketClose) | |||
| 1087 | + | awaitLifecycleStage(t, socketClosed, "old socket closed") | |||
| 1088 | + | ||||
| 1089 | + | awaitLifecycleStage(t, catalogueCloseEntered, "old catalogue close") | |||
| 1090 | + | assertNewDaemonStartupPending(t, newDurableOpen, newListen, "catalogue close") | |||
| 1091 | + | close(allowCatalogueClose) | |||
| 1092 | + | awaitLifecycleStage(t, catalogueClosed, "old catalogue closed") | |||
| 1093 | + | ||||
| 1094 | + | awaitLifecycleStage(t, ownerReleaseEntered, "old lifecycle owner release") | |||
| 1095 | + | assertNewDaemonStartupPending(t, newDurableOpen, newListen, "owner release") | |||
| 1096 | + | close(allowOwnerRelease) | |||
| 1097 | + | awaitLifecycleStage(t, ownerReleased, "old lifecycle owner released") | |||
| 1098 | + | ||||
| 1099 | + | require.NoError(t, awaitLifecycleResult(t, oldDone, "old daemon teardown")) | |||
| 1100 | + | require.NoError(t, awaitLifecycleResult(t, newDone, "new daemon startup")) | |||
| 1101 | + | require.Equal(t, []string{"socket-close", "catalogue-close", "owner-release"}, []string{ | |||
| 1102 | + | <-teardownEvents, | |||
| 1103 | + | <-teardownEvents, | |||
| 1104 | + | <-teardownEvents, | |||
| 1105 | + | }) | |||
| 1106 | + | require.Equal(t, []string{"owner-acquired", "durable-open", "listen"}, []string{ | |||
| 1107 | + | <-startupEvents, | |||
| 1108 | + | <-startupEvents, | |||
| 1109 | + | <-startupEvents, | |||
| 1110 | + | }) | |||
| 1111 | + | } | |||
| 1112 | + | ||||
| 1113 | + | func awaitLifecycleStage(t *testing.T, stage <-chan struct{}, name string) { | |||
| 1114 | + | t.Helper() | |||
| 1115 | + | select { | |||
| 1116 | + | case <-stage: | |||
| 1117 | + | case <-time.After(3 * time.Second): | |||
| 1118 | + | t.Fatalf("timed out waiting for %s", name) | |||
| 1119 | + | } | |||
| 1120 | + | } | |||
| 1121 | + | ||||
| 1122 | + | func awaitLifecycleResult(t *testing.T, result <-chan error, name string) error { | |||
| 1123 | + | t.Helper() | |||
| 1124 | + | select { | |||
| 1125 | + | case err := <-result: | |||
| 1126 | + | return err | |||
| 1127 | + | case <-time.After(3 * time.Second): | |||
| 1128 | + | t.Fatalf("timed out waiting for %s", name) | |||
| 1129 | + | return nil | |||
| 1130 | + | } | |||
| 1131 | + | } | |||
| 1132 | + | ||||
| 1133 | + | func assertLifecycleStagePending(t *testing.T, stage <-chan struct{}, name string) { | |||
| 1134 | + | t.Helper() | |||
| 1135 | + | select { | |||
| 1136 | + | case <-stage: | |||
| 1137 | + | t.Fatalf("unexpected %s", name) | |||
| 1138 | + | default: | |||
| 1139 | + | } | |||
| 1140 | + | } | |||
| 1141 | + | ||||
| 1142 | + | func assertLifecycleResultPending(t *testing.T, result <-chan error, name string) { | |||
| 1143 | + | t.Helper() | |||
| 1144 | + | select { | |||
| 1145 | + | case err := <-result: | |||
| 1146 | + | t.Fatalf("unexpected %s: %v", name, err) | |||
| 1147 | + | default: | |||
| 1148 | + | } | |||
| 1149 | + | } | |||
| 1150 | + | ||||
| 1151 | + | func assertNewDaemonStartupPending(t *testing.T, durableOpen, listen <-chan struct{}, heldAt string) { | |||
| 1152 | + | t.Helper() | |||
| 1153 | + | assertLifecycleStagePending(t, durableOpen, "new durable open while old daemon held at "+heldAt) | |||
| 1154 | + | assertLifecycleStagePending(t, listen, "new listen while old daemon held at "+heldAt) | |||
| 1155 | + | } | |||
| 1156 | + | ||||
| 1157 | + | type lifecycleControlledListener struct { | |||
| 1158 | + | ports.Listener | |||
| 1159 | + | entered chan struct{} | |||
| 1160 | + | proceed chan struct{} | |||
| 1161 | + | closed chan struct{} | |||
| 1162 | + | events chan<- string | |||
| 1163 | + | once sync.Once | |||
| 1164 | + | err error | |||
| 1165 | + | } | |||
| 1166 | + | ||||
| 1167 | + | func (l *lifecycleControlledListener) Close() error { | |||
| 1168 | + | l.once.Do(func() { | |||
| 1169 | + | l.events <- "socket-close" | |||
| 1170 | + | close(l.entered) | |||
| 1171 | + | <-l.proceed | |||
| 1172 | + | l.err = l.Listener.Close() | |||
| 1173 | + | close(l.closed) | |||
| 1174 | + | }) | |||
| 1175 | + | return l.err | |||
| 1176 | + | } | |||
| 1177 | + | ||||
| 1178 | + | type lifecycleControlledCatalogue struct { | |||
| 1179 | + | ports.Catalogue | |||
| 1180 | + | entered chan struct{} | |||
| 1181 | + | proceed chan struct{} | |||
| 1182 | + | closed chan struct{} | |||
| 1183 | + | events chan<- string | |||
| 1184 | + | once sync.Once | |||
| 1185 | + | err error | |||
| 1186 | + | } | |||
| 1187 | + | ||||
| 1188 | + | func (c *lifecycleControlledCatalogue) Close() error { | |||
| 1189 | + | c.once.Do(func() { | |||
| 1190 | + | c.events <- "catalogue-close" | |||
| 1191 | + | close(c.entered) | |||
| 1192 | + | <-c.proceed | |||
| 1193 | + | c.err = c.Catalogue.Close() | |||
| 1194 | + | close(c.closed) | |||
| 1195 | + | }) | |||
| 1196 | + | return c.err | |||
| 1197 | + | } | |||
| 1198 | + | ||||
| 1199 | + | type integrationTransport struct{} | |||
| 1200 | + | ||||
| 1201 | + | func (*integrationTransport) Send(ports.Frame) error { return nil } | |||
| 1202 | + | func (*integrationTransport) Recv() (ports.Frame, error) { return ports.Frame{}, io.EOF } | |||
| 1203 | + | func (*integrationTransport) Close() error { return nil } | |||
| 1204 | + | func (*integrationTransport) LocalAddr() net.Addr { return nil } | |||
| 1205 | + | func (*integrationTransport) RemoteAddr() net.Addr { return nil } | |||
| 1206 | + | ||||
| 425 | func TestIntegration_KillAllShutsDownDaemon(t *testing.T) { | 1207 | func TestIntegration_KillAllShutsDownDaemon(t *testing.T) { | ||
| 426 | sz := domain.Size{Cols: 80, Rows: 24} | 1208 | sz := domain.Size{Cols: 80, Rows: 24} | ||
| 427 | dir, served := startDaemon(t, daemon.WithShell("/bin/sh", []string{"-c", "sleep 30"})) | 1209 | dir, served := startDaemon(t, daemon.WithShell("/bin/sh", []string{"-c", "sleep 30"})) | ||
M internal/app/offline_kill_retry_test.go +50 -57
| @@ -3,86 +3,79 @@ package app | |||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | "errors" | 5 | "errors" | ||
| 6 | + | "os" | |||
| 6 | "path/filepath" | 7 | "path/filepath" | ||
| 7 | "testing" | 8 | "testing" | ||
| 8 | "time" | 9 | "time" | ||
| 9 | 10 | ||||
| 10 | - | "github.com/bnema/vev/internal/persist" | |||
| 11 | "github.com/stretchr/testify/require" | 11 | "github.com/stretchr/testify/require" | ||
| 12 | - | ) | |||
| 13 | 12 | ||||
| 14 | - | type offlineSnapshotFake struct { | 13 | + | "github.com/bnema/vev/internal/domain" |
| 15 | - | calls []string | 14 | + | "github.com/bnema/vev/internal/persist" |
| 16 | - | deleteErr error | 15 | + | ) |
| 17 | - | deleteLegacyErr error | |||
| 18 | - | tombstoneErr error | |||
| 19 | - | deleteTombstoneErr error | |||
| 20 | - | } | |||
| 21 | 16 | ||||
| 22 | - | func (f *offlineSnapshotFake) Tombstone(context.Context, string) error { | 17 | + | func TestRunOfflineNamedKillUsesOpenCatalogueSeam(t *testing.T) { |
| 23 | - | f.calls = append(f.calls, "tombstone") | 18 | + | stateRoot := t.TempDir() |
| 24 | - | return f.tombstoneErr | 19 | + | t.Setenv("XDG_STATE_HOME", stateRoot) |
| 25 | - | } | 20 | + | stateDir := filepath.Join(stateRoot, "vev") |
| 26 | 21 | ||||
| 27 | - | func (f *offlineSnapshotFake) Delete(context.Context, string) error { | 22 | + | p := newTestPersister(t, stateDir) |
| 28 | - | f.calls = append(f.calls, "incremental") | 23 | + | now := time.Now().UnixNano() |
| 29 | - | return f.deleteErr | 24 | + | require.NoError(t, p.Save(persist.Record{Name: "named", IncarnationID: domain.IncarnationID{1}, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now})) |
| 30 | - | } | 25 | + | require.NoError(t, p.Close()) |
| 31 | 26 | ||||
| 32 | - | func (f *offlineSnapshotFake) DeleteLegacy(context.Context, string) error { | 27 | + | original := openCatalogue |
| 33 | - | f.calls = append(f.calls, "legacy") | 28 | + | t.Cleanup(func() { openCatalogue = original }) |
| 34 | - | return f.deleteLegacyErr | 29 | + | wantErr := errors.New("open seam called") |
| 35 | - | } | 30 | + | called := false |
| 31 | + | openCatalogue = func(gotStateDir string) (persist.OpenResult, error) { | |||
| 32 | + | called = true | |||
| 33 | + | require.Equal(t, stateDir, gotStateDir) | |||
| 34 | + | return persist.OpenResult{}, wantErr | |||
| 35 | + | } | |||
| 36 | 36 | ||||
| 37 | - | func (f *offlineSnapshotFake) DeleteTombstone(context.Context, string) error { | 37 | + | err := runOfflineNamedKill(context.Background(), "named") |
| 38 | - | f.calls = append(f.calls, "delete tombstone") | 38 | + | require.ErrorIs(t, err, wantErr) |
| 39 | - | return f.deleteTombstoneErr | 39 | + | require.True(t, called) |
| 40 | } | 40 | } | ||
| 41 | 41 | ||||
| 42 | - | func TestRunKillOfflineRetriesSnapshotDeletion(t *testing.T) { | 42 | + | func TestRunKillOfflineUsesIncarnationDeletionProtocol(t *testing.T) { |
| 43 | - | for _, tt := range []struct { | 43 | + | tests := []struct { |
| 44 | - | name string | 44 | + | name string |
| 45 | - | incrementalErr error | 45 | + | crashHook func(*testing.T, string) |
| 46 | - | legacyErr error | |||
| 47 | }{ | 46 | }{ | ||
| 48 | - | {name: "legacy failure", legacyErr: errors.New("legacy unlink failed")}, | 47 | + | {name: "normal"}, |
| 49 | - | {name: "incremental failure", incrementalErr: errors.New("incremental quarantine failed")}, | 48 | + | { |
| 50 | - | {name: "both failures", incrementalErr: errors.New("incremental quarantine failed"), legacyErr: errors.New("legacy unlink failed")}, | 49 | + | name: "stray tmp is ignored", |
| 51 | - | } { | 50 | + | crashHook: func(t *testing.T, stateDir string) { |
| 51 | + | t.Helper() | |||
| 52 | + | storePath := persist.StorePath(stateDir) | |||
| 53 | + | require.NoError(t, os.WriteFile(storePath+".tmp", []byte("partial rewrite"), 0o600)) | |||
| 54 | + | }, | |||
| 55 | + | }, | |||
| 56 | + | } | |||
| 57 | + | ||||
| 58 | + | for _, tt := range tests { | |||
| 52 | t.Run(tt.name, func(t *testing.T) { | 59 | t.Run(tt.name, func(t *testing.T) { | ||
| 53 | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | 60 | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | ||
| 54 | t.Setenv("XDG_STATE_HOME", stateRoot) | 61 | t.Setenv("XDG_STATE_HOME", stateRoot) | ||
| 55 | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | 62 | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | ||
| 63 | + | stateDir := filepath.Join(stateRoot, "vev") | |||
| 56 | 64 | ||||
| 57 | - | p, err := persist.Open(filepath.Join(stateRoot, "vev")) | 65 | + | id := domain.IncarnationID{1} |
| 58 | - | require.NoError(t, err) | 66 | + | p := newTestPersister(t, stateDir) |
| 59 | now := time.Now().UnixNano() | 67 | now := time.Now().UnixNano() | ||
| 60 | - | require.NoError(t, p.Save(persist.Record{Name: "named", Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now})) | 68 | + | require.NoError(t, p.Save(persist.Record{Name: "named", IncarnationID: id, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now})) |
| 61 | require.NoError(t, p.Close()) | 69 | require.NoError(t, p.Close()) | ||
| 62 | - | 70 | + | source := filepath.Join(stateDir, "snapshots", "sessions", id.String()) | |
| 63 | - | fake := &offlineSnapshotFake{deleteErr: tt.incrementalErr, deleteLegacyErr: tt.legacyErr} | 71 | + | require.NoError(t, os.MkdirAll(source, 0o700)) |
| 64 | - | originalFactory := newOfflineSnapshotRepository | 72 | + | require.NoError(t, os.WriteFile(filepath.Join(source, "sentinel"), []byte("durable"), 0o600)) |
| 65 | - | newOfflineSnapshotRepository = func(string) offlineSnapshotSource { return fake } | 73 | + | if tt.crashHook != nil { |
| 66 | - | t.Cleanup(func() { newOfflineSnapshotRepository = originalFactory }) | 74 | + | tt.crashHook(t, stateDir) |
| 67 | - | ||||
| 68 | - | err = runKill(context.Background(), "named", false, false) | |||
| 69 | - | if tt.incrementalErr != nil { | |||
| 70 | - | require.ErrorIs(t, err, tt.incrementalErr) | |||
| 71 | - | } | |||
| 72 | - | if tt.legacyErr != nil { | |||
| 73 | - | require.ErrorIs(t, err, tt.legacyErr) | |||
| 74 | } | 75 | } | ||
| 75 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy"}, fake.calls) | |||
| 76 | - | records, err := persist.LoadReadOnly(filepath.Join(stateRoot, "vev")) | |||
| 77 | - | require.NoError(t, err) | |||
| 78 | - | require.Len(t, records, 1, "a failed source delete must retain retry identity") | |||
| 79 | 76 | ||||
| 80 | - | fake.deleteErr, fake.deleteLegacyErr = nil, nil | |||
| 81 | require.NoError(t, runKill(context.Background(), "named", false, false)) | 77 | require.NoError(t, runKill(context.Background(), "named", false, false)) | ||
| 82 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy", "tombstone", "incremental", "legacy", "delete tombstone"}, fake.calls) | 78 | + | require.NoDirExists(t, source) |
| 83 | - | records, err = persist.LoadReadOnly(filepath.Join(stateRoot, "vev")) | |||
| 84 | - | require.NoError(t, err) | |||
| 85 | - | require.Empty(t, records) | |||
| 86 | }) | 79 | }) | ||
| 87 | } | 80 | } | ||
| 88 | } | 81 | } | ||
M internal/app/run.go +254 -143
| @@ -32,6 +32,7 @@ import ( | |||||
| 32 | "github.com/bnema/vev/internal/adapters/config" | 32 | "github.com/bnema/vev/internal/adapters/config" | ||
| 33 | "github.com/bnema/vev/internal/adapters/dgram" | 33 | "github.com/bnema/vev/internal/adapters/dgram" | ||
| 34 | "github.com/bnema/vev/internal/adapters/ipc" | 34 | "github.com/bnema/vev/internal/adapters/ipc" | ||
| 35 | + | "github.com/bnema/vev/internal/adapters/lifecycle" | |||
| 35 | "github.com/bnema/vev/internal/adapters/noticefile" | 36 | "github.com/bnema/vev/internal/adapters/noticefile" | ||
| 36 | "github.com/bnema/vev/internal/adapters/observability" | 37 | "github.com/bnema/vev/internal/adapters/observability" | ||
| 37 | "github.com/bnema/vev/internal/adapters/pty" | 38 | "github.com/bnema/vev/internal/adapters/pty" | ||
| @@ -48,8 +49,9 @@ import ( | |||||
| 48 | "github.com/bnema/vev/internal/usecase/client" | 49 | "github.com/bnema/vev/internal/usecase/client" | ||
| 49 | "github.com/bnema/vev/internal/usecase/confirm" | 50 | "github.com/bnema/vev/internal/usecase/confirm" | ||
| 50 | "github.com/bnema/vev/internal/usecase/daemon" | 51 | "github.com/bnema/vev/internal/usecase/daemon" | ||
| 52 | + | "github.com/bnema/vev/internal/usecase/recovery" | |||
| 51 | pdgram "github.com/bnema/vev/pkg/dgram" | 53 | pdgram "github.com/bnema/vev/pkg/dgram" | ||
| 52 | - | "github.com/bnema/vev/pkg/kv" | 54 | + | "github.com/bnema/vev/pkg/safedir" |
| 53 | ) | 55 | ) | ||
| 54 | 56 | ||||
| 55 | // cmdKind identifies which sub-command the CLI parsed. | 57 | // cmdKind identifies which sub-command the CLI parsed. | ||
| @@ -114,6 +116,11 @@ var ( | |||||
| 114 | version = "0.1.0-dev" | 116 | version = "0.1.0-dev" | ||
| 115 | commit = "none" | 117 | commit = "none" | ||
| 116 | date = "unknown" | 118 | date = "unknown" | ||
| 119 | + | ||||
| 120 | + | openCatalogue = persist.OpenOrCreate | |||
| 121 | + | listenDaemon = func(dir string, observer ports.SerializedRuntimeObserver) (ports.Listener, error) { | |||
| 122 | + | return ipc.Listen(dir, ipc.WithRuntimeObserver(observer)) | |||
| 123 | + | } | |||
| 117 | ) | 124 | ) | ||
| 118 | 125 | ||||
| 119 | // versionLine renders the --version output. | 126 | // versionLine renders the --version output. | ||
| @@ -410,17 +417,130 @@ func pprofAddrIsLoopback(addr string) bool { | |||||
| 410 | return ip != nil && ip.IsLoopback() | 417 | return ip != nil && ip.IsLoopback() | ||
| 411 | } | 418 | } | ||
| 412 | 419 | ||||
| 420 | + | type ownedDaemonStart func(context.Context) error | |||
| 421 | + | ||||
| 422 | + | type lifecycleOwnership interface { | |||
| 423 | + | Release() error | |||
| 424 | + | } | |||
| 425 | + | ||||
| 426 | + | func joinLifecycleReleaseError(retErr *error, owner lifecycleOwnership) { | |||
| 427 | + | *retErr = errors.Join(*retErr, owner.Release()) | |||
| 428 | + | } | |||
| 429 | + | ||||
| 430 | + | type lifecycleStartupDeps struct { | |||
| 431 | + | ensurePrivate func(string) error | |||
| 432 | + | acquire func(context.Context, string, time.Duration) (lifecycleOwnership, error) | |||
| 433 | + | log *slog.Logger | |||
| 434 | + | } | |||
| 435 | + | ||||
| 436 | + | const lifecycleAcquireRetry = 25 * time.Millisecond | |||
| 437 | + | ||||
| 438 | + | func lifecycleStartupDependencies(log *slog.Logger) lifecycleStartupDeps { | |||
| 439 | + | return lifecycleStartupDeps{ | |||
| 440 | + | ensurePrivate: safedir.EnsurePrivate, | |||
| 441 | + | acquire: func(ctx context.Context, runtimeDir string, retry time.Duration) (lifecycleOwnership, error) { | |||
| 442 | + | return lifecycle.Acquire(ctx, runtimeDir, retry) | |||
| 443 | + | }, | |||
| 444 | + | log: log, | |||
| 445 | + | } | |||
| 446 | + | } | |||
| 447 | + | ||||
| 448 | + | func runWithLifecycleOwner(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart) error { | |||
| 449 | + | return runWithLifecycleOwnerDeps(ctx, runtimeDir, stateDir, start, lifecycleStartupDependencies(nil)) | |||
| 450 | + | } | |||
| 451 | + | ||||
| 452 | + | func runWithLifecycleOwnerDeps(ctx context.Context, runtimeDir, stateDir string, start ownedDaemonStart, deps lifecycleStartupDeps) (retErr error) { | |||
| 453 | + | log := deps.log | |||
| 454 | + | if log == nil { | |||
| 455 | + | log = slog.Default() | |||
| 456 | + | } | |||
| 457 | + | if err := deps.ensurePrivate(runtimeDir); err != nil { | |||
| 458 | + | return fmt.Errorf("vev: secure runtime directory: %w", err) | |||
| 459 | + | } | |||
| 460 | + | if err := deps.ensurePrivate(stateDir); err != nil { | |||
| 461 | + | return fmt.Errorf("vev: secure state directory: %w", err) | |||
| 462 | + | } | |||
| 463 | + | lockPath := lifecycle.Path(runtimeDir) | |||
| 464 | + | log.Info("lifecycle_owner_wait", "path", lockPath) | |||
| 465 | + | owner, err := deps.acquire(ctx, runtimeDir, lifecycleAcquireRetry) | |||
| 466 | + | if err != nil { | |||
| 467 | + | log.Warn("lifecycle_owner_wait_failed", "path", lockPath, "reason_code", "acquire-failed") | |||
| 468 | + | return fmt.Errorf("vev: acquire lifecycle ownership: %w", err) | |||
| 469 | + | } | |||
| 470 | + | log.Info("lifecycle_owner_acquired", "path", lockPath) | |||
| 471 | + | defer func() { | |||
| 472 | + | releaseErr := owner.Release() | |||
| 473 | + | if releaseErr != nil { | |||
| 474 | + | log.Error("lifecycle_owner_release_failed", "path", lockPath, "reason_code", "release-failed") | |||
| 475 | + | } else { | |||
| 476 | + | log.Info("lifecycle_owner_released", "path", lockPath) | |||
| 477 | + | } | |||
| 478 | + | retErr = errors.Join(retErr, releaseErr) | |||
| 479 | + | }() | |||
| 480 | + | if start == nil { | |||
| 481 | + | return errors.New("vev: nil owned daemon startup") | |||
| 482 | + | } | |||
| 483 | + | return start(ctx) | |||
| 484 | + | } | |||
| 485 | + | ||||
| 413 | // runDaemon runs the daemon in the foreground (the hidden --daemon path, | 486 | // runDaemon runs the daemon in the foreground (the hidden --daemon path, | ||
| 414 | - | // entered by an auto-spawned child): it sets up logging, binds the socket, | 487 | + | // entered by an auto-spawned child) while holding exclusive lifecycle |
| 415 | - | // constructs the daemon, and serves until the last session exits or a | 488 | + | // ownership through complete teardown. |
| 416 | - | // termination signal arrives (graceful shutdown notifies attached clients). | |||
| 417 | func runDaemon() (retErr error) { | 489 | func runDaemon() (retErr error) { | ||
| 490 | + | ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) | |||
| 491 | + | defer stop() | |||
| 418 | log, logCloser, err := configureLogging(logging.Daemon, true) | 492 | log, logCloser, err := configureLogging(logging.Daemon, true) | ||
| 419 | if err != nil { | 493 | if err != nil { | ||
| 420 | return err | 494 | return err | ||
| 421 | } | 495 | } | ||
| 422 | - | defer func() { _ = logCloser.Close() }() | 496 | + | defer func() { retErr = errors.Join(retErr, logCloser.Close()) }() |
| 497 | + | return runWithLifecycleOwnerDeps(ctx, ipc.SocketDir(), platform.StateDir(), func(ctx context.Context) error { | |||
| 498 | + | return runDaemonOwnedWithLogger(ctx, log) | |||
| 499 | + | }, lifecycleStartupDependencies(log)) | |||
| 500 | + | } | |||
| 501 | + | ||||
| 502 | + | func logCatalogueRecovery(log *slog.Logger, records []domain.CatalogueRecord, recoveryMode string) { | |||
| 503 | + | log.Info("catalogue_validated", "records", len(records), "recovery", recoveryMode) | |||
| 504 | + | } | |||
| 505 | + | ||||
| 506 | + | func logStartupRecoveryCounts(log *slog.Logger, records []domain.CatalogueRecord, restoring int) { | |||
| 507 | + | healthy, fresh, broken := 0, 0, 0 | |||
| 508 | + | for _, record := range records { | |||
| 509 | + | switch { | |||
| 510 | + | case record.DegradedReason != "": | |||
| 511 | + | broken++ | |||
| 512 | + | case record.Committed == nil: | |||
| 513 | + | fresh++ | |||
| 514 | + | default: | |||
| 515 | + | healthy++ | |||
| 516 | + | } | |||
| 517 | + | } | |||
| 518 | + | log.Info("daemon_startup_complete", "healthy", healthy, "fresh", fresh, "restoring", restoring, "broken", broken) | |||
| 519 | + | } | |||
| 520 | + | ||||
| 521 | + | func constructDaemonBeforeSocketPublication( | |||
| 522 | + | construct func() *daemon.Daemon, | |||
| 523 | + | prepare func(*daemon.Daemon) error, | |||
| 524 | + | listen func() (ports.Listener, error), | |||
| 525 | + | ) (*daemon.Daemon, ports.Listener, error) { | |||
| 526 | + | d := construct() | |||
| 527 | + | if err := prepare(d); err != nil { | |||
| 528 | + | return d, nil, err | |||
| 529 | + | } | |||
| 530 | + | ln, err := listen() | |||
| 531 | + | return d, ln, err | |||
| 532 | + | } | |||
| 423 | 533 | ||||
| 534 | + | func runDaemonOwned(ctx context.Context) (retErr error) { | |||
| 535 | + | log, logCloser, err := configureLogging(logging.Daemon, true) | |||
| 536 | + | if err != nil { | |||
| 537 | + | return err | |||
| 538 | + | } | |||
| 539 | + | defer func() { retErr = errors.Join(retErr, logCloser.Close()) }() | |||
| 540 | + | return runDaemonOwnedWithLogger(ctx, log) | |||
| 541 | + | } | |||
| 542 | + | ||||
| 543 | + | func runDaemonOwnedWithLogger(ctx context.Context, log *slog.Logger) (retErr error) { | |||
| 424 | clk := clock.New() | 544 | clk := clock.New() | ||
| 425 | observer, observerCloser, err := newPerformanceTrace(clk) | 545 | observer, observerCloser, err := newPerformanceTrace(clk) | ||
| 426 | if err != nil { | 546 | if err != nil { | ||
| @@ -429,17 +549,6 @@ func runDaemon() (retErr error) { | |||||
| 429 | if observerCloser != nil { | 549 | if observerCloser != nil { | ||
| 430 | defer func() { retErr = errors.Join(retErr, observerCloser.Close()) }() | 550 | defer func() { retErr = errors.Join(retErr, observerCloser.Close()) }() | ||
| 431 | } | 551 | } | ||
| 432 | - | // Every accepted local connection shares this process's serialized sink. | |||
| 433 | - | ln, err := ipc.Listen(ipc.SocketDir(), ipc.WithRuntimeObserver(observer)) | |||
| 434 | - | if err != nil { | |||
| 435 | - | log.Error("daemon listen failed", "socket_dir", ipc.SocketDir(), "err", err) | |||
| 436 | - | return fmt.Errorf("vev: daemon listen: %w", err) | |||
| 437 | - | } | |||
| 438 | - | defer func() { _ = ln.Close() }() | |||
| 439 | - | ||||
| 440 | - | ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) | |||
| 441 | - | defer stop() | |||
| 442 | - | ||||
| 443 | if addr := os.Getenv("VEV_PPROF_ADDR"); addr != "" { | 552 | if addr := os.Getenv("VEV_PPROF_ADDR"); addr != "" { | ||
| 444 | if !pprofAddrIsLoopback(addr) { | 553 | if !pprofAddrIsLoopback(addr) { | ||
| 445 | log.Warn("pprof bound to non-loopback address; /debug/pprof is unauthenticated", "addr", addr) | 554 | log.Warn("pprof bound to non-loopback address; /debug/pprof is unauthenticated", "addr", addr) | ||
| @@ -452,7 +561,6 @@ func runDaemon() (retErr error) { | |||||
| 452 | log.Info("pprof enabled", "addr", addr) | 561 | log.Info("pprof enabled", "addr", addr) | ||
| 453 | } | 562 | } | ||
| 454 | 563 | ||||
| 455 | - | log.Info("daemon starting", "socket", ln.Addr()) | |||
| 456 | daemonOpts := []daemon.Option(nil) | 564 | daemonOpts := []daemon.Option(nil) | ||
| 457 | if observer != nil { | 565 | if observer != nil { | ||
| 458 | daemonOpts = append(daemonOpts, daemon.WithRuntimeObserver(observer)) | 566 | daemonOpts = append(daemonOpts, daemon.WithRuntimeObserver(observer)) | ||
| @@ -467,23 +575,54 @@ func runDaemon() (retErr error) { | |||||
| 467 | daemonOpts = append(daemonOpts, daemon.WithConfig(cfg)) | 575 | daemonOpts = append(daemonOpts, daemon.WithConfig(cfg)) | ||
| 468 | daemonOpts = append(daemonOpts, daemon.WithBarScriptCommandRunner(shellcmd.New())) | 576 | daemonOpts = append(daemonOpts, daemon.WithBarScriptCommandRunner(shellcmd.New())) | ||
| 469 | daemonOpts = append(daemonOpts, daemon.WithProcessInspector(platform.NewProcessInspector()), daemon.WithDirOrHome(platform.DirOrHome)) | 577 | daemonOpts = append(daemonOpts, daemon.WithProcessInspector(platform.NewProcessInspector()), daemon.WithDirOrHome(platform.DirOrHome)) | ||
| 470 | - | snapshotRepository := snapshotadapter.NewRepository(snapshotDir()) | 578 | + | snapshotRepository := snapshotadapter.NewRepositoryWithLogger(snapshotDir(), log) |
| 471 | - | daemonOpts = append(daemonOpts, daemon.WithSnapshotRepository(snapshotRepository, snapshotRepository)) | 579 | + | daemonOpts = append(daemonOpts, daemon.WithSnapshotRepository(snapshotRepository)) |
| 472 | - | daemonOpts = append(daemonOpts, daemon.WithNoticeStore(noticefile.New(platform.StateDir()))) | 580 | + | noticeStore := noticefile.New(platform.StateDir()) |
| 473 | - | storePath := persist.StorePath(platform.StateDir()) | 581 | + | daemonOpts = append(daemonOpts, daemon.WithNoticeStore(noticeStore)) |
| 474 | - | var storeErr error | 582 | + | stateDir := platform.StateDir() |
| 475 | - | if store, err := kv.Open(storePath); err != nil { | 583 | + | storePath := persist.StorePath(stateDir) |
| 476 | - | log.Warn("opening session store failed; persistence disabled", "path", storePath, "err", err) | 584 | + | opened, err := openCatalogue(stateDir) |
| 477 | - | storeErr = err | 585 | + | if err != nil { |
| 478 | - | } else { | 586 | + | log.Error("catalogue_validation_failed", "path", storePath, "reason_code", "open-failed") |
| 479 | - | log.Info("session persistence enabled", "path", storePath) | 587 | + | if errors.Is(err, persist.ErrCatalogueUnreadable) { |
| 480 | - | daemonOpts = append(daemonOpts, daemon.WithStore(store)) | 588 | + | return unreadableCatalogueError(stateDir) |
| 481 | - | } | 589 | + | } |
| 482 | - | d := daemon.New(pty.NewFactory(), clk, log, daemonOpts...) | 590 | + | return fmt.Errorf("vev: open durable session state %s: %w", storePath, err) |
| 483 | - | if storeErr != nil { | 591 | + | } |
| 484 | - | d.NotifyGlobal(domain.NoticeWarn, domain.NoticePersistDisabled, | 592 | + | recoveryMode := "current" |
| 485 | - | "session persistence is disabled; sessions will not survive daemon restarts", storeErr) | 593 | + | if opened.NewInstall { |
| 594 | + | recoveryMode = "new-install" | |||
| 595 | + | } | |||
| 596 | + | logCatalogueRecovery(log, opened.Records, recoveryMode) | |||
| 597 | + | coordinator := recovery.NewCoordinator(opened.Catalogue, snapshotRepository, rand.Reader) | |||
| 598 | + | daemonOpts = append(daemonOpts, daemon.WithRecoveryCoordinator(coordinator)) | |||
| 599 | + | daemonOpts = append(daemonOpts, daemon.WithDurableMaintenance(opened.Catalogue, snapshotRepository)) | |||
| 600 | + | log.Info("session persistence enabled", "path", storePath) | |||
| 601 | + | daemonOpts = append(daemonOpts, daemon.WithCatalogue(opened.Catalogue, opened.Records)) | |||
| 602 | + | ||||
| 603 | + | // Construct the catalogue-backed expected-session registry before socket | |||
| 604 | + | // publication. Phase 3 snapshot restoration remains asynchronous in Serve. | |||
| 605 | + | d, ln, err := constructDaemonBeforeSocketPublication( | |||
| 606 | + | func() *daemon.Daemon { return daemon.New(pty.NewFactory(), clk, log, daemonOpts...) }, | |||
| 607 | + | func(d *daemon.Daemon) error { | |||
| 608 | + | if err := d.CollectStartupGarbage(ctx); err != nil { | |||
| 609 | + | // GC is best-effort, but it is fully finished before socket | |||
| 610 | + | // publication. A failed pass leaves durable state untouched and | |||
| 611 | + | // restoration retains its per-session failure isolation. | |||
| 612 | + | log.Warn("snapshot_garbage_collection_failed", "err", err) | |||
| 613 | + | } | |||
| 614 | + | return nil | |||
| 615 | + | }, | |||
| 616 | + | func() (ports.Listener, error) { return listenDaemon(ipc.SocketDir(), observer) }, | |||
| 617 | + | ) | |||
| 618 | + | if err != nil { | |||
| 619 | + | closeErr := opened.Catalogue.Close() | |||
| 620 | + | log.Error("daemon startup preparation failed", "socket_dir", ipc.SocketDir(), "err", err) | |||
| 621 | + | return errors.Join(fmt.Errorf("vev: prepare daemon startup: %w", err), closeErr) | |||
| 486 | } | 622 | } | ||
| 623 | + | defer func() { _ = ln.Close() }() | |||
| 624 | + | log.Info("daemon starting", "socket", ln.Addr()) | |||
| 625 | + | ||||
| 487 | watchCtx, stopWatch := context.WithCancel(ctx) | 626 | watchCtx, stopWatch := context.WithCancel(ctx) | ||
| 488 | defer stopWatch() | 627 | defer stopWatch() | ||
| 489 | go func() { | 628 | go func() { | ||
| @@ -695,7 +834,7 @@ func (d localDaemonDialer) Dial(ctx context.Context) (ports.Transport, error) { | |||||
| 695 | return ipc.DialContext(ctx, dir, ipc.WithRuntimeObserver(d.observer)) | 834 | return ipc.DialContext(ctx, dir, ipc.WithRuntimeObserver(d.observer)) | ||
| 696 | } | 835 | } | ||
| 697 | } | 836 | } | ||
| 698 | - | return ensureDaemon(ctx, d.dir, dial, realSpawn, defaultBackoff) | 837 | + | return ensureDaemonWithLifecycle(ctx, d.dir, dial, realSpawn, defaultBackoff) |
| 699 | } | 838 | } | ||
| 700 | 839 | ||||
| 701 | func detachedLocalHello(name, cwd string) ports.Hello { | 840 | func detachedLocalHello(name, cwd string) ports.Hello { | ||
| @@ -713,7 +852,7 @@ func detachedLocalHello(name, cwd string) ports.Hello { | |||||
| 713 | } | 852 | } | ||
| 714 | 853 | ||||
| 715 | func createDetachedLocalSession(ctx context.Context, name string) error { | 854 | func createDetachedLocalSession(ctx context.Context, name string) error { | ||
| 716 | - | transport, err := ensureDaemon(ctx, ipc.SocketDir(), realDial, realSpawn, defaultBackoff) | 855 | + | transport, err := ensureDaemonWithLifecycle(ctx, ipc.SocketDir(), realDial, realSpawn, defaultBackoff) |
| 717 | if err != nil { | 856 | if err != nil { | ||
| 718 | return err | 857 | return err | ||
| 719 | } | 858 | } | ||
| @@ -776,40 +915,26 @@ func requestDaemonStop(ctx context.Context) error { | |||||
| 776 | ctx, cancel := context.WithTimeout(ctx, daemonStopTimeout) | 915 | ctx, cancel := context.WithTimeout(ctx, daemonStopTimeout) | ||
| 777 | defer cancel() | 916 | defer cancel() | ||
| 778 | 917 | ||||
| 779 | - | done := make(chan error, 1) | 918 | + | transport, owner, err := waitForDaemonOrLifecycle(ctx, ipc.SocketDir(), realDial, defaultBackoff) |
| 780 | - | go func() { | 919 | + | if err != nil { |
| 781 | - | transport, err := realDial(ctx, ipc.SocketDir()) | 920 | + | return fmt.Errorf("vev: stopping daemon: %w", err) |
| 782 | - | if err != nil { | 921 | + | } |
| 783 | - | done <- fmt.Errorf("vev: no daemon running") | 922 | + | if owner != nil { |
| 784 | - | return | 923 | + | return errors.Join(errors.New("vev: no daemon running"), owner.Release()) |
| 785 | - | } | 924 | + | } |
| 786 | - | defer func() { _ = transport.Close() }() | 925 | + | defer func() { _ = transport.Close() }() |
| 787 | - | closed := make(chan struct{}) | 926 | + | if err := transport.Send(ports.Frame{Type: ports.MsgKill, Payload: ports.MarshalKill(ports.Kill{All: true})}); err != nil { |
| 788 | - | go func() { | 927 | + | return fmt.Errorf("vev: requesting daemon stop: %w", err) |
| 789 | - | select { | 928 | + | } |
| 790 | - | case <-ctx.Done(): | 929 | + | if _, err := transport.Recv(); err != nil && !errors.Is(err, io.EOF) { |
| 791 | - | _ = transport.Close() | 930 | + | return fmt.Errorf("vev: reading daemon stop reply: %w", err) |
| 792 | - | case <-closed: | 931 | + | } |
| 793 | - | } | |||
| 794 | - | }() | |||
| 795 | - | defer close(closed) | |||
| 796 | - | if err := transport.Send(ports.Frame{Type: ports.MsgKill, Payload: ports.MarshalKill(ports.Kill{All: true})}); err != nil { | |||
| 797 | - | done <- fmt.Errorf("vev: requesting daemon stop: %w", err) | |||
| 798 | - | return | |||
| 799 | - | } | |||
| 800 | - | if _, err := transport.Recv(); err != nil && !errors.Is(err, io.EOF) { | |||
| 801 | - | done <- fmt.Errorf("vev: reading daemon stop reply: %w", err) | |||
| 802 | - | return | |||
| 803 | - | } | |||
| 804 | - | done <- nil | |||
| 805 | - | }() | |||
| 806 | 932 | ||||
| 807 | - | select { | 933 | + | owner, err = waitForLifecycleAvailability(ctx, ipc.SocketDir(), defaultBackoff) |
| 808 | - | case err := <-done: | 934 | + | if err != nil { |
| 809 | - | return err | 935 | + | return fmt.Errorf("vev: waiting for daemon ownership transfer: %w", err) |
| 810 | - | case <-ctx.Done(): | |||
| 811 | - | return fmt.Errorf("vev: stopping daemon: %w", ctx.Err()) | |||
| 812 | } | 936 | } | ||
| 937 | + | return owner.Release() | |||
| 813 | } | 938 | } | ||
| 814 | 939 | ||||
| 815 | // runStdio is the hidden remote-side mode used by `ssh host vev _stdio`: it | 940 | // runStdio is the hidden remote-side mode used by `ssh host vev _stdio`: it | ||
| @@ -831,7 +956,7 @@ func runStdio(ctx context.Context) (retErr error) { | |||||
| 831 | if observerCloser != nil { | 956 | if observerCloser != nil { | ||
| 832 | defer func() { retErr = errors.Join(retErr, observerCloser.Close()) }() | 957 | defer func() { retErr = errors.Join(retErr, observerCloser.Close()) }() | ||
| 833 | } | 958 | } | ||
| 834 | - | transport, err := ensureDaemon(ctx, ipc.SocketDir(), func(ctx context.Context, dir string) (ports.Transport, error) { | 959 | + | transport, err := ensureDaemonWithLifecycle(ctx, ipc.SocketDir(), func(ctx context.Context, dir string) (ports.Transport, error) { |
| 835 | return ipc.DialContext(ctx, dir, ipc.WithRuntimeObserver(observer)) | 960 | return ipc.DialContext(ctx, dir, ipc.WithRuntimeObserver(observer)) | ||
| 836 | }, realSpawn, defaultBackoff) | 961 | }, realSpawn, defaultBackoff) | ||
| 837 | if err != nil { | 962 | if err != nil { | ||
| @@ -946,7 +1071,7 @@ func runUDPProxy(ctx context.Context, session string, ready io.Writer) (retErr e | |||||
| 946 | return err | 1071 | return err | ||
| 947 | } | 1072 | } | ||
| 948 | defer func() { _ = conn.Close() }() | 1073 | defer func() { _ = conn.Close() }() | ||
| 949 | - | daemonTr, err := ensureDaemon(ctx, ipc.SocketDir(), func(ctx context.Context, dir string) (ports.Transport, error) { | 1074 | + | daemonTr, err := ensureDaemonWithLifecycle(ctx, ipc.SocketDir(), func(ctx context.Context, dir string) (ports.Transport, error) { |
| 950 | return ipc.DialContext(ctx, dir, ipc.WithRuntimeObserver(observer)) | 1075 | return ipc.DialContext(ctx, dir, ipc.WithRuntimeObserver(observer)) | ||
| 951 | }, realSpawn, defaultBackoff) | 1076 | }, realSpawn, defaultBackoff) | ||
| 952 | if err != nil { | 1077 | if err != nil { | ||
| @@ -1163,16 +1288,27 @@ func proxyTransports(ctx context.Context, a, b ports.Transport, log *slog.Logger | |||||
| 1163 | 1288 | ||||
| 1164 | // runList prints the daemon's session listing. With no daemon running, it | 1289 | // runList prints the daemon's session listing. With no daemon running, it | ||
| 1165 | // falls back to the persisted stopped-session records. | 1290 | // falls back to the persisted stopped-session records. | ||
| 1166 | - | func runList(ctx context.Context) error { | 1291 | + | func runList(ctx context.Context) (retErr error) { |
| 1167 | - | transport, err := realDial(ctx, ipc.SocketDir()) | 1292 | + | transport, owner, err := waitForDaemonOrLifecycle(ctx, ipc.SocketDir(), realDial, defaultBackoff) |
| 1168 | if err != nil { | 1293 | if err != nil { | ||
| 1294 | + | return fmt.Errorf("vev: waiting for durable session state: %w", err) | |||
| 1295 | + | } | |||
| 1296 | + | if owner != nil { | |||
| 1297 | + | defer joinLifecycleReleaseError(&retErr, owner) | |||
| 1169 | records, loadErr := persist.LoadReadOnly(platform.StateDir()) | 1298 | records, loadErr := persist.LoadReadOnly(platform.StateDir()) | ||
| 1170 | if loadErr != nil { | 1299 | if loadErr != nil { | ||
| 1300 | + | if errors.Is(loadErr, persist.ErrCatalogueUnreadable) { | |||
| 1301 | + | return unreadableCatalogueError(platform.StateDir()) | |||
| 1302 | + | } | |||
| 1171 | return fmt.Errorf("vev: reading stored sessions: %w", loadErr) | 1303 | return fmt.Errorf("vev: reading stored sessions: %w", loadErr) | ||
| 1172 | } | 1304 | } | ||
| 1173 | infos := make([]ports.SessionInfo, 0, len(records)) | 1305 | infos := make([]ports.SessionInfo, 0, len(records)) | ||
| 1174 | for _, r := range records { | 1306 | for _, r := range records { | ||
| 1175 | - | infos = append(infos, ports.SessionInfo{Name: r.Name, Stopped: true}) | 1307 | + | state := ports.SessionStopped |
| 1308 | + | if r.DegradedReason != "" { | |||
| 1309 | + | state = ports.SessionBroken | |||
| 1310 | + | } | |||
| 1311 | + | infos = append(infos, ports.SessionInfo{Name: r.Name, State: state}) | |||
| 1176 | } | 1312 | } | ||
| 1177 | printSessions(os.Stdout, infos) | 1313 | printSessions(os.Stdout, infos) | ||
| 1178 | return nil | 1314 | return nil | ||
| @@ -1213,11 +1349,17 @@ func printSessions(w io.Writer, sessions []ports.SessionInfo) { | |||||
| 1213 | state := "running" | 1349 | state := "running" | ||
| 1214 | tabs := fmt.Sprintf("%d", s.Tabs) | 1350 | tabs := fmt.Sprintf("%d", s.Tabs) | ||
| 1215 | attached := "no" | 1351 | attached := "no" | ||
| 1216 | - | if s.Stopped { | 1352 | + | switch s.State { |
| 1353 | + | case ports.SessionStopped: | |||
| 1217 | state = "stopped" | 1354 | state = "stopped" | ||
| 1218 | tabs = "-" | 1355 | tabs = "-" | ||
| 1219 | - | } else if s.Ephemeral { | 1356 | + | case ports.SessionBroken: |
| 1220 | - | state = "temporary" | 1357 | + | state = "broken" |
| 1358 | + | tabs = "-" | |||
| 1359 | + | default: | |||
| 1360 | + | if s.Ephemeral { | |||
| 1361 | + | state = "temporary" | |||
| 1362 | + | } | |||
| 1221 | } | 1363 | } | ||
| 1222 | if s.Attached { | 1364 | if s.Attached { | ||
| 1223 | attached = "yes" | 1365 | attached = "yes" | ||
| @@ -1227,83 +1369,52 @@ func printSessions(w io.Writer, sessions []ports.SessionInfo) { | |||||
| 1227 | _ = tw.Flush() | 1369 | _ = tw.Flush() | ||
| 1228 | } | 1370 | } | ||
| 1229 | 1371 | ||||
| 1230 | - | // offlineSnapshotSource owns the durable source and tombstone transitions for | 1372 | + | func unreadableCatalogueError(stateDir string) error { |
| 1231 | - | // a daemon-less kill. It is intentionally local to the composition root: the | 1373 | + | return fmt.Errorf("%w: vev: durable session state at %s cannot be read and was left untouched.\n"+ |
| 1232 | - | // daemon uses the repository through its narrower ports. | 1374 | + | "vev does not erase it automatically. To start fresh, remove it:\n"+ |
| 1233 | - | type offlineSnapshotSource interface { | 1375 | + | " rm -rf %s", persist.ErrCatalogueUnreadable, stateDir, stateDir) |
| 1234 | - | Tombstone(context.Context, string) error | |||
| 1235 | - | Delete(context.Context, string) error | |||
| 1236 | - | DeleteLegacy(context.Context, string) error | |||
| 1237 | - | DeleteTombstone(context.Context, string) error | |||
| 1238 | } | 1376 | } | ||
| 1239 | 1377 | ||||
| 1240 | - | var newOfflineSnapshotRepository = func(dir string) offlineSnapshotSource { | 1378 | + | func runOfflineNamedKill(ctx context.Context, name string) (retErr error) { |
| 1241 | - | return snapshotadapter.NewRepository(dir) | 1379 | + | stateDir := platform.StateDir() |
| 1242 | - | } | 1380 | + | if _, err := os.Stat(persist.StorePath(stateDir)); errors.Is(err, os.ErrNotExist) { |
| 1381 | + | return fmt.Errorf("vev: no such session: %s", name) | |||
| 1382 | + | } else if err != nil { | |||
| 1383 | + | return fmt.Errorf("vev: reading stored sessions: %w", err) | |||
| 1384 | + | } | |||
| 1385 | + | repository := snapshotadapter.NewRepository(snapshotDir()) | |||
| 1386 | + | opened, err := openCatalogue(stateDir) | |||
| 1387 | + | if err != nil { | |||
| 1388 | + | if errors.Is(err, persist.ErrCatalogueUnreadable) { | |||
| 1389 | + | return unreadableCatalogueError(stateDir) | |||
| 1390 | + | } | |||
| 1391 | + | return fmt.Errorf("vev: opening stored sessions: %w", err) | |||
| 1392 | + | } | |||
| 1393 | + | defer func() { retErr = errors.Join(retErr, opened.Catalogue.Close()) }() | |||
| 1243 | 1394 | ||||
| 1244 | - | func phaseError(phase string, err error) error { | 1395 | + | if _, ok, err := opened.Catalogue.Record(name); err != nil { |
| 1245 | - | if err == nil { | 1396 | + | return fmt.Errorf("vev: reading stored session: %w", err) |
| 1246 | - | return nil | 1397 | + | } else if !ok { |
| 1398 | + | return fmt.Errorf("vev: no such session: %s", name) | |||
| 1399 | + | } | |||
| 1400 | + | coordinator := recovery.NewCoordinator(opened.Catalogue, repository, rand.Reader) | |||
| 1401 | + | if err := coordinator.Delete(ctx, name); err != nil { | |||
| 1402 | + | return fmt.Errorf("vev: deleting stored session: %w", err) | |||
| 1247 | } | 1403 | } | ||
| 1248 | - | return fmt.Errorf("vev: %s: %w", phase, err) | 1404 | + | return nil |
| 1249 | } | 1405 | } | ||
| 1250 | 1406 | ||||
| 1251 | // runKill asks the daemon to terminate a named session, every session, or the daemon. | 1407 | // runKill asks the daemon to terminate a named session, every session, or the daemon. | ||
| 1252 | - | func runKill(ctx context.Context, name string, all, daemon bool) error { | 1408 | + | func runKill(ctx context.Context, name string, all, daemon bool) (retErr error) { |
| 1253 | - | transport, err := realDial(ctx, ipc.SocketDir()) | 1409 | + | transport, owner, err := waitForDaemonOrLifecycle(ctx, ipc.SocketDir(), realDial, defaultBackoff) |
| 1254 | if err != nil { | 1410 | if err != nil { | ||
| 1411 | + | return fmt.Errorf("vev: waiting for durable session state: %w", err) | |||
| 1412 | + | } | |||
| 1413 | + | if owner != nil { | |||
| 1414 | + | defer joinLifecycleReleaseError(&retErr, owner) | |||
| 1255 | if name != "" && !all && !daemon { | 1415 | if name != "" && !all && !daemon { | ||
| 1256 | - | storePath := persist.StorePath(platform.StateDir()) | 1416 | + | if err := runOfflineNamedKill(ctx, name); err != nil { |
| 1257 | - | if _, statErr := os.Stat(storePath); errors.Is(statErr, os.ErrNotExist) { | 1417 | + | return err |
| 1258 | - | return fmt.Errorf("vev: no such session: %s", name) | |||
| 1259 | - | } else if statErr != nil { | |||
| 1260 | - | return fmt.Errorf("vev: reading stored sessions: %w", statErr) | |||
| 1261 | - | } | |||
| 1262 | - | records, loadErr := persist.LoadReadOnly(platform.StateDir()) | |||
| 1263 | - | if loadErr != nil { | |||
| 1264 | - | return fmt.Errorf("vev: reading stored sessions: %w", loadErr) | |||
| 1265 | - | } | |||
| 1266 | - | found := false | |||
| 1267 | - | for _, record := range records { | |||
| 1268 | - | if record.Name == name { | |||
| 1269 | - | found = true | |||
| 1270 | - | break | |||
| 1271 | - | } | |||
| 1272 | - | } | |||
| 1273 | - | if !found { | |||
| 1274 | - | return fmt.Errorf("vev: no such session: %s", name) | |||
| 1275 | - | } | |||
| 1276 | - | p, openErr := persist.Open(platform.StateDir()) | |||
| 1277 | - | if openErr != nil { | |||
| 1278 | - | return fmt.Errorf("vev: opening stored sessions: %w", openErr) | |||
| 1279 | - | } | |||
| 1280 | - | snapshots := newOfflineSnapshotRepository(snapshotDir()) | |||
| 1281 | - | // The durable marker is the commit point for a logical kill. Keep it and | |||
| 1282 | - | // the persisted record until both independently recoverable sources have | |||
| 1283 | - | // crossed their deletion durability boundaries. | |||
| 1284 | - | if tombstoneErr := snapshots.Tombstone(ctx, name); tombstoneErr != nil { | |||
| 1285 | - | _ = p.Close() | |||
| 1286 | - | return fmt.Errorf("vev: marking session killed: %w", tombstoneErr) | |||
| 1287 | - | } | |||
| 1288 | - | incrementalErr := snapshots.Delete(ctx, name) | |||
| 1289 | - | legacyErr := snapshots.DeleteLegacy(ctx, name) | |||
| 1290 | - | if incrementalErr != nil || legacyErr != nil { | |||
| 1291 | - | _ = p.Close() | |||
| 1292 | - | return errors.Join( | |||
| 1293 | - | phaseError("deleting incremental session snapshot", incrementalErr), | |||
| 1294 | - | phaseError("deleting legacy session snapshot", legacyErr), | |||
| 1295 | - | ) | |||
| 1296 | - | } | |||
| 1297 | - | if deleteErr := p.Delete(name); deleteErr != nil { | |||
| 1298 | - | _ = p.Close() | |||
| 1299 | - | return fmt.Errorf("vev: deleting stored session: %w", deleteErr) | |||
| 1300 | - | } | |||
| 1301 | - | if tombstoneErr := snapshots.DeleteTombstone(ctx, name); tombstoneErr != nil { | |||
| 1302 | - | _ = p.Close() | |||
| 1303 | - | return fmt.Errorf("vev: clearing killed session marker: %w", tombstoneErr) | |||
| 1304 | - | } | |||
| 1305 | - | if closeErr := p.Close(); closeErr != nil { | |||
| 1306 | - | return fmt.Errorf("vev: closing stored sessions: %w", closeErr) | |||
| 1307 | } | 1418 | } | ||
| 1308 | printKillSuccess(name, all, daemon) | 1419 | printKillSuccess(name, all, daemon) | ||
| 1309 | return nil | 1420 | return nil | ||
M internal/app/run_test.go +437 -25
| @@ -1,10 +1,13 @@ | |||||
| 1 | package app | 1 | package app | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | + | "bufio" | |||
| 4 | "bytes" | 5 | "bytes" | ||
| 5 | "context" | 6 | "context" | ||
| 7 | + | "encoding/json" | |||
| 6 | "errors" | 8 | "errors" | ||
| 7 | "io" | 9 | "io" | ||
| 10 | + | "log/slog" | |||
| 8 | "net" | 11 | "net" | ||
| 9 | "os" | 12 | "os" | ||
| 10 | "os/exec" | 13 | "os/exec" | ||
| @@ -13,17 +16,395 @@ import ( | |||||
| 13 | "testing" | 16 | "testing" | ||
| 14 | "time" | 17 | "time" | ||
| 15 | 18 | ||||
| 16 | - | "github.com/bnema/vev/internal/adapters/snapshot" | 19 | + | "github.com/bnema/vev/internal/adapters/lifecycle" |
| 17 | "github.com/bnema/vev/internal/domain" | 20 | "github.com/bnema/vev/internal/domain" | ||
| 18 | "github.com/bnema/vev/internal/persist" | 21 | "github.com/bnema/vev/internal/persist" | ||
| 19 | "github.com/bnema/vev/internal/ports" | 22 | "github.com/bnema/vev/internal/ports" | ||
| 20 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | 23 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | ||
| 21 | "github.com/bnema/vev/internal/usecase/client" | 24 | "github.com/bnema/vev/internal/usecase/client" | ||
| 22 | "github.com/bnema/vev/internal/usecase/confirm" | 25 | "github.com/bnema/vev/internal/usecase/confirm" | ||
| 26 | + | "github.com/bnema/vev/internal/usecase/daemon" | |||
| 27 | + | "github.com/bnema/vev/pkg/kv" | |||
| 28 | + | "github.com/bnema/vev/pkg/safedir" | |||
| 23 | "github.com/stretchr/testify/mock" | 29 | "github.com/stretchr/testify/mock" | ||
| 24 | "github.com/stretchr/testify/require" | 30 | "github.com/stretchr/testify/require" | ||
| 25 | ) | 31 | ) | ||
| 26 | 32 | ||||
| 33 | + | func decodeJSONLogs(t *testing.T, data []byte) []map[string]any { | |||
| 34 | + | t.Helper() | |||
| 35 | + | var entries []map[string]any | |||
| 36 | + | scanner := bufio.NewScanner(bytes.NewReader(data)) | |||
| 37 | + | for scanner.Scan() { | |||
| 38 | + | var entry map[string]any | |||
| 39 | + | require.NoError(t, json.Unmarshal(scanner.Bytes(), &entry)) | |||
| 40 | + | entries = append(entries, entry) | |||
| 41 | + | } | |||
| 42 | + | require.NoError(t, scanner.Err()) | |||
| 43 | + | return entries | |||
| 44 | + | } | |||
| 45 | + | ||||
| 46 | + | func eventNames(entries []map[string]any) []string { | |||
| 47 | + | names := make([]string, 0, len(entries)) | |||
| 48 | + | for _, entry := range entries { | |||
| 49 | + | if name, ok := entry["msg"].(string); ok { | |||
| 50 | + | names = append(names, name) | |||
| 51 | + | } | |||
| 52 | + | } | |||
| 53 | + | return names | |||
| 54 | + | } | |||
| 55 | + | ||||
| 56 | + | func requireEvent(t *testing.T, entries []map[string]any, name string) map[string]any { | |||
| 57 | + | t.Helper() | |||
| 58 | + | for _, entry := range entries { | |||
| 59 | + | if entry["msg"] == name { | |||
| 60 | + | return entry | |||
| 61 | + | } | |||
| 62 | + | } | |||
| 63 | + | t.Fatalf("event %q not found in %v", name, eventNames(entries)) | |||
| 64 | + | return nil | |||
| 65 | + | } | |||
| 66 | + | ||||
| 67 | + | func TestRecoveryObservability(t *testing.T) { | |||
| 68 | + | var logBuffer bytes.Buffer | |||
| 69 | + | log := slog.New(slog.NewJSONHandler(&logBuffer, nil)) | |||
| 70 | + | owner := fakeLifecycleOwnership{release: func() error { return nil }} | |||
| 71 | + | deps := lifecycleStartupDeps{ | |||
| 72 | + | ensurePrivate: func(string) error { return nil }, | |||
| 73 | + | acquire: func(context.Context, string, time.Duration) (lifecycleOwnership, error) { | |||
| 74 | + | return owner, nil | |||
| 75 | + | }, | |||
| 76 | + | log: log, | |||
| 77 | + | } | |||
| 78 | + | ref := &domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}} | |||
| 79 | + | records := []domain.CatalogueRecord{ | |||
| 80 | + | {Committed: ref}, | |||
| 81 | + | {Committed: ref}, | |||
| 82 | + | {}, | |||
| 83 | + | {Committed: ref, DegradedReason: "checkpoint validation failed"}, | |||
| 84 | + | } | |||
| 85 | + | require.NoError(t, runWithLifecycleOwnerDeps(context.Background(), "/runtime/vev", "/state/vev", func(context.Context) error { | |||
| 86 | + | logCatalogueRecovery(log, records, "current") | |||
| 87 | + | logStartupRecoveryCounts(log, records, 0) | |||
| 88 | + | return nil | |||
| 89 | + | }, deps)) | |||
| 90 | + | ||||
| 91 | + | entries := decodeJSONLogs(t, logBuffer.Bytes()) | |||
| 92 | + | for _, name := range []string{"lifecycle_owner_wait", "lifecycle_owner_acquired", "catalogue_validated", "daemon_startup_complete", "lifecycle_owner_released"} { | |||
| 93 | + | require.Contains(t, eventNames(entries), name) | |||
| 94 | + | } | |||
| 95 | + | startup := requireEvent(t, entries, "daemon_startup_complete") | |||
| 96 | + | require.EqualValues(t, 2, startup["healthy"]) | |||
| 97 | + | require.EqualValues(t, 1, startup["fresh"]) | |||
| 98 | + | require.EqualValues(t, 0, startup["restoring"]) | |||
| 99 | + | require.EqualValues(t, 1, startup["broken"]) | |||
| 100 | + | require.NotContains(t, eventNames(entries), "interrupted_transaction_recovery_complete") | |||
| 101 | + | } | |||
| 102 | + | ||||
| 103 | + | // A session-scoped conflict no longer aborts startup, so the daemon comes up | |||
| 104 | + | // looking healthy. Startup must therefore name the fenced session in the log, | |||
| 105 | + | // persist a notice for it, and never report the recovery as clean. | |||
| 106 | + | type fakeLifecycleOwnership struct { | |||
| 107 | + | release func() error | |||
| 108 | + | } | |||
| 109 | + | ||||
| 110 | + | func (o fakeLifecycleOwnership) Release() error { return o.release() } | |||
| 111 | + | ||||
| 112 | + | type fakeLifecycleProbe struct { | |||
| 113 | + | owner lifecycleOwnership | |||
| 114 | + | err error | |||
| 115 | + | } | |||
| 116 | + | ||||
| 117 | + | func (p fakeLifecycleProbe) TryAcquire(string) (lifecycleOwnership, error) { | |||
| 118 | + | return p.owner, p.err | |||
| 119 | + | } | |||
| 120 | + | ||||
| 121 | + | func TestJoinLifecycleReleaseError(t *testing.T) { | |||
| 122 | + | operationErr := errors.New("operation failed") | |||
| 123 | + | releaseErr := errors.New("release failed") | |||
| 124 | + | ||||
| 125 | + | tests := []struct { | |||
| 126 | + | name string | |||
| 127 | + | operationErr error | |||
| 128 | + | releaseErr error | |||
| 129 | + | wantErrors []error | |||
| 130 | + | }{ | |||
| 131 | + | { | |||
| 132 | + | name: "successful operation preserves release failure", | |||
| 133 | + | releaseErr: releaseErr, | |||
| 134 | + | wantErrors: []error{releaseErr}, | |||
| 135 | + | }, | |||
| 136 | + | { | |||
| 137 | + | name: "operation and release failures are joined", | |||
| 138 | + | operationErr: operationErr, | |||
| 139 | + | releaseErr: releaseErr, | |||
| 140 | + | wantErrors: []error{operationErr, releaseErr}, | |||
| 141 | + | }, | |||
| 142 | + | } | |||
| 143 | + | ||||
| 144 | + | for _, tt := range tests { | |||
| 145 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 146 | + | got := tt.operationErr | |||
| 147 | + | joinLifecycleReleaseError(&got, fakeLifecycleOwnership{release: func() error { | |||
| 148 | + | return tt.releaseErr | |||
| 149 | + | }}) | |||
| 150 | + | ||||
| 151 | + | for _, wantErr := range tt.wantErrors { | |||
| 152 | + | require.ErrorIs(t, got, wantErr) | |||
| 153 | + | } | |||
| 154 | + | }) | |||
| 155 | + | } | |||
| 156 | + | } | |||
| 157 | + | ||||
| 158 | + | func TestOfflineCommandsPropagateLifecycleReleaseErrors(t *testing.T) { | |||
| 159 | + | releaseErr := errors.New("release failed") | |||
| 160 | + | ||||
| 161 | + | tests := []struct { | |||
| 162 | + | name string | |||
| 163 | + | run func(context.Context) error | |||
| 164 | + | wantOperationErr string | |||
| 165 | + | }{ | |||
| 166 | + | { | |||
| 167 | + | name: "list joins release failure to success", | |||
| 168 | + | run: runList, | |||
| 169 | + | }, | |||
| 170 | + | { | |||
| 171 | + | name: "kill joins release failure to operation failure", | |||
| 172 | + | run: func(ctx context.Context) error { | |||
| 173 | + | return runKill(ctx, "", false, false) | |||
| 174 | + | }, | |||
| 175 | + | wantOperationErr: "vev: no daemon running", | |||
| 176 | + | }, | |||
| 177 | + | } | |||
| 178 | + | ||||
| 179 | + | originalProbe := daemonLifecycleProbe | |||
| 180 | + | t.Cleanup(func() { daemonLifecycleProbe = originalProbe }) | |||
| 181 | + | for _, tt := range tests { | |||
| 182 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 183 | + | t.Setenv("XDG_RUNTIME_DIR", t.TempDir()) | |||
| 184 | + | t.Setenv("XDG_STATE_HOME", t.TempDir()) | |||
| 185 | + | daemonLifecycleProbe = fakeLifecycleProbe{ | |||
| 186 | + | owner: fakeLifecycleOwnership{release: func() error { return releaseErr }}, | |||
| 187 | + | } | |||
| 188 | + | ||||
| 189 | + | err := tt.run(context.Background()) | |||
| 190 | + | require.ErrorIs(t, err, releaseErr) | |||
| 191 | + | if tt.wantOperationErr != "" { | |||
| 192 | + | require.ErrorContains(t, err, tt.wantOperationErr) | |||
| 193 | + | } | |||
| 194 | + | }) | |||
| 195 | + | } | |||
| 196 | + | } | |||
| 197 | + | ||||
| 198 | + | func TestCatalogueFailureDoesNotListen(t *testing.T) { | |||
| 199 | + | originalOpenCatalogue, originalListenDaemon := openCatalogue, listenDaemon | |||
| 200 | + | t.Cleanup(func() { | |||
| 201 | + | openCatalogue = originalOpenCatalogue | |||
| 202 | + | listenDaemon = originalListenDaemon | |||
| 203 | + | }) | |||
| 204 | + | ||||
| 205 | + | for _, catalogueErr := range []error{errors.New("catalogue corrupt"), errors.New("catalogue unavailable")} { | |||
| 206 | + | t.Run(catalogueErr.Error(), func(t *testing.T) { | |||
| 207 | + | runtimeRoot, stateRoot := t.TempDir(), t.TempDir() | |||
| 208 | + | runtimeDir := filepath.Join(runtimeRoot, "vev") | |||
| 209 | + | stateDir := filepath.Join(stateRoot, "vev") | |||
| 210 | + | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | |||
| 211 | + | t.Setenv("XDG_STATE_HOME", stateRoot) | |||
| 212 | + | ||||
| 213 | + | listenCalls := 0 | |||
| 214 | + | openCatalogue = func(gotStateDir string) (persist.OpenResult, error) { | |||
| 215 | + | require.Equal(t, stateDir, gotStateDir) | |||
| 216 | + | return persist.OpenResult{}, catalogueErr | |||
| 217 | + | } | |||
| 218 | + | listenDaemon = func(string, ports.SerializedRuntimeObserver) (ports.Listener, error) { | |||
| 219 | + | listenCalls++ | |||
| 220 | + | return nil, errors.New("IPC listen must not run after catalogue failure") | |||
| 221 | + | } | |||
| 222 | + | ||||
| 223 | + | err := runWithLifecycleOwner(context.Background(), runtimeDir, stateDir, func(ctx context.Context) error { | |||
| 224 | + | return runDaemonOwnedWithLogger(ctx, slog.New(slog.NewTextHandler(io.Discard, nil))) | |||
| 225 | + | }) | |||
| 226 | + | require.ErrorIs(t, err, catalogueErr) | |||
| 227 | + | require.Zero(t, listenCalls) | |||
| 228 | + | require.NoFileExists(t, filepath.Join(runtimeDir, "daemon.sock")) | |||
| 229 | + | ||||
| 230 | + | owner, acquireErr := lifecycle.TryAcquire(runtimeDir) | |||
| 231 | + | require.NoError(t, acquireErr, "catalogue failure must release lifecycle ownership") | |||
| 232 | + | require.NoError(t, owner.Release()) | |||
| 233 | + | }) | |||
| 234 | + | } | |||
| 235 | + | } | |||
| 236 | + | ||||
| 237 | + | func TestCatalogueRegistryConstructionPrecedesSocketPublication(t *testing.T) { | |||
| 238 | + | var events []string | |||
| 239 | + | _, _, err := constructDaemonBeforeSocketPublication( | |||
| 240 | + | func() *daemon.Daemon { | |||
| 241 | + | events = append(events, "catalogue-registry") | |||
| 242 | + | return nil | |||
| 243 | + | }, | |||
| 244 | + | func(*daemon.Daemon) error { | |||
| 245 | + | require.Equal(t, []string{"catalogue-registry"}, events) | |||
| 246 | + | events = append(events, "startup-garbage-collection") | |||
| 247 | + | return nil | |||
| 248 | + | }, | |||
| 249 | + | func() (ports.Listener, error) { | |||
| 250 | + | require.Equal(t, []string{"catalogue-registry", "startup-garbage-collection"}, events) | |||
| 251 | + | events = append(events, "socket-publication") | |||
| 252 | + | return nil, nil | |||
| 253 | + | }, | |||
| 254 | + | ) | |||
| 255 | + | require.NoError(t, err) | |||
| 256 | + | require.Equal(t, []string{"catalogue-registry", "startup-garbage-collection", "socket-publication"}, events) | |||
| 257 | + | } | |||
| 258 | + | ||||
| 259 | + | func TestLifecycleOwnershipPrecedesDaemonStartup(t *testing.T) { | |||
| 260 | + | t.Run("ownership prefix is exact", func(t *testing.T) { | |||
| 261 | + | var events []string | |||
| 262 | + | owner := fakeLifecycleOwnership{release: func() error { | |||
| 263 | + | events = append(events, "unlock") | |||
| 264 | + | return nil | |||
| 265 | + | }} | |||
| 266 | + | deps := lifecycleStartupDeps{ | |||
| 267 | + | ensurePrivate: func(path string) error { | |||
| 268 | + | if path == "runtime" { | |||
| 269 | + | events = append(events, "ensure-runtime") | |||
| 270 | + | } else { | |||
| 271 | + | events = append(events, "ensure-state") | |||
| 272 | + | } | |||
| 273 | + | return nil | |||
| 274 | + | }, | |||
| 275 | + | acquire: func(context.Context, string, time.Duration) (lifecycleOwnership, error) { | |||
| 276 | + | events = append(events, "lock") | |||
| 277 | + | return owner, nil | |||
| 278 | + | }, | |||
| 279 | + | } | |||
| 280 | + | ||||
| 281 | + | err := runWithLifecycleOwnerDeps(context.Background(), "runtime", "state", func(context.Context) error { | |||
| 282 | + | events = append(events, "durable-open", "listen", "serve") | |||
| 283 | + | return nil | |||
| 284 | + | }, deps) | |||
| 285 | + | require.NoError(t, err) | |||
| 286 | + | require.Equal(t, []string{"ensure-runtime", "ensure-state", "lock"}, events[:3]) | |||
| 287 | + | require.Equal(t, []string{"ensure-runtime", "ensure-state", "lock", "durable-open", "listen", "serve", "unlock"}, events) | |||
| 288 | + | }) | |||
| 289 | + | ||||
| 290 | + | t.Run("callback observes held lock and failure releases it", func(t *testing.T) { | |||
| 291 | + | runtimeDir := filepath.Join(t.TempDir(), "runtime") | |||
| 292 | + | stateDir := filepath.Join(t.TempDir(), "state") | |||
| 293 | + | startErr := errors.New("catalogue corrupt") | |||
| 294 | + | ||||
| 295 | + | err := runWithLifecycleOwner(context.Background(), runtimeDir, stateDir, func(context.Context) error { | |||
| 296 | + | _, lockErr := lifecycle.TryAcquire(runtimeDir) | |||
| 297 | + | require.ErrorIs(t, lockErr, lifecycle.ErrBusy) | |||
| 298 | + | return startErr | |||
| 299 | + | }) | |||
| 300 | + | require.ErrorIs(t, err, startErr) | |||
| 301 | + | ||||
| 302 | + | owner, err := lifecycle.TryAcquire(runtimeDir) | |||
| 303 | + | require.NoError(t, err, "callback failure must release lifecycle ownership") | |||
| 304 | + | require.NoError(t, owner.Release()) | |||
| 305 | + | }) | |||
| 306 | + | ||||
| 307 | + | t.Run("busy and unavailable ownership fail before startup", func(t *testing.T) { | |||
| 308 | + | runtimeDir := filepath.Join(t.TempDir(), "runtime") | |||
| 309 | + | stateDir := filepath.Join(t.TempDir(), "state") | |||
| 310 | + | owner, err := lifecycle.TryAcquire(runtimeDir) | |||
| 311 | + | require.NoError(t, err) | |||
| 312 | + | defer func() { require.NoError(t, owner.Release()) }() | |||
| 313 | + | ||||
| 314 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 315 | + | cancel() | |||
| 316 | + | started := false | |||
| 317 | + | err = runWithLifecycleOwner(ctx, runtimeDir, stateDir, func(context.Context) error { | |||
| 318 | + | started = true | |||
| 319 | + | return nil | |||
| 320 | + | }) | |||
| 321 | + | require.ErrorIs(t, err, context.Canceled) | |||
| 322 | + | require.False(t, started) | |||
| 323 | + | ||||
| 324 | + | badState := filepath.Join(t.TempDir(), "state-file") | |||
| 325 | + | require.NoError(t, os.WriteFile(badState, []byte("unavailable"), 0o600)) | |||
| 326 | + | err = runWithLifecycleOwner(context.Background(), filepath.Join(t.TempDir(), "other-runtime"), badState, func(context.Context) error { | |||
| 327 | + | started = true | |||
| 328 | + | return nil | |||
| 329 | + | }) | |||
| 330 | + | require.Error(t, err) | |||
| 331 | + | require.False(t, started) | |||
| 332 | + | }) | |||
| 333 | + | ||||
| 334 | + | t.Run("corrupt catalogue fails before socket publication", func(t *testing.T) { | |||
| 335 | + | runtimeRoot, stateRoot := t.TempDir(), t.TempDir() | |||
| 336 | + | runtimeDir := filepath.Join(runtimeRoot, "vev") | |||
| 337 | + | stateDir := filepath.Join(stateRoot, "vev") | |||
| 338 | + | require.NoError(t, safedir.EnsurePrivate(stateDir)) | |||
| 339 | + | store, err := kv.Open(persist.StorePath(stateDir)) | |||
| 340 | + | require.NoError(t, err) | |||
| 341 | + | require.NoError(t, store.Set([]byte("work"), []byte("malformed catalogue value"))) | |||
| 342 | + | require.NoError(t, store.Close()) | |||
| 343 | + | ||||
| 344 | + | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | |||
| 345 | + | t.Setenv("XDG_STATE_HOME", stateRoot) | |||
| 346 | + | before, err := os.ReadFile(persist.StorePath(stateDir)) | |||
| 347 | + | require.NoError(t, err) | |||
| 348 | + | err = runWithLifecycleOwner(context.Background(), runtimeDir, stateDir, runDaemonOwned) | |||
| 349 | + | require.Error(t, err) | |||
| 350 | + | require.Contains(t, err.Error(), stateDir) | |||
| 351 | + | require.Contains(t, err.Error(), "rm -rf "+stateDir) | |||
| 352 | + | after, readErr := os.ReadFile(persist.StorePath(stateDir)) | |||
| 353 | + | require.NoError(t, readErr) | |||
| 354 | + | require.Equal(t, before, after, "failed startup must leave the catalogue untouched") | |||
| 355 | + | _, statErr := os.Stat(filepath.Join(runtimeDir, "daemon.sock")) | |||
| 356 | + | require.ErrorIs(t, statErr, os.ErrNotExist) | |||
| 357 | + | }) | |||
| 358 | + | ||||
| 359 | + | t.Run("proven absence creates the catalogue before socket publication", func(t *testing.T) { | |||
| 360 | + | runtimeRoot, err := os.MkdirTemp("/tmp", "vev-") | |||
| 361 | + | require.NoError(t, err) | |||
| 362 | + | t.Cleanup(func() { require.NoError(t, os.RemoveAll(runtimeRoot)) }) | |||
| 363 | + | stateRoot := t.TempDir() | |||
| 364 | + | runtimeDir := filepath.Join(runtimeRoot, "vev") | |||
| 365 | + | stateDir := filepath.Join(stateRoot, "vev") | |||
| 366 | + | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | |||
| 367 | + | t.Setenv("XDG_STATE_HOME", stateRoot) | |||
| 368 | + | ctx, cancel := context.WithCancel(t.Context()) | |||
| 369 | + | result := make(chan error, 1) | |||
| 370 | + | go func() { | |||
| 371 | + | result <- runWithLifecycleOwner(ctx, runtimeDir, stateDir, runDaemonOwned) | |||
| 372 | + | }() | |||
| 373 | + | t.Cleanup(func() { | |||
| 374 | + | cancel() | |||
| 375 | + | select { | |||
| 376 | + | case err := <-result: | |||
| 377 | + | require.NoError(t, err) | |||
| 378 | + | case <-time.After(5 * time.Second): | |||
| 379 | + | t.Error("daemon lifecycle did not stop after cancellation") | |||
| 380 | + | } | |||
| 381 | + | }) | |||
| 382 | + | ||||
| 383 | + | require.Eventually(t, func() bool { | |||
| 384 | + | _, err := os.Stat(filepath.Join(runtimeDir, "daemon.sock")) | |||
| 385 | + | return err == nil | |||
| 386 | + | }, 5*time.Second, time.Millisecond, "daemon socket was not published") | |||
| 387 | + | require.FileExists(t, persist.StorePath(stateDir)) | |||
| 388 | + | }) | |||
| 389 | + | ||||
| 390 | + | t.Run("callback and release errors are joined", func(t *testing.T) { | |||
| 391 | + | startErr := errors.New("catalogue unavailable") | |||
| 392 | + | releaseErr := errors.New("unlock failed") | |||
| 393 | + | deps := lifecycleStartupDeps{ | |||
| 394 | + | ensurePrivate: func(string) error { return nil }, | |||
| 395 | + | acquire: func(context.Context, string, time.Duration) (lifecycleOwnership, error) { | |||
| 396 | + | return fakeLifecycleOwnership{release: func() error { return releaseErr }}, nil | |||
| 397 | + | }, | |||
| 398 | + | } | |||
| 399 | + | ||||
| 400 | + | err := runWithLifecycleOwnerDeps(context.Background(), "runtime", "state", func(context.Context) error { | |||
| 401 | + | return startErr | |||
| 402 | + | }, deps) | |||
| 403 | + | require.ErrorIs(t, err, startErr) | |||
| 404 | + | require.ErrorIs(t, err, releaseErr) | |||
| 405 | + | }) | |||
| 406 | + | } | |||
| 407 | + | ||||
| 27 | // scriptRecv makes a transport yield frames in order, then wait for done | 408 | // scriptRecv makes a transport yield frames in order, then wait for done | ||
| 28 | // before returning EOF. The shared done channel coordinates proxy readers | 409 | // before returning EOF. The shared done channel coordinates proxy readers | ||
| 29 | // without relying on scheduling or sleeps. | 410 | // without relying on scheduling or sleeps. | ||
| @@ -266,11 +647,26 @@ func TestParseArgsNewRejectsUnsafeSessionName(t *testing.T) { | |||||
| 266 | } | 647 | } | ||
| 267 | } | 648 | } | ||
| 268 | 649 | ||||
| 650 | + | func TestListShowsBroken(t *testing.T) { | |||
| 651 | + | var out bytes.Buffer | |||
| 652 | + | printSessions(&out, []ports.SessionInfo{ | |||
| 653 | + | {Name: "fresh", State: ports.SessionStopped}, | |||
| 654 | + | {Name: "loading", State: ports.SessionStopped}, | |||
| 655 | + | {Name: "broken", State: ports.SessionBroken}, | |||
| 656 | + | }) | |||
| 657 | + | require.Contains(t, out.String(), "fresh") | |||
| 658 | + | require.Contains(t, out.String(), "stopped") | |||
| 659 | + | require.Contains(t, out.String(), "loading") | |||
| 660 | + | require.Contains(t, out.String(), "broken") | |||
| 661 | + | require.NotContains(t, out.String(), "restoring") | |||
| 662 | + | require.NotContains(t, out.String(), "degraded") | |||
| 663 | + | } | |||
| 664 | + | ||||
| 269 | func TestPrintSessionsShowsStoppedState(t *testing.T) { | 665 | func TestPrintSessionsShowsStoppedState(t *testing.T) { | ||
| 270 | var out bytes.Buffer | 666 | var out bytes.Buffer | ||
| 271 | printSessions(&out, []ports.SessionInfo{ | 667 | printSessions(&out, []ports.SessionInfo{ | ||
| 272 | - | {Name: "main", Tabs: 2, Attached: true}, | 668 | + | {Name: "main", State: ports.SessionRunning, Tabs: 2, Attached: true}, |
| 273 | - | {Name: "old", Stopped: true}, | 669 | + | {Name: "old", State: ports.SessionStopped}, |
| 274 | }) | 670 | }) | ||
| 275 | got := out.String() | 671 | got := out.String() | ||
| 276 | for _, want := range []string{"NAME", "STATE", "main", "running", "2", "yes", "old", "stopped", "-"} { | 672 | for _, want := range []string{"NAME", "STATE", "main", "running", "2", "yes", "old", "stopped", "-"} { | ||
| @@ -283,9 +679,9 @@ func TestPrintSessionsShowsStoppedState(t *testing.T) { | |||||
| 283 | func TestPrintSessionsMarksEphemeral(t *testing.T) { | 679 | func TestPrintSessionsMarksEphemeral(t *testing.T) { | ||
| 284 | var buf bytes.Buffer | 680 | var buf bytes.Buffer | ||
| 285 | printSessions(&buf, []ports.SessionInfo{ | 681 | printSessions(&buf, []ports.SessionInfo{ | ||
| 286 | - | {Name: "0", Ephemeral: true, Tabs: 1, Attached: false}, | 682 | + | {Name: "0", State: ports.SessionRunning, Ephemeral: true, Tabs: 1, Attached: false}, |
| 287 | - | {Name: "work", Tabs: 2, Attached: true}, | 683 | + | {Name: "work", State: ports.SessionRunning, Tabs: 2, Attached: true}, |
| 288 | - | {Name: "old", Stopped: true}, | 684 | + | {Name: "old", State: ports.SessionStopped}, |
| 289 | }) | 685 | }) | ||
| 290 | out := buf.String() | 686 | out := buf.String() | ||
| 291 | for _, want := range []string{"0", "temporary", "work", "running", "old", "stopped"} { | 687 | for _, want := range []string{"0", "temporary", "work", "running", "old", "stopped"} { | ||
| @@ -293,17 +689,21 @@ func TestPrintSessionsMarksEphemeral(t *testing.T) { | |||||
| 293 | } | 689 | } | ||
| 294 | } | 690 | } | ||
| 295 | 691 | ||||
| 692 | + | func newTestPersister(t *testing.T, stateDir string) *persist.Persister { | |||
| 693 | + | t.Helper() | |||
| 694 | + | store, err := persist.OpenStore(persist.StorePath(stateDir)) | |||
| 695 | + | require.NoError(t, err) | |||
| 696 | + | return persist.New(store) | |||
| 697 | + | } | |||
| 698 | + | ||||
| 296 | func TestRunListReadsStoppedSessionsWithoutDaemon(t *testing.T) { | 699 | func TestRunListReadsStoppedSessionsWithoutDaemon(t *testing.T) { | ||
| 297 | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | 700 | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | ||
| 298 | t.Setenv("XDG_STATE_HOME", stateRoot) | 701 | t.Setenv("XDG_STATE_HOME", stateRoot) | ||
| 299 | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | 702 | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | ||
| 300 | 703 | ||||
| 301 | - | p, err := persist.Open(filepath.Join(stateRoot, "vev")) | 704 | + | p := newTestPersister(t, filepath.Join(stateRoot, "vev")) |
| 302 | - | if err != nil { | |||
| 303 | - | t.Fatalf("persist.Open error = %v", err) | |||
| 304 | - | } | |||
| 305 | now := time.Now().UnixNano() | 705 | now := time.Now().UnixNano() | ||
| 306 | - | if err := p.Save(persist.Record{Name: "stored", Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now}); err != nil { | 706 | + | if err := p.Save(persist.Record{Name: "stored", IncarnationID: domain.IncarnationID{1}, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now}); err != nil { |
| 307 | t.Fatalf("Save error = %v", err) | 707 | t.Fatalf("Save error = %v", err) | ||
| 308 | } | 708 | } | ||
| 309 | if err := p.Close(); err != nil { | 709 | if err := p.Close(); err != nil { | ||
| @@ -322,6 +722,30 @@ func TestRunListReadsStoppedSessionsWithoutDaemon(t *testing.T) { | |||||
| 322 | } | 722 | } | ||
| 323 | } | 723 | } | ||
| 324 | 724 | ||||
| 725 | + | func TestRunListUnreadableCatalogueProvidesResetGuidance(t *testing.T) { | |||
| 726 | + | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | |||
| 727 | + | stateDir := filepath.Join(stateRoot, "vev") | |||
| 728 | + | runtimeDir := filepath.Join(runtimeRoot, "vev") | |||
| 729 | + | t.Setenv("XDG_STATE_HOME", stateRoot) | |||
| 730 | + | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | |||
| 731 | + | require.NoError(t, safedir.EnsurePrivate(stateDir)) | |||
| 732 | + | ||||
| 733 | + | catalogue := persist.StorePath(stateDir) | |||
| 734 | + | before := []byte("corrupt catalogue") | |||
| 735 | + | require.NoError(t, os.WriteFile(catalogue, before, 0o600)) | |||
| 736 | + | ||||
| 737 | + | var runErr error | |||
| 738 | + | stdout := captureStdout(t, func() { runErr = Run([]string{"ls"}) }) | |||
| 739 | + | require.Empty(t, stdout) | |||
| 740 | + | require.Error(t, runErr) | |||
| 741 | + | require.ErrorContains(t, runErr, stateDir) | |||
| 742 | + | require.ErrorContains(t, runErr, "rm -rf "+stateDir) | |||
| 743 | + | require.NoFileExists(t, filepath.Join(runtimeDir, "daemon.sock")) | |||
| 744 | + | after, err := os.ReadFile(catalogue) | |||
| 745 | + | require.NoError(t, err) | |||
| 746 | + | require.Equal(t, before, after) | |||
| 747 | + | } | |||
| 748 | + | ||||
| 325 | func TestRunKillMissingStoppedSessionDoesNotCreateStore(t *testing.T) { | 749 | func TestRunKillMissingStoppedSessionDoesNotCreateStore(t *testing.T) { | ||
| 326 | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | 750 | stateRoot, runtimeRoot := t.TempDir(), t.TempDir() | ||
| 327 | t.Setenv("XDG_STATE_HOME", stateRoot) | 751 | t.Setenv("XDG_STATE_HOME", stateRoot) | ||
| @@ -341,19 +765,14 @@ func TestRunKillDeletesStoppedSessionWithoutDaemon(t *testing.T) { | |||||
| 341 | t.Setenv("XDG_STATE_HOME", stateRoot) | 765 | t.Setenv("XDG_STATE_HOME", stateRoot) | ||
| 342 | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | 766 | t.Setenv("XDG_RUNTIME_DIR", runtimeRoot) | ||
| 343 | 767 | ||||
| 344 | - | p, err := persist.Open(filepath.Join(stateRoot, "vev")) | 768 | + | p := newTestPersister(t, filepath.Join(stateRoot, "vev")) |
| 345 | - | if err != nil { | |||
| 346 | - | t.Fatalf("persist.Open error = %v", err) | |||
| 347 | - | } | |||
| 348 | now := time.Now().UnixNano() | 769 | now := time.Now().UnixNano() | ||
| 349 | - | if err := p.Save(persist.Record{Name: "stored", Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now}); err != nil { | 770 | + | if err := p.Save(persist.Record{Name: "stored", IncarnationID: domain.IncarnationID{1}, Cwd: t.TempDir(), CreatedAt: now, UpdatedAt: now}); err != nil { |
| 350 | t.Fatalf("Save error = %v", err) | 771 | t.Fatalf("Save error = %v", err) | ||
| 351 | } | 772 | } | ||
| 352 | if err := p.Close(); err != nil { | 773 | if err := p.Close(); err != nil { | ||
| 353 | t.Fatalf("Close error = %v", err) | 774 | t.Fatalf("Close error = %v", err) | ||
| 354 | } | 775 | } | ||
| 355 | - | snapshots := snapshot.NewRepository(filepath.Join(stateRoot, "vev", "snapshots")) | |||
| 356 | - | ||||
| 357 | got := captureStdout(t, func() { | 776 | got := captureStdout(t, func() { | ||
| 358 | if err := runKill(context.Background(), "stored", false, false); err != nil { | 777 | if err := runKill(context.Background(), "stored", false, false); err != nil { | ||
| 359 | t.Fatalf("runKill error = %v", err) | 778 | t.Fatalf("runKill error = %v", err) | ||
| @@ -369,13 +788,6 @@ func TestRunKillDeletesStoppedSessionWithoutDaemon(t *testing.T) { | |||||
| 369 | if len(records) != 0 { | 788 | if len(records) != 0 { | ||
| 370 | t.Fatalf("records after kill = %#v, want none", records) | 789 | t.Fatalf("records after kill = %#v, want none", records) | ||
| 371 | } | 790 | } | ||
| 372 | - | names, err := snapshots.List(context.Background()) | |||
| 373 | - | if err != nil { | |||
| 374 | - | t.Fatalf("snapshot List error = %v", err) | |||
| 375 | - | } | |||
| 376 | - | if len(names) != 0 { | |||
| 377 | - | t.Fatalf("snapshots after kill = %#v, want none", names) | |||
| 378 | - | } | |||
| 379 | } | 791 | } | ||
| 380 | 792 | ||||
| 381 | func captureStdout(t *testing.T, fn func()) string { | 793 | func captureStdout(t *testing.T, fn func()) string { | ||
M internal/app/spawn.go +123 -24
| @@ -13,6 +13,7 @@ import ( | |||||
| 13 | "time" | 13 | "time" | ||
| 14 | 14 | ||||
| 15 | "github.com/bnema/vev/internal/adapters/ipc" | 15 | "github.com/bnema/vev/internal/adapters/ipc" | ||
| 16 | + | "github.com/bnema/vev/internal/adapters/lifecycle" | |||
| 16 | "github.com/bnema/vev/internal/platform" | 17 | "github.com/bnema/vev/internal/platform" | ||
| 17 | "github.com/bnema/vev/internal/ports" | 18 | "github.com/bnema/vev/internal/ports" | ||
| 18 | "github.com/bnema/vev/pkg/safedir" | 19 | "github.com/bnema/vev/pkg/safedir" | ||
| @@ -39,6 +40,18 @@ type dialFunc func(ctx context.Context, dir string) (ports.Transport, error) | |||||
| 39 | // re-exec a real binary. | 40 | // re-exec a real binary. | ||
| 40 | type spawnFunc func() error | 41 | type spawnFunc func() error | ||
| 41 | 42 | ||||
| 43 | + | type lifecycleProbe interface { | |||
| 44 | + | TryAcquire(string) (lifecycleOwnership, error) | |||
| 45 | + | } | |||
| 46 | + | ||||
| 47 | + | type osLifecycleProbe struct{} | |||
| 48 | + | ||||
| 49 | + | func (osLifecycleProbe) TryAcquire(runtimeDir string) (lifecycleOwnership, error) { | |||
| 50 | + | return lifecycle.TryAcquire(runtimeDir) | |||
| 51 | + | } | |||
| 52 | + | ||||
| 53 | + | var daemonLifecycleProbe lifecycleProbe = osLifecycleProbe{} | |||
| 54 | + | ||||
| 42 | // backoffConfig parameterises retry-dial timing so tests can shrink it. | 55 | // backoffConfig parameterises retry-dial timing so tests can shrink it. | ||
| 43 | type backoffConfig struct { | 56 | type backoffConfig struct { | ||
| 44 | initial time.Duration | 57 | initial time.Duration | ||
| @@ -54,10 +67,110 @@ var defaultBackoff = backoffConfig{ | |||||
| 54 | total: 5 * time.Second, | 67 | total: 5 * time.Second, | ||
| 55 | } | 68 | } | ||
| 56 | 69 | ||||
| 70 | + | func retryAttempts[T any](ctx context.Context, cfg backoffConfig, attempt func() (T, bool, error)) (T, error) { | |||
| 71 | + | deadline := time.Now().Add(cfg.total) | |||
| 72 | + | backoff := cfg.initial | |||
| 73 | + | if backoff <= 0 { | |||
| 74 | + | backoff = time.Millisecond | |||
| 75 | + | } | |||
| 76 | + | for { | |||
| 77 | + | if err := ctx.Err(); err != nil { | |||
| 78 | + | var zero T | |||
| 79 | + | return zero, err | |||
| 80 | + | } | |||
| 81 | + | result, done, err := attempt() | |||
| 82 | + | if err != nil { | |||
| 83 | + | var zero T | |||
| 84 | + | return zero, err | |||
| 85 | + | } | |||
| 86 | + | if done { | |||
| 87 | + | return result, nil | |||
| 88 | + | } | |||
| 89 | + | if !time.Now().Before(deadline) { | |||
| 90 | + | var zero T | |||
| 91 | + | return zero, ErrDaemonUnreachable | |||
| 92 | + | } | |||
| 93 | + | if err := waitBackoff(ctx, backoff); err != nil { | |||
| 94 | + | var zero T | |||
| 95 | + | return zero, err | |||
| 96 | + | } | |||
| 97 | + | if backoff *= 2; backoff > cfg.max { | |||
| 98 | + | backoff = cfg.max | |||
| 99 | + | } | |||
| 100 | + | } | |||
| 101 | + | } | |||
| 102 | + | ||||
| 103 | + | func waitForLifecycleAvailability(ctx context.Context, runtimeDir string, cfg backoffConfig) (lifecycleOwnership, error) { | |||
| 104 | + | return retryAttempts(ctx, cfg, func() (lifecycleOwnership, bool, error) { | |||
| 105 | + | owner, err := daemonLifecycleProbe.TryAcquire(runtimeDir) | |||
| 106 | + | if err == nil { | |||
| 107 | + | return owner, true, nil | |||
| 108 | + | } | |||
| 109 | + | if !errors.Is(err, lifecycle.ErrBusy) { | |||
| 110 | + | return nil, false, err | |||
| 111 | + | } | |||
| 112 | + | return nil, false, nil | |||
| 113 | + | }) | |||
| 114 | + | } | |||
| 115 | + | ||||
| 116 | + | type daemonOrLifecycle struct { | |||
| 117 | + | transport ports.Transport | |||
| 118 | + | owner lifecycleOwnership | |||
| 119 | + | } | |||
| 120 | + | ||||
| 121 | + | func waitForDaemonOrLifecycle(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, lifecycleOwnership, error) { | |||
| 122 | + | result, err := retryAttempts(ctx, cfg, func() (daemonOrLifecycle, bool, error) { | |||
| 123 | + | if transport, err := dial(ctx, dir); err == nil { | |||
| 124 | + | return daemonOrLifecycle{transport: transport}, true, nil | |||
| 125 | + | } | |||
| 126 | + | owner, err := daemonLifecycleProbe.TryAcquire(dir) | |||
| 127 | + | if err == nil { | |||
| 128 | + | return daemonOrLifecycle{owner: owner}, true, nil | |||
| 129 | + | } | |||
| 130 | + | if !errors.Is(err, lifecycle.ErrBusy) { | |||
| 131 | + | return daemonOrLifecycle{}, false, err | |||
| 132 | + | } | |||
| 133 | + | return daemonOrLifecycle{}, false, nil | |||
| 134 | + | }) | |||
| 135 | + | return result.transport, result.owner, err | |||
| 136 | + | } | |||
| 137 | + | ||||
| 138 | + | func waitBackoff(ctx context.Context, duration time.Duration) error { | |||
| 139 | + | timer := time.NewTimer(duration) | |||
| 140 | + | defer timer.Stop() | |||
| 141 | + | select { | |||
| 142 | + | case <-ctx.Done(): | |||
| 143 | + | return ctx.Err() | |||
| 144 | + | case <-timer.C: | |||
| 145 | + | return nil | |||
| 146 | + | } | |||
| 147 | + | } | |||
| 148 | + | ||||
| 149 | + | // ensureDaemonWithLifecycle never elects a spawner while another lifecycle | |||
| 150 | + | // owner may still be initializing or tearing down durable state. | |||
| 151 | + | func ensureDaemonWithLifecycle(ctx context.Context, dir string, dial dialFunc, spawn spawnFunc, cfg backoffConfig) (ports.Transport, error) { | |||
| 152 | + | if err := ctx.Err(); err != nil { | |||
| 153 | + | return nil, err | |||
| 154 | + | } | |||
| 155 | + | if transport, err := dial(ctx, dir); err == nil { | |||
| 156 | + | return transport, nil | |||
| 157 | + | } | |||
| 158 | + | transport, owner, err := waitForDaemonOrLifecycle(ctx, dir, dial, cfg) | |||
| 159 | + | if err != nil { | |||
| 160 | + | return nil, err | |||
| 161 | + | } | |||
| 162 | + | if transport != nil { | |||
| 163 | + | return transport, nil | |||
| 164 | + | } | |||
| 165 | + | if err := owner.Release(); err != nil { | |||
| 166 | + | return nil, fmt.Errorf("vev: release lifecycle spawn probe: %w", err) | |||
| 167 | + | } | |||
| 168 | + | return ensureDaemon(ctx, dir, dial, spawn, cfg) | |||
| 169 | + | } | |||
| 170 | + | ||||
| 57 | // ensureDaemon returns a transport to a running daemon, spawning one if | 171 | // ensureDaemon returns a transport to a running daemon, spawning one if | ||
| 58 | - | // necessary. It first tries a plain dial; on failure it elects a single | 172 | + | // necessary after ensureDaemonWithLifecycle has established lifecycle |
| 59 | - | // spawner via an mkdir lock (losers simply wait), then retry-dials with | 173 | + | // availability. Tests also exercise this lower-level spawn election directly. |
| 60 | - | // backoff until the socket is live or the budget expires. | |||
| 61 | func ensureDaemon(ctx context.Context, dir string, dial dialFunc, spawn spawnFunc, cfg backoffConfig) (ports.Transport, error) { | 174 | func ensureDaemon(ctx context.Context, dir string, dial dialFunc, spawn spawnFunc, cfg backoffConfig) (ports.Transport, error) { | ||
| 62 | if err := ctx.Err(); err != nil { | 175 | if err := ctx.Err(); err != nil { | ||
| 63 | return nil, err | 176 | return nil, err | ||
| @@ -124,28 +237,14 @@ func acquireSpawnLock(dir string) (release func(), acquired bool, err error) { | |||||
| 124 | // retryDial dials repeatedly with exponential backoff until the daemon | 237 | // retryDial dials repeatedly with exponential backoff until the daemon | ||
| 125 | // answers, the context is cancelled, or the total budget is exhausted. | 238 | // answers, the context is cancelled, or the total budget is exhausted. | ||
| 126 | func retryDial(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, error) { | 239 | func retryDial(ctx context.Context, dir string, dial dialFunc, cfg backoffConfig) (ports.Transport, error) { | ||
| 127 | - | deadline := time.Now().Add(cfg.total) | 240 | + | transport, err := retryAttempts(ctx, cfg, func() (ports.Transport, bool, error) { |
| 128 | - | backoff := cfg.initial | 241 | + | transport, err := dial(ctx, dir) |
| 129 | - | for { | 242 | + | return transport, err == nil, nil |
| 130 | - | if err := ctx.Err(); err != nil { | 243 | + | }) |
| 131 | - | return nil, err | 244 | + | if errors.Is(err, ErrDaemonUnreachable) { |
| 132 | - | } | 245 | + | slog.Error("daemon did not become reachable before retry budget expired", "socket_dir", dir, "budget", cfg.total) |
| 133 | - | if t, err := dial(ctx, dir); err == nil { | |||
| 134 | - | return t, nil | |||
| 135 | - | } | |||
| 136 | - | if !time.Now().Before(deadline) { | |||
| 137 | - | slog.Error("daemon did not become reachable before retry budget expired", "socket_dir", dir, "budget", cfg.total) | |||
| 138 | - | return nil, ErrDaemonUnreachable | |||
| 139 | - | } | |||
| 140 | - | select { | |||
| 141 | - | case <-ctx.Done(): | |||
| 142 | - | return nil, ctx.Err() | |||
| 143 | - | case <-time.After(backoff): | |||
| 144 | - | } | |||
| 145 | - | if backoff *= 2; backoff > cfg.max { | |||
| 146 | - | backoff = cfg.max | |||
| 147 | - | } | |||
| 148 | } | 246 | } | ||
| 247 | + | return transport, err | |||
| 149 | } | 248 | } | ||
| 150 | 249 | ||||
| 151 | // realDial is the production dialer. | 250 | // realDial is the production dialer. | ||
M internal/app/spawn_test.go +30 -0
| @@ -258,6 +258,36 @@ func TestAcquireSpawnLockTakesOverStaleLock(t *testing.T) { | |||||
| 258 | release() | 258 | release() | ||
| 259 | } | 259 | } | ||
| 260 | 260 | ||||
| 261 | + | func TestRetryAttemptsChecksOperationErrorBeforeDeadline(t *testing.T) { | |||
| 262 | + | attemptErr := errors.New("attempt failed") | |||
| 263 | + | calls := 0 | |||
| 264 | + | _, err := retryAttempts(context.Background(), backoffConfig{}, func() (struct{}, bool, error) { | |||
| 265 | + | calls++ | |||
| 266 | + | return struct{}{}, false, attemptErr | |||
| 267 | + | }) | |||
| 268 | + | if !errors.Is(err, attemptErr) { | |||
| 269 | + | t.Fatalf("retryAttempts error = %v, want %v", err, attemptErr) | |||
| 270 | + | } | |||
| 271 | + | if calls != 1 { | |||
| 272 | + | t.Fatalf("attempt called %d times, want 1", calls) | |||
| 273 | + | } | |||
| 274 | + | } | |||
| 275 | + | ||||
| 276 | + | func TestRetryAttemptsClampsNonPositiveInitialDelay(t *testing.T) { | |||
| 277 | + | started := time.Now() | |||
| 278 | + | calls := 0 | |||
| 279 | + | _, err := retryAttempts(context.Background(), backoffConfig{total: time.Second, max: time.Second}, func() (struct{}, bool, error) { | |||
| 280 | + | calls++ | |||
| 281 | + | return struct{}{}, calls == 2, nil | |||
| 282 | + | }) | |||
| 283 | + | if err != nil { | |||
| 284 | + | t.Fatalf("retryAttempts: %v", err) | |||
| 285 | + | } | |||
| 286 | + | if elapsed := time.Since(started); elapsed < 500*time.Microsecond { | |||
| 287 | + | t.Fatalf("retryAttempts retried after %s, want a positive initial delay", elapsed) | |||
| 288 | + | } | |||
| 289 | + | } | |||
| 290 | + | ||||
| 261 | func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) { | 291 | func TestRetryDialSucceedsAfterTransientFailures(t *testing.T) { | ||
| 262 | want := portsmocks.NewMockTransport(t) | 292 | want := portsmocks.NewMockTransport(t) | ||
| 263 | var calls atomic.Int32 | 293 | var calls atomic.Int32 | ||
A internal/domain/recovery.go +134 -0
| @@ -0,0 +1,134 @@ | |||||
| 1 | + | package domain | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "encoding/hex" | |||
| 5 | + | "errors" | |||
| 6 | + | "fmt" | |||
| 7 | + | "io" | |||
| 8 | + | "slices" | |||
| 9 | + | "strings" | |||
| 10 | + | ) | |||
| 11 | + | ||||
| 12 | + | // IncarnationID permanently distinguishes uses of the same session name. | |||
| 13 | + | type IncarnationID [16]byte | |||
| 14 | + | ||||
| 15 | + | // CheckpointRef identifies one immutable snapshot manifest. | |||
| 16 | + | type CheckpointRef struct { | |||
| 17 | + | Generation uint64 | |||
| 18 | + | ManifestDigest [32]byte | |||
| 19 | + | } | |||
| 20 | + | ||||
| 21 | + | // CatalogueRecord is the durable source of truth for one named session. | |||
| 22 | + | type CatalogueRecord struct { | |||
| 23 | + | Name string | |||
| 24 | + | IncarnationID IncarnationID | |||
| 25 | + | Cwd string | |||
| 26 | + | CreatedAt int64 | |||
| 27 | + | UpdatedAt int64 | |||
| 28 | + | LastUsedSeq uint64 | |||
| 29 | + | TabNames []string | |||
| 30 | + | Committed *CheckpointRef | |||
| 31 | + | DegradedReason string | |||
| 32 | + | } | |||
| 33 | + | ||||
| 34 | + | // Equal reports canonical value equality between checkpoint references. | |||
| 35 | + | func (r *CheckpointRef) Equal(other *CheckpointRef) bool { | |||
| 36 | + | if r == nil || other == nil { | |||
| 37 | + | return r == nil && other == nil | |||
| 38 | + | } | |||
| 39 | + | return *r == *other | |||
| 40 | + | } | |||
| 41 | + | ||||
| 42 | + | // Equal reports canonical value equality, including ordered tab names and | |||
| 43 | + | // pointed-to checkpoint values rather than pointer identity. | |||
| 44 | + | func (r CatalogueRecord) Equal(other CatalogueRecord) bool { | |||
| 45 | + | return r.Name == other.Name && r.IncarnationID == other.IncarnationID && r.Cwd == other.Cwd && | |||
| 46 | + | r.CreatedAt == other.CreatedAt && r.UpdatedAt == other.UpdatedAt && r.LastUsedSeq == other.LastUsedSeq && | |||
| 47 | + | slices.Equal(r.TabNames, other.TabNames) && r.Committed.Equal(other.Committed) && | |||
| 48 | + | r.DegradedReason == other.DegradedReason | |||
| 49 | + | } | |||
| 50 | + | ||||
| 51 | + | // CatalogueMetadataUpdate changes mutable runtime metadata for one catalogue | |||
| 52 | + | // incarnation without carrying authority-owned recovery or checkpoint fields. | |||
| 53 | + | type CatalogueMetadataUpdate struct { | |||
| 54 | + | Name string | |||
| 55 | + | IncarnationID IncarnationID | |||
| 56 | + | Cwd *string | |||
| 57 | + | UpdatedAt *int64 | |||
| 58 | + | LastUsedSeq *uint64 | |||
| 59 | + | TabNames *[]string | |||
| 60 | + | } | |||
| 61 | + | ||||
| 62 | + | // MetadataUpdate returns a complete mutable-metadata update for r. Catalogue | |||
| 63 | + | // implementations retain all recovery and checkpoint fields they own. | |||
| 64 | + | func (r CatalogueRecord) MetadataUpdate() CatalogueMetadataUpdate { | |||
| 65 | + | cwd, updatedAt, lastUsedSeq := r.Cwd, r.UpdatedAt, r.LastUsedSeq | |||
| 66 | + | tabNames := append([]string(nil), r.TabNames...) | |||
| 67 | + | return CatalogueMetadataUpdate{ | |||
| 68 | + | Name: r.Name, IncarnationID: r.IncarnationID, | |||
| 69 | + | Cwd: &cwd, UpdatedAt: &updatedAt, LastUsedSeq: &lastUsedSeq, TabNames: &tabNames, | |||
| 70 | + | } | |||
| 71 | + | } | |||
| 72 | + | ||||
| 73 | + | // NewIncarnationID reads a new non-zero identity from r. | |||
| 74 | + | func NewIncarnationID(r io.Reader) (IncarnationID, error) { | |||
| 75 | + | var id IncarnationID | |||
| 76 | + | if _, err := io.ReadFull(r, id[:]); err != nil { | |||
| 77 | + | return id, err | |||
| 78 | + | } | |||
| 79 | + | if id == (IncarnationID{}) { | |||
| 80 | + | return id, errors.New("zero incarnation ID") | |||
| 81 | + | } | |||
| 82 | + | return id, nil | |||
| 83 | + | } | |||
| 84 | + | ||||
| 85 | + | func (id IncarnationID) String() string { | |||
| 86 | + | return hex.EncodeToString(id[:]) | |||
| 87 | + | } | |||
| 88 | + | ||||
| 89 | + | // MarshalText emits the canonical lowercase hexadecimal identity. | |||
| 90 | + | func (id IncarnationID) MarshalText() ([]byte, error) { | |||
| 91 | + | if id == (IncarnationID{}) { | |||
| 92 | + | return nil, errors.New("zero incarnation ID") | |||
| 93 | + | } | |||
| 94 | + | return []byte(id.String()), nil | |||
| 95 | + | } | |||
| 96 | + | ||||
| 97 | + | // UnmarshalText accepts only a canonical lowercase non-zero identity. | |||
| 98 | + | func (id *IncarnationID) UnmarshalText(text []byte) error { | |||
| 99 | + | if len(text) != 32 || strings.ToLower(string(text)) != string(text) { | |||
| 100 | + | return errors.New("invalid incarnation ID") | |||
| 101 | + | } | |||
| 102 | + | var decoded IncarnationID | |||
| 103 | + | if _, err := hex.Decode(decoded[:], text); err != nil { | |||
| 104 | + | return fmt.Errorf("invalid incarnation ID: %w", err) | |||
| 105 | + | } | |||
| 106 | + | if decoded == (IncarnationID{}) { | |||
| 107 | + | return errors.New("zero incarnation ID") | |||
| 108 | + | } | |||
| 109 | + | *id = decoded | |||
| 110 | + | return nil | |||
| 111 | + | } | |||
| 112 | + | ||||
| 113 | + | // Validate rejects catalogue states that cannot be restored deterministically. | |||
| 114 | + | func (r CatalogueRecord) Validate() error { | |||
| 115 | + | if err := ValidateSessionName(r.Name); err != nil { | |||
| 116 | + | return err | |||
| 117 | + | } | |||
| 118 | + | if r.IncarnationID == (IncarnationID{}) { | |||
| 119 | + | return errors.New("zero catalogue incarnation ID") | |||
| 120 | + | } | |||
| 121 | + | ||||
| 122 | + | if r.DegradedReason != "" && r.Committed == nil { | |||
| 123 | + | return errors.New("broken session has no committed checkpoint") | |||
| 124 | + | } | |||
| 125 | + | if r.Committed != nil { | |||
| 126 | + | if r.Committed.Generation == 0 { | |||
| 127 | + | return errors.New("zero checkpoint generation") | |||
| 128 | + | } | |||
| 129 | + | if r.Committed.ManifestDigest == ([32]byte{}) { | |||
| 130 | + | return errors.New("zero checkpoint manifest digest") | |||
| 131 | + | } | |||
| 132 | + | } | |||
| 133 | + | return nil | |||
| 134 | + | } | |||
A internal/domain/recovery_test.go +74 -0
| @@ -0,0 +1,74 @@ | |||||
| 1 | + | package domain | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "crypto/rand" | |||
| 5 | + | "testing" | |||
| 6 | + | ||||
| 7 | + | "github.com/stretchr/testify/require" | |||
| 8 | + | ) | |||
| 9 | + | ||||
| 10 | + | func TestNewIncarnationID(t *testing.T) { | |||
| 11 | + | id, err := NewIncarnationID(rand.Reader) | |||
| 12 | + | require.NoError(t, err) | |||
| 13 | + | require.NotEqual(t, IncarnationID{}, id) | |||
| 14 | + | require.Len(t, id.String(), 32) | |||
| 15 | + | } | |||
| 16 | + | ||||
| 17 | + | func TestIncarnationIDText(t *testing.T) { | |||
| 18 | + | id := IncarnationID{0: 0xab, 15: 0xcd} | |||
| 19 | + | text, err := id.MarshalText() | |||
| 20 | + | require.NoError(t, err) | |||
| 21 | + | require.Equal(t, "ab0000000000000000000000000000cd", string(text)) | |||
| 22 | + | var decoded IncarnationID | |||
| 23 | + | require.NoError(t, decoded.UnmarshalText(text)) | |||
| 24 | + | require.Equal(t, id, decoded) | |||
| 25 | + | require.Error(t, decoded.UnmarshalText([]byte("AB0000000000000000000000000000CD"))) | |||
| 26 | + | ||||
| 27 | + | _, err = (IncarnationID{}).MarshalText() | |||
| 28 | + | require.Error(t, err) | |||
| 29 | + | require.Error(t, decoded.UnmarshalText([]byte("00000000000000000000000000000000"))) | |||
| 30 | + | require.Equal(t, id, decoded, "rejected zero text must not mutate the receiver") | |||
| 31 | + | } | |||
| 32 | + | ||||
| 33 | + | func TestCatalogueRecordValidate(t *testing.T) { | |||
| 34 | + | validRef := &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}} | |||
| 35 | + | tests := []struct { | |||
| 36 | + | name string | |||
| 37 | + | record CatalogueRecord | |||
| 38 | + | wantValid bool | |||
| 39 | + | }{ | |||
| 40 | + | {name: "fresh valid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}}, wantValid: true}, | |||
| 41 | + | {name: "healthy with committed checkpoint valid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, Committed: &CheckpointRef{Generation: 7, ManifestDigest: [32]byte{1}}}, wantValid: true}, | |||
| 42 | + | {name: "invalid session name", record: CatalogueRecord{Name: "bad/name", IncarnationID: IncarnationID{1}}}, | |||
| 43 | + | {name: "zero incarnation", record: CatalogueRecord{Name: "work"}}, | |||
| 44 | + | {name: "broken with committed checkpoint valid", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, Committed: validRef, DegradedReason: "missing manifest"}, wantValid: true}, | |||
| 45 | + | {name: "broken without committed checkpoint", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, DegradedReason: "missing manifest"}}, | |||
| 46 | + | {name: "zero checkpoint generation", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, Committed: &CheckpointRef{ManifestDigest: [32]byte{1}}}}, | |||
| 47 | + | {name: "zero checkpoint digest", record: CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, Committed: &CheckpointRef{Generation: 7}}}, | |||
| 48 | + | } | |||
| 49 | + | for _, tt := range tests { | |||
| 50 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 51 | + | err := tt.record.Validate() | |||
| 52 | + | if tt.wantValid { | |||
| 53 | + | require.NoError(t, err) | |||
| 54 | + | } else { | |||
| 55 | + | require.Error(t, err) | |||
| 56 | + | } | |||
| 57 | + | }) | |||
| 58 | + | } | |||
| 59 | + | } | |||
| 60 | + | ||||
| 61 | + | func TestCatalogueRecordEqualUsesCheckpointValuesAndOrderedTabs(t *testing.T) { | |||
| 62 | + | committed := CheckpointRef{Generation: 2, ManifestDigest: [32]byte{2}} | |||
| 63 | + | left := CatalogueRecord{Name: "work", IncarnationID: IncarnationID{1}, TabNames: []string{"shell", "logs"}, Committed: &committed} | |||
| 64 | + | right := left | |||
| 65 | + | right.TabNames = append([]string(nil), left.TabNames...) | |||
| 66 | + | committedCopy := committed | |||
| 67 | + | right.Committed = &committedCopy | |||
| 68 | + | require.True(t, left.Equal(right)) | |||
| 69 | + | ||||
| 70 | + | right.TabNames[0], right.TabNames[1] = right.TabNames[1], right.TabNames[0] | |||
| 71 | + | require.False(t, left.Equal(right)) | |||
| 72 | + | require.True(t, (*CheckpointRef)(nil).Equal(nil)) | |||
| 73 | + | require.False(t, left.Committed.Equal(nil)) | |||
| 74 | + | } | |||
A internal/persist/catalogue_test.go +236 -0
| @@ -0,0 +1,236 @@ | |||||
| 1 | + | package persist | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "os" | |||
| 5 | + | "path/filepath" | |||
| 6 | + | "testing" | |||
| 7 | + | ||||
| 8 | + | "github.com/stretchr/testify/require" | |||
| 9 | + | ||||
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 11 | + | "github.com/bnema/vev/pkg/kv" | |||
| 12 | + | "github.com/bnema/vev/pkg/safedir" | |||
| 13 | + | ) | |||
| 14 | + | ||||
| 15 | + | func TestCatalogue(t *testing.T) { | |||
| 16 | + | t.Run("record-round-trip", testCatalogueRecordRoundTrip) | |||
| 17 | + | t.Run("duplicate-incarnations", testCatalogueDuplicateIncarnations) | |||
| 18 | + | t.Run("open-fails-closed", testCatalogueOpenFailsClosed) | |||
| 19 | + | t.Run("apply-rename-replace", testCatalogueApplyRenameReplace) | |||
| 20 | + | t.Run("metadata-update-preserves-authority", testCatalogueMetadataUpdatePreservesAuthority) | |||
| 21 | + | t.Run("metadata-update-is-deferred", testCatalogueMetadataUpdateIsDeferred) | |||
| 22 | + | t.Run("read-only-malformed", testCatalogueLoadReadOnlyRejectsMalformedValue) | |||
| 23 | + | t.Run("read-only-fresh-install", testCatalogueLoadReadOnlyFreshInstallHasNoSessions) | |||
| 24 | + | t.Run("read-only-ignores-stray-tmp", testCatalogueLoadReadOnlyIgnoresStrayTmp) | |||
| 25 | + | } | |||
| 26 | + | ||||
| 27 | + | func testCatalogueRecordRoundTrip(t *testing.T) { | |||
| 28 | + | ref1 := &domain.CheckpointRef{Generation: 9, ManifestDigest: [32]byte{1}} | |||
| 29 | + | tests := []domain.CatalogueRecord{ | |||
| 30 | + | {Name: "fresh", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp"}, | |||
| 31 | + | {Name: "healthy", IncarnationID: domain.IncarnationID{2}, Committed: ref1}, | |||
| 32 | + | {Name: "broken", IncarnationID: domain.IncarnationID{4}, Committed: ref1, DegradedReason: "invalid manifest"}, | |||
| 33 | + | {Name: "other-fresh", IncarnationID: domain.IncarnationID{5}}, | |||
| 34 | + | } | |||
| 35 | + | for _, record := range tests { | |||
| 36 | + | t.Run(record.Name, func(t *testing.T) { | |||
| 37 | + | encoded, err := encodeRecordValue(record) | |||
| 38 | + | require.NoError(t, err) | |||
| 39 | + | got, err := decodeRecordValue(record.Name, encoded) | |||
| 40 | + | require.NoError(t, err) | |||
| 41 | + | require.Equal(t, record, got) | |||
| 42 | + | for prefix := range len(encoded) { | |||
| 43 | + | _, err := decodeRecordValue(record.Name, encoded[:prefix]) | |||
| 44 | + | require.Error(t, err, "prefix length %d", prefix) | |||
| 45 | + | } | |||
| 46 | + | require.Error(t, func() error { _, err := decodeRecordValue(record.Name, append(encoded, 0)); return err }()) | |||
| 47 | + | }) | |||
| 48 | + | } | |||
| 49 | + | } | |||
| 50 | + | ||||
| 51 | + | func TestDecodeRecordValueRejectsMalformedCheckpointMarker(t *testing.T) { | |||
| 52 | + | encoded, err := encodeRecordValue(validRecord("work", 1)) | |||
| 53 | + | require.NoError(t, err) | |||
| 54 | + | encoded[checkpointMarkerOffset(t, encoded)] = 99 | |||
| 55 | + | ||||
| 56 | + | _, err = decodeRecordValue("work", encoded) | |||
| 57 | + | require.ErrorIs(t, err, errMalformedRecord) | |||
| 58 | + | } | |||
| 59 | + | ||||
| 60 | + | func checkpointMarkerOffset(t *testing.T, encoded []byte) int { | |||
| 61 | + | t.Helper() | |||
| 62 | + | r := valueReader{data: encoded} | |||
| 63 | + | _, ok := r.take(len(catalogueMagic) + 2 + len(domain.IncarnationID{})) | |||
| 64 | + | require.True(t, ok) | |||
| 65 | + | _, ok = r.str() | |||
| 66 | + | require.True(t, ok) | |||
| 67 | + | _, ok = r.take(8 + 8 + 8) | |||
| 68 | + | require.True(t, ok) | |||
| 69 | + | count, ok := r.u32() | |||
| 70 | + | require.True(t, ok) | |||
| 71 | + | for range count { | |||
| 72 | + | _, ok = r.str() | |||
| 73 | + | require.True(t, ok) | |||
| 74 | + | } | |||
| 75 | + | return len(encoded) - r.remaining() | |||
| 76 | + | } | |||
| 77 | + | ||||
| 78 | + | func testCatalogueDuplicateIncarnations(t *testing.T) { | |||
| 79 | + | id := domain.IncarnationID{1} | |||
| 80 | + | require.Error(t, validateUniqueIncarnations([]domain.CatalogueRecord{ | |||
| 81 | + | {Name: "one", IncarnationID: id}, | |||
| 82 | + | {Name: "two", IncarnationID: id}, | |||
| 83 | + | })) | |||
| 84 | + | } | |||
| 85 | + | ||||
| 86 | + | func testCatalogueOpenFailsClosed(t *testing.T) { | |||
| 87 | + | dir := privateDir(t) | |||
| 88 | + | _, _, err := openCurrentCatalogue(dir, false) | |||
| 89 | + | require.Error(t, err) | |||
| 90 | + | ||||
| 91 | + | require.NoError(t, safedir.EnsurePrivate(dir)) | |||
| 92 | + | raw := []byte("corrupt catalogue") | |||
| 93 | + | require.NoError(t, os.WriteFile(StorePath(dir), raw, 0o600)) | |||
| 94 | + | _, _, err = openCurrentCatalogue(dir, false) | |||
| 95 | + | require.ErrorIs(t, err, kv.ErrCorrupt) | |||
| 96 | + | after, readErr := os.ReadFile(StorePath(dir)) | |||
| 97 | + | require.NoError(t, readErr) | |||
| 98 | + | require.Equal(t, raw, after) | |||
| 99 | + | } | |||
| 100 | + | ||||
| 101 | + | func testCatalogueApplyRenameReplace(t *testing.T) { | |||
| 102 | + | dir := privateDir(t) | |||
| 103 | + | p, records, err := openCurrentCatalogue(dir, true) | |||
| 104 | + | require.NoError(t, err) | |||
| 105 | + | require.Empty(t, records) | |||
| 106 | + | defer func() { require.NoError(t, p.Close()) }() | |||
| 107 | + | ||||
| 108 | + | one := domain.CatalogueRecord{Name: "one", IncarnationID: domain.IncarnationID{1}} | |||
| 109 | + | two := domain.CatalogueRecord{Name: "two", IncarnationID: domain.IncarnationID{2}} | |||
| 110 | + | require.NoError(t, p.Apply(map[string]*domain.CatalogueRecord{"one": &one, "two": &two})) | |||
| 111 | + | renamed := one | |||
| 112 | + | renamed.Name = "renamed" | |||
| 113 | + | require.NoError(t, p.Rename("one", renamed)) | |||
| 114 | + | two.Cwd = "/next" | |||
| 115 | + | require.NoError(t, p.Replace("two", two)) | |||
| 116 | + | ||||
| 117 | + | got, err := p.LoadCatalogue() | |||
| 118 | + | require.NoError(t, err) | |||
| 119 | + | require.ElementsMatch(t, []domain.CatalogueRecord{renamed, two}, got) | |||
| 120 | + | } | |||
| 121 | + | ||||
| 122 | + | func testCatalogueMetadataUpdatePreservesAuthority(t *testing.T) { | |||
| 123 | + | dir := privateDir(t) | |||
| 124 | + | p, _, err := openCurrentCatalogue(dir, true) | |||
| 125 | + | require.NoError(t, err) | |||
| 126 | + | defer func() { require.NoError(t, p.Close()) }() | |||
| 127 | + | ||||
| 128 | + | committed := &domain.CheckpointRef{Generation: 9, ManifestDigest: [32]byte{1}} | |||
| 129 | + | original := domain.CatalogueRecord{ | |||
| 130 | + | Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/old", CreatedAt: 11, | |||
| 131 | + | UpdatedAt: 12, LastUsedSeq: 13, TabNames: []string{"old"}, | |||
| 132 | + | Committed: committed, DegradedReason: "repair pending", | |||
| 133 | + | } | |||
| 134 | + | require.NoError(t, p.Create(original)) | |||
| 135 | + | ||||
| 136 | + | next := domain.CatalogueRecord{Name: "work", IncarnationID: original.IncarnationID, Cwd: "/new", UpdatedAt: 22, LastUsedSeq: 23, TabNames: []string{"editor", "logs"}} | |||
| 137 | + | update := next.MetadataUpdate() | |||
| 138 | + | require.NoError(t, p.UpdateMetadata(update)) | |||
| 139 | + | got, ok, err := p.Record("work") | |||
| 140 | + | require.NoError(t, err) | |||
| 141 | + | require.True(t, ok) | |||
| 142 | + | expected := original | |||
| 143 | + | expected.Cwd, expected.UpdatedAt, expected.LastUsedSeq, expected.TabNames = next.Cwd, next.UpdatedAt, next.LastUsedSeq, next.TabNames | |||
| 144 | + | require.Equal(t, expected, got) | |||
| 145 | + | ||||
| 146 | + | lastUsedSeq := uint64(24) | |||
| 147 | + | require.NoError(t, p.UpdateMetadata(domain.CatalogueMetadataUpdate{ | |||
| 148 | + | Name: "work", IncarnationID: original.IncarnationID, LastUsedSeq: &lastUsedSeq, | |||
| 149 | + | })) | |||
| 150 | + | expected.LastUsedSeq = lastUsedSeq | |||
| 151 | + | partiallyUpdated, ok, err := p.Record("work") | |||
| 152 | + | require.NoError(t, err) | |||
| 153 | + | require.True(t, ok) | |||
| 154 | + | require.Equal(t, expected, partiallyUpdated, "a partial metadata update must not clear other mutable or authority-owned fields") | |||
| 155 | + | ||||
| 156 | + | stale := update | |||
| 157 | + | stale.IncarnationID = domain.IncarnationID{2} | |||
| 158 | + | require.Error(t, p.UpdateMetadata(stale)) | |||
| 159 | + | unchanged, ok, err := p.Record("work") | |||
| 160 | + | require.NoError(t, err) | |||
| 161 | + | require.True(t, ok) | |||
| 162 | + | require.Equal(t, expected, unchanged) | |||
| 163 | + | } | |||
| 164 | + | ||||
| 165 | + | func testCatalogueMetadataUpdateIsDeferred(t *testing.T) { | |||
| 166 | + | dir := privateDir(t) | |||
| 167 | + | p, _, err := openCurrentCatalogue(dir, true) | |||
| 168 | + | require.NoError(t, err) | |||
| 169 | + | defer func() { require.NoError(t, p.Close()) }() | |||
| 170 | + | ||||
| 171 | + | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/old"} | |||
| 172 | + | require.NoError(t, p.Create(record)) | |||
| 173 | + | before, err := os.ReadFile(StorePath(dir)) | |||
| 174 | + | require.NoError(t, err) | |||
| 175 | + | ||||
| 176 | + | cwd := "/buffered" | |||
| 177 | + | require.NoError(t, p.UpdateMetadata(domain.CatalogueMetadataUpdate{ | |||
| 178 | + | Name: record.Name, IncarnationID: record.IncarnationID, Cwd: &cwd, | |||
| 179 | + | })) | |||
| 180 | + | afterBuffered, err := os.ReadFile(StorePath(dir)) | |||
| 181 | + | require.NoError(t, err) | |||
| 182 | + | require.Equal(t, before, afterBuffered, "metadata must remain buffered before Sync") | |||
| 183 | + | ||||
| 184 | + | require.NoError(t, p.Sync()) | |||
| 185 | + | afterSync, err := os.ReadFile(StorePath(dir)) | |||
| 186 | + | require.NoError(t, err) | |||
| 187 | + | require.NotEqual(t, before, afterSync) | |||
| 188 | + | ||||
| 189 | + | cwd = "/next-identity" | |||
| 190 | + | require.NoError(t, p.UpdateMetadata(domain.CatalogueMetadataUpdate{ | |||
| 191 | + | Name: record.Name, IncarnationID: record.IncarnationID, Cwd: &cwd, | |||
| 192 | + | })) | |||
| 193 | + | record.Cwd = cwd | |||
| 194 | + | record.UpdatedAt = 2 | |||
| 195 | + | require.NoError(t, p.Replace(record.Name, record)) | |||
| 196 | + | afterIdentity, err := os.ReadFile(StorePath(dir)) | |||
| 197 | + | require.NoError(t, err) | |||
| 198 | + | require.NotEqual(t, afterSync, afterIdentity, "identity writes must sync before returning") | |||
| 199 | + | } | |||
| 200 | + | ||||
| 201 | + | func testCatalogueLoadReadOnlyRejectsMalformedValue(t *testing.T) { | |||
| 202 | + | dir := privateDir(t) | |||
| 203 | + | store, err := kv.Open(filepath.Join(dir, filename)) | |||
| 204 | + | require.NoError(t, err) | |||
| 205 | + | require.NoError(t, store.Set([]byte("work"), []byte("malformed"))) | |||
| 206 | + | require.NoError(t, store.Close()) | |||
| 207 | + | _, err = LoadCatalogueReadOnly(dir) | |||
| 208 | + | require.Error(t, err) | |||
| 209 | + | } | |||
| 210 | + | ||||
| 211 | + | // testCatalogueLoadReadOnlyFreshInstallHasNoSessions covers `vev ls` on a | |||
| 212 | + | // machine with no daemon and no catalogue. This must yield an empty catalogue, | |||
| 213 | + | // not an error. | |||
| 214 | + | func testCatalogueLoadReadOnlyFreshInstallHasNoSessions(t *testing.T) { | |||
| 215 | + | dir := privateDir(t) | |||
| 216 | + | records, err := LoadCatalogueReadOnly(dir) | |||
| 217 | + | require.NoError(t, err) | |||
| 218 | + | require.Empty(t, records) | |||
| 219 | + | } | |||
| 220 | + | ||||
| 221 | + | func testCatalogueLoadReadOnlyIgnoresStrayTmp(t *testing.T) { | |||
| 222 | + | dir := privateDir(t) | |||
| 223 | + | p, _, err := openCurrentCatalogue(dir, true) | |||
| 224 | + | require.NoError(t, err) | |||
| 225 | + | want := validRecord("work", 1) | |||
| 226 | + | require.NoError(t, p.Create(want)) | |||
| 227 | + | require.NoError(t, p.Close()) | |||
| 228 | + | require.NoError(t, os.WriteFile(StorePath(dir)+".tmp", []byte("partial rewrite"), 0o600)) | |||
| 229 | + | ||||
| 230 | + | records, err := LoadCatalogueReadOnly(dir) | |||
| 231 | + | require.NoError(t, err) | |||
| 232 | + | require.Equal(t, []domain.CatalogueRecord{want}, records) | |||
| 233 | + | tmp, err := os.ReadFile(StorePath(dir) + ".tmp") | |||
| 234 | + | require.NoError(t, err) | |||
| 235 | + | require.Equal(t, []byte("partial rewrite"), tmp) | |||
| 236 | + | } | |||
M internal/persist/persist.go +393 -78
| @@ -1,174 +1,489 @@ | |||||
| 1 | package persist | 1 | package persist | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | - | "log/slog" | 4 | + | "errors" |
| 5 | + | "fmt" | |||
| 6 | + | "os" | |||
| 5 | "path/filepath" | 7 | "path/filepath" | ||
| 6 | "sort" | 8 | "sort" | ||
| 7 | "sync" | 9 | "sync" | ||
| 8 | 10 | ||||
| 11 | + | "github.com/bnema/vev/internal/domain" | |||
| 9 | "github.com/bnema/vev/internal/ports" | 12 | "github.com/bnema/vev/internal/ports" | ||
| 10 | "github.com/bnema/vev/pkg/kv" | 13 | "github.com/bnema/vev/pkg/kv" | ||
| 11 | ) | 14 | ) | ||
| 12 | 15 | ||||
| 13 | const filename = "sessions.kv" | 16 | const filename = "sessions.kv" | ||
| 14 | 17 | ||||
| 15 | - | // StorePath returns the canonical path for the persisted session metadata store. | 18 | + | var errPersistenceUnavailable = errors.New("persist: catalogue unavailable") |
| 16 | - | func StorePath(dir string) string { | 19 | + | |
| 17 | - | return filepath.Join(dir, filename) | 20 | + | // ErrCatalogueDurability reports a failed identity write whose durability is |
| 21 | + | // ambiguous. A Persister remains fenced after this error so rejected state | |||
| 22 | + | // cannot be made durable by a later flush. | |||
| 23 | + | var ErrCatalogueDurability = errors.New("persist: catalogue durability rejected") | |||
| 24 | + | ||||
| 25 | + | // ErrCatalogueUnreadable reports durable state that exists but cannot be | |||
| 26 | + | // decoded. vev never repairs or erases it; the operator resets explicitly. | |||
| 27 | + | var ErrCatalogueUnreadable = errors.New("persist: catalogue unreadable") | |||
| 28 | + | ||||
| 29 | + | // OpenResult is the outcome of opening durable session state at startup. | |||
| 30 | + | type OpenResult struct { | |||
| 31 | + | Catalogue *Persister | |||
| 32 | + | Records []domain.CatalogueRecord | |||
| 33 | + | NewInstall bool | |||
| 18 | } | 34 | } | ||
| 19 | 35 | ||||
| 20 | - | // Persister stores named session metadata. A nil Store makes all mutating | 36 | + | func StorePath(dir string) string { return filepath.Join(dir, filename) } |
| 21 | - | // operations no-ops, so callers can keep persistence optional. | 37 | + | |
| 22 | type Persister struct { | 38 | type Persister struct { | ||
| 23 | store ports.Store | 39 | store ports.Store | ||
| 24 | mu sync.Mutex | 40 | mu sync.Mutex | ||
| 41 | + | ||||
| 42 | + | incarnationOwners map[domain.IncarnationID]string | |||
| 43 | + | nameIncarnations map[string]domain.IncarnationID | |||
| 44 | + | incarnationIndex bool | |||
| 45 | + | terminalErr error | |||
| 25 | } | 46 | } | ||
| 26 | 47 | ||||
| 27 | - | // Open opens the session persister under dir. | 48 | + | // KVStore adapts the reusable whole-file store to the persistence port. |
| 28 | - | func Open(dir string) (*Persister, error) { | 49 | + | type KVStore struct{ store *kv.Store } |
| 29 | - | store, err := kv.Open(StorePath(dir)) | 50 | + | |
| 51 | + | func OpenStore(path string) (*KVStore, error) { | |||
| 52 | + | store, err := kv.Open(path) | |||
| 30 | if err != nil { | 53 | if err != nil { | ||
| 31 | return nil, err | 54 | return nil, err | ||
| 32 | } | 55 | } | ||
| 33 | - | return New(store), nil | 56 | + | return &KVStore{store: store}, nil |
| 57 | + | } | |||
| 58 | + | ||||
| 59 | + | func (s *KVStore) Get(key []byte) ([]byte, bool) { return s.store.Get(key) } | |||
| 60 | + | func (s *KVStore) Set(key, value []byte) error { return s.store.Set(key, value) } | |||
| 61 | + | func (s *KVStore) Delete(key []byte) error { return s.store.Delete(key) } | |||
| 62 | + | func (s *KVStore) Range(fn func(key, value []byte) bool) { | |||
| 63 | + | s.store.Range(fn) | |||
| 64 | + | } | |||
| 65 | + | func (s *KVStore) Sync() error { return s.store.Sync() } | |||
| 66 | + | func (s *KVStore) Close() error { return s.store.Close() } | |||
| 67 | + | ||||
| 68 | + | func (s *KVStore) CloseWithoutSync() error { return s.store.CloseWithoutSync() } | |||
| 69 | + | ||||
| 70 | + | // Open opens an existing strict catalogue and never creates an unproven empty one. | |||
| 71 | + | func Open(dir string) (*Persister, error) { | |||
| 72 | + | p, _, err := openCurrentCatalogue(dir, false) | |||
| 73 | + | return p, err | |||
| 74 | + | } | |||
| 75 | + | ||||
| 76 | + | // OpenOrCreate opens the catalogue, or creates a proven-empty one when no | |||
| 77 | + | // durable state exists. Any state that exists but does not decode is returned | |||
| 78 | + | // as ErrCatalogueUnreadable and left untouched on disk. | |||
| 79 | + | func OpenOrCreate(dir string) (OpenResult, error) { | |||
| 80 | + | path := StorePath(dir) | |||
| 81 | + | _, statErr := os.Stat(path) | |||
| 82 | + | existed := statErr == nil | |||
| 83 | + | if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { | |||
| 84 | + | return OpenResult{}, fmt.Errorf("%w: %s: %w", ErrCatalogueUnreadable, path, statErr) | |||
| 85 | + | } | |||
| 86 | + | catalogue, records, err := openCurrentCatalogue(dir, true) | |||
| 87 | + | if err != nil { | |||
| 88 | + | return OpenResult{}, fmt.Errorf("%w: %s: %w", ErrCatalogueUnreadable, path, err) | |||
| 89 | + | } | |||
| 90 | + | return OpenResult{Catalogue: catalogue, Records: records, NewInstall: !existed}, nil | |||
| 34 | } | 91 | } | ||
| 35 | 92 | ||||
| 36 | - | // New wraps store in a Persister. Passing nil creates a no-op persister. | 93 | + | func New(store ports.Store) *Persister { return &Persister{store: store} } |
| 37 | - | func New(store ports.Store) *Persister { | 94 | + | |
| 38 | - | return &Persister{store: store} | 95 | + | func cataloguePresent(path string) (bool, error) { |
| 96 | + | if _, err := os.Stat(path); err == nil { | |||
| 97 | + | return true, nil | |||
| 98 | + | } else if !errors.Is(err, os.ErrNotExist) { | |||
| 99 | + | return false, err | |||
| 100 | + | } | |||
| 101 | + | return false, nil | |||
| 39 | } | 102 | } | ||
| 40 | 103 | ||||
| 41 | - | // LoadReadOnly replays the session store under dir without mutating it. | 104 | + | func openCurrentCatalogue(dir string, createProvenEmpty bool) (*Persister, []domain.CatalogueRecord, error) { |
| 42 | - | func LoadReadOnly(dir string) ([]Record, error) { | 105 | + | path := StorePath(dir) |
| 43 | - | data, err := kv.Replay(StorePath(dir)) | 106 | + | present, err := cataloguePresent(path) |
| 44 | if err != nil { | 107 | if err != nil { | ||
| 108 | + | return nil, nil, err | |||
| 109 | + | } | |||
| 110 | + | if !present && !createProvenEmpty { | |||
| 111 | + | return nil, nil, fmt.Errorf("%w: no catalogue candidates", errPersistenceUnavailable) | |||
| 112 | + | } | |||
| 113 | + | store, err := OpenStore(path) | |||
| 114 | + | if err != nil { | |||
| 115 | + | return nil, nil, err | |||
| 116 | + | } | |||
| 117 | + | p := New(store) | |||
| 118 | + | records, err := p.LoadCatalogue() | |||
| 119 | + | if err != nil { | |||
| 120 | + | return nil, nil, errors.Join(err, p.Close()) | |||
| 121 | + | } | |||
| 122 | + | return p, records, nil | |||
| 123 | + | } | |||
| 124 | + | ||||
| 125 | + | func LoadReadOnly(dir string) ([]domain.CatalogueRecord, error) { return LoadCatalogueReadOnly(dir) } | |||
| 126 | + | ||||
| 127 | + | // LoadCatalogueReadOnly loads the catalogue without retaining a store handle. | |||
| 128 | + | // A fresh install with no main catalogue is not an error. Stray temporary | |||
| 129 | + | // files are ignored, while malformed main files fail closed. | |||
| 130 | + | func LoadCatalogueReadOnly(dir string) ([]domain.CatalogueRecord, error) { | |||
| 131 | + | p, records, err := openCurrentCatalogue(dir, false) | |||
| 132 | + | if err != nil { | |||
| 133 | + | if errors.Is(err, errPersistenceUnavailable) { | |||
| 134 | + | return []domain.CatalogueRecord{}, nil | |||
| 135 | + | } | |||
| 136 | + | return nil, fmt.Errorf("%w: %s: %w", ErrCatalogueUnreadable, StorePath(dir), err) | |||
| 137 | + | } | |||
| 138 | + | if err := p.Close(); err != nil { | |||
| 45 | return nil, err | 139 | return nil, err | ||
| 46 | } | 140 | } | ||
| 47 | - | return decodeAll(data) | 141 | + | return records, nil |
| 48 | } | 142 | } | ||
| 49 | 143 | ||||
| 50 | - | // Save persists a structural session record and syncs it to disk. | 144 | + | func (p *Persister) Save(record domain.CatalogueRecord) error { return p.Replace(record.Name, record) } |
| 51 | - | func (p *Persister) Save(r Record) error { | 145 | + | func (p *Persister) Create(record domain.CatalogueRecord) error { |
| 52 | - | if p == nil || p.store == nil { | 146 | + | if p == nil { |
| 53 | return nil | 147 | return nil | ||
| 54 | } | 148 | } | ||
| 149 | + | if p.store == nil { | |||
| 150 | + | return errPersistenceUnavailable | |||
| 151 | + | } | |||
| 55 | p.mu.Lock() | 152 | p.mu.Lock() | ||
| 56 | defer p.mu.Unlock() | 153 | defer p.mu.Unlock() | ||
| 57 | - | value, err := encodeRecordValue(r) | 154 | + | if err := p.terminalLocked(); err != nil { |
| 58 | - | if err != nil { | |||
| 59 | return err | 155 | return err | ||
| 60 | } | 156 | } | ||
| 61 | - | if err := p.store.Set([]byte(r.Name), value); err != nil { | 157 | + | if _, exists := p.store.Get([]byte(record.Name)); exists { |
| 62 | - | return err | 158 | + | return errors.New("persist: session already exists") |
| 63 | } | 159 | } | ||
| 64 | - | return p.store.Sync() | 160 | + | return p.applyLocked(map[string]*domain.CatalogueRecord{record.Name: &record}, true) |
| 65 | } | 161 | } | ||
| 66 | 162 | ||||
| 67 | - | // Touch updates hot-path session cwd metadata without forcing an fsync. | 163 | + | func (p *Persister) Records() ([]domain.CatalogueRecord, error) { return p.LoadCatalogue() } |
| 68 | - | func (p *Persister) Touch(name, cwd string, at int64) error { | 164 | + | func (p *Persister) Record(name string) (domain.CatalogueRecord, bool, error) { |
| 69 | - | if p == nil || p.store == nil { | 165 | + | if p == nil { |
| 166 | + | return domain.CatalogueRecord{}, false, nil | |||
| 167 | + | } | |||
| 168 | + | if p.store == nil { | |||
| 169 | + | return domain.CatalogueRecord{}, false, errPersistenceUnavailable | |||
| 170 | + | } | |||
| 171 | + | p.mu.Lock() | |||
| 172 | + | defer p.mu.Unlock() | |||
| 173 | + | if err := p.terminalLocked(); err != nil { | |||
| 174 | + | return domain.CatalogueRecord{}, false, err | |||
| 175 | + | } | |||
| 176 | + | value, ok := p.store.Get([]byte(name)) | |||
| 177 | + | if !ok { | |||
| 178 | + | return domain.CatalogueRecord{}, false, nil | |||
| 179 | + | } | |||
| 180 | + | record, err := decodeRecordValue(name, value) | |||
| 181 | + | if err != nil { | |||
| 182 | + | return domain.CatalogueRecord{}, false, err | |||
| 183 | + | } | |||
| 184 | + | return record, true, nil | |||
| 185 | + | } | |||
| 186 | + | func (p *Persister) Apply(records map[string]*domain.CatalogueRecord) error { | |||
| 187 | + | if p == nil { | |||
| 70 | return nil | 188 | return nil | ||
| 71 | } | 189 | } | ||
| 72 | - | if name == "" { | 190 | + | if p.store == nil { |
| 73 | - | return errEmptyName | 191 | + | return errPersistenceUnavailable |
| 74 | } | 192 | } | ||
| 75 | p.mu.Lock() | 193 | p.mu.Lock() | ||
| 76 | defer p.mu.Unlock() | 194 | defer p.mu.Unlock() | ||
| 195 | + | if err := p.terminalLocked(); err != nil { | |||
| 196 | + | return err | |||
| 197 | + | } | |||
| 198 | + | return p.applyLocked(records, true) | |||
| 199 | + | } | |||
| 200 | + | func (p *Persister) applyLocked(records map[string]*domain.CatalogueRecord, durable bool) error { | |||
| 201 | + | if len(records) == 0 { | |||
| 202 | + | return errors.New("persist: empty catalogue batch") | |||
| 203 | + | } | |||
| 204 | + | if err := p.ensureIncarnationIndexLocked(); err != nil { | |||
| 205 | + | return err | |||
| 206 | + | } | |||
| 207 | + | type mutation struct { | |||
| 208 | + | key []byte | |||
| 209 | + | value []byte | |||
| 210 | + | delete bool | |||
| 211 | + | } | |||
| 212 | + | changes := make([]mutation, 0, len(records)) | |||
| 213 | + | keys := make([]string, 0, len(records)) | |||
| 214 | + | for name := range records { | |||
| 215 | + | keys = append(keys, name) | |||
| 216 | + | } | |||
| 217 | + | sort.Strings(keys) | |||
| 218 | + | for _, name := range keys { | |||
| 219 | + | record := records[name] | |||
| 220 | + | if err := domain.ValidateSessionName(name); err != nil { | |||
| 221 | + | return err | |||
| 222 | + | } | |||
| 223 | + | if record == nil { | |||
| 224 | + | changes = append(changes, mutation{key: []byte(name), delete: true}) | |||
| 225 | + | continue | |||
| 226 | + | } | |||
| 227 | + | if record.Name != name { | |||
| 228 | + | return errors.New("persist: catalogue key/name mismatch") | |||
| 229 | + | } | |||
| 230 | + | value, err := encodeRecordValue(*record) | |||
| 231 | + | if err != nil { | |||
| 232 | + | return err | |||
| 233 | + | } | |||
| 234 | + | changes = append(changes, mutation{key: []byte(name), value: value}) | |||
| 235 | + | } | |||
| 236 | + | incoming := make(map[domain.IncarnationID]string, len(records)) | |||
| 237 | + | for name, record := range records { | |||
| 238 | + | if record == nil { | |||
| 239 | + | continue | |||
| 240 | + | } | |||
| 241 | + | if previous, ok := incoming[record.IncarnationID]; ok && previous != name { | |||
| 242 | + | return fmt.Errorf("persist: duplicate incarnation for %q and %q", previous, name) | |||
| 243 | + | } | |||
| 244 | + | incoming[record.IncarnationID] = name | |||
| 245 | + | if owner, ok := p.incarnationOwners[record.IncarnationID]; ok && owner != name { | |||
| 246 | + | ownerNext, ownerTouched := records[owner] | |||
| 247 | + | if !ownerTouched || (ownerNext != nil && ownerNext.IncarnationID == record.IncarnationID) { | |||
| 248 | + | return fmt.Errorf("persist: duplicate incarnation for %q and %q", owner, name) | |||
| 249 | + | } | |||
| 250 | + | } | |||
| 251 | + | } | |||
| 252 | + | previous := make(map[string]storedValue, len(changes)) | |||
| 253 | + | for _, change := range changes { | |||
| 254 | + | value, ok := p.store.Get(change.key) | |||
| 255 | + | previous[string(change.key)] = storedValue{value: value, exists: ok} | |||
| 256 | + | } | |||
| 257 | + | for _, change := range changes { | |||
| 258 | + | var err error | |||
| 259 | + | if change.delete { | |||
| 260 | + | err = p.store.Delete(change.key) | |||
| 261 | + | } else { | |||
| 262 | + | err = p.store.Set(change.key, change.value) | |||
| 263 | + | } | |||
| 264 | + | if err != nil { | |||
| 265 | + | return p.fenceLocked(errors.Join(err, p.restoreLocked(previous))) | |||
| 266 | + | } | |||
| 267 | + | } | |||
| 268 | + | if durable { | |||
| 269 | + | if err := p.store.Sync(); err != nil { | |||
| 270 | + | return p.fenceLocked(errors.Join(err, p.restoreLocked(previous))) | |||
| 271 | + | } | |||
| 272 | + | } | |||
| 273 | + | for name := range records { | |||
| 274 | + | if id, ok := p.nameIncarnations[name]; ok { | |||
| 275 | + | delete(p.incarnationOwners, id) | |||
| 276 | + | delete(p.nameIncarnations, name) | |||
| 277 | + | } | |||
| 278 | + | } | |||
| 279 | + | for name, record := range records { | |||
| 280 | + | if record != nil { | |||
| 281 | + | p.incarnationOwners[record.IncarnationID] = name | |||
| 282 | + | p.nameIncarnations[name] = record.IncarnationID | |||
| 283 | + | } | |||
| 284 | + | } | |||
| 285 | + | return nil | |||
| 286 | + | } | |||
| 77 | 287 | ||||
| 78 | - | createdAt := at | 288 | + | type storedValue struct { |
| 79 | - | lastUsedSeq := uint64(0) | 289 | + | value []byte |
| 80 | - | var tabNames []string | 290 | + | exists bool |
| 81 | - | if v, ok := p.store.Get([]byte(name)); ok { | 291 | + | } |
| 82 | - | if r, err := decodeRecordValue(name, v); err == nil { | 292 | + | |
| 83 | - | createdAt = r.CreatedAt | 293 | + | // restoreLocked restores only keys affected by the rejected identity mutation. |
| 84 | - | lastUsedSeq = r.LastUsedSeq | 294 | + | // It intentionally does not Sync: the Persister is fenced immediately after. |
| 85 | - | tabNames = r.TabNames | 295 | + | func (p *Persister) restoreLocked(previous map[string]storedValue) error { |
| 296 | + | var err error | |||
| 297 | + | for key, old := range previous { | |||
| 298 | + | if old.exists { | |||
| 299 | + | err = errors.Join(err, p.store.Set([]byte(key), old.value)) | |||
| 300 | + | } else { | |||
| 301 | + | err = errors.Join(err, p.store.Delete([]byte(key))) | |||
| 86 | } | 302 | } | ||
| 87 | } | 303 | } | ||
| 304 | + | return err | |||
| 305 | + | } | |||
| 88 | 306 | ||||
| 89 | - | value, err := encodeRecordValue(Record{Name: name, Cwd: cwd, CreatedAt: createdAt, UpdatedAt: at, LastUsedSeq: lastUsedSeq, TabNames: tabNames}) | 307 | + | func (p *Persister) fenceLocked(cause error) error { |
| 90 | - | if err != nil { | 308 | + | if p.terminalErr == nil { |
| 309 | + | p.terminalErr = fmt.Errorf("%w: %w", ErrCatalogueDurability, cause) | |||
| 310 | + | } | |||
| 311 | + | p.incarnationIndex = false | |||
| 312 | + | return p.terminalErr | |||
| 313 | + | } | |||
| 314 | + | ||||
| 315 | + | func (p *Persister) terminalLocked() error { return p.terminalErr } | |||
| 316 | + | ||||
| 317 | + | func (p *Persister) ensureIncarnationIndexLocked() error { | |||
| 318 | + | if err := p.terminalLocked(); err != nil { | |||
| 91 | return err | 319 | return err | ||
| 92 | } | 320 | } | ||
| 93 | - | return p.store.Set([]byte(name), value) | 321 | + | if p.incarnationIndex { |
| 322 | + | return nil | |||
| 323 | + | } | |||
| 324 | + | owners := make(map[domain.IncarnationID]string) | |||
| 325 | + | names := make(map[string]domain.IncarnationID) | |||
| 326 | + | var indexErr error | |||
| 327 | + | p.store.Range(func(key, value []byte) bool { | |||
| 328 | + | name := string(key) | |||
| 329 | + | record, err := decodeRecordValue(name, value) | |||
| 330 | + | if err != nil { | |||
| 331 | + | indexErr = err | |||
| 332 | + | return false | |||
| 333 | + | } | |||
| 334 | + | if previous, ok := owners[record.IncarnationID]; ok { | |||
| 335 | + | indexErr = fmt.Errorf("persist: duplicate incarnation for %q and %q", previous, name) | |||
| 336 | + | return false | |||
| 337 | + | } | |||
| 338 | + | owners[record.IncarnationID] = name | |||
| 339 | + | names[name] = record.IncarnationID | |||
| 340 | + | return true | |||
| 341 | + | }) | |||
| 342 | + | if indexErr != nil { | |||
| 343 | + | return indexErr | |||
| 344 | + | } | |||
| 345 | + | p.incarnationOwners = owners | |||
| 346 | + | p.nameIncarnations = names | |||
| 347 | + | p.incarnationIndex = true | |||
| 348 | + | return nil | |||
| 349 | + | } | |||
| 350 | + | func (p *Persister) Rename(oldName string, next domain.CatalogueRecord) error { | |||
| 351 | + | if p == nil { | |||
| 352 | + | return nil | |||
| 353 | + | } | |||
| 354 | + | if p.store == nil { | |||
| 355 | + | return errPersistenceUnavailable | |||
| 356 | + | } | |||
| 357 | + | if oldName == next.Name { | |||
| 358 | + | return errors.New("persist: rename requires distinct names") | |||
| 359 | + | } | |||
| 360 | + | return p.Apply(map[string]*domain.CatalogueRecord{oldName: nil, next.Name: &next}) | |||
| 361 | + | } | |||
| 362 | + | func (p *Persister) Replace(name string, next domain.CatalogueRecord) error { | |||
| 363 | + | return p.Apply(map[string]*domain.CatalogueRecord{name: &next}) | |||
| 94 | } | 364 | } | ||
| 95 | 365 | ||||
| 96 | - | // TouchMRU updates hot-path session recency without forcing an fsync. | 366 | + | // UpdateMetadata buffers ordinary session metadata on the existing authoritative |
| 97 | - | func (p *Persister) TouchMRU(name string, lastUsedSeq uint64) error { | 367 | + | // incarnation. Recovery and transaction fields are retained from the stored |
| 98 | - | if p == nil || p.store == nil { | 368 | + | // record rather than accepted from a potentially stale runtime copy. Sync or the |
| 369 | + | // next identity operation makes the update durable. | |||
| 370 | + | func (p *Persister) UpdateMetadata(update domain.CatalogueMetadataUpdate) error { | |||
| 371 | + | if p == nil { | |||
| 99 | return nil | 372 | return nil | ||
| 100 | } | 373 | } | ||
| 101 | - | if name == "" { | 374 | + | if p.store == nil { |
| 102 | - | return errEmptyName | 375 | + | return errPersistenceUnavailable |
| 103 | } | 376 | } | ||
| 104 | p.mu.Lock() | 377 | p.mu.Lock() | ||
| 105 | defer p.mu.Unlock() | 378 | defer p.mu.Unlock() | ||
| 106 | - | 379 | + | if err := p.terminalLocked(); err != nil { | |
| 107 | - | r := Record{Name: name, LastUsedSeq: lastUsedSeq} | 380 | + | return err |
| 108 | - | if v, ok := p.store.Get([]byte(name)); ok { | |||
| 109 | - | if decoded, err := decodeRecordValue(name, v); err == nil { | |||
| 110 | - | r = decoded | |||
| 111 | - | r.LastUsedSeq = lastUsedSeq | |||
| 112 | - | } | |||
| 113 | } | 381 | } | ||
| 114 | - | value, err := encodeRecordValue(r) | 382 | + | value, ok := p.store.Get([]byte(update.Name)) |
| 383 | + | if !ok { | |||
| 384 | + | return errors.New("persist: session not found") | |||
| 385 | + | } | |||
| 386 | + | current, err := decodeRecordValue(update.Name, value) | |||
| 115 | if err != nil { | 387 | if err != nil { | ||
| 116 | return err | 388 | return err | ||
| 117 | } | 389 | } | ||
| 118 | - | return p.store.Set([]byte(name), value) | 390 | + | if update.IncarnationID == (domain.IncarnationID{}) || current.IncarnationID != update.IncarnationID { |
| 391 | + | return errors.New("persist: session incarnation changed") | |||
| 392 | + | } | |||
| 393 | + | if update.Cwd != nil { | |||
| 394 | + | current.Cwd = *update.Cwd | |||
| 395 | + | } | |||
| 396 | + | if update.UpdatedAt != nil { | |||
| 397 | + | current.UpdatedAt = *update.UpdatedAt | |||
| 398 | + | } | |||
| 399 | + | if update.LastUsedSeq != nil { | |||
| 400 | + | current.LastUsedSeq = *update.LastUsedSeq | |||
| 401 | + | } | |||
| 402 | + | if update.TabNames != nil { | |||
| 403 | + | current.TabNames = append([]string(nil), (*update.TabNames)...) | |||
| 404 | + | } | |||
| 405 | + | return p.applyLocked(map[string]*domain.CatalogueRecord{current.Name: ¤t}, false) | |||
| 119 | } | 406 | } | ||
| 120 | 407 | ||||
| 121 | - | // Delete removes a persisted session record and syncs the structural change. | 408 | + | // Sync makes every buffered catalogue mutation durable. |
| 122 | - | func (p *Persister) Delete(name string) error { | 409 | + | func (p *Persister) Sync() error { |
| 123 | - | if p == nil || p.store == nil { | 410 | + | if p == nil { |
| 124 | return nil | 411 | return nil | ||
| 125 | } | 412 | } | ||
| 126 | - | if name == "" { | 413 | + | if p.store == nil { |
| 127 | - | return errEmptyName | 414 | + | return errPersistenceUnavailable |
| 128 | } | 415 | } | ||
| 129 | p.mu.Lock() | 416 | p.mu.Lock() | ||
| 130 | defer p.mu.Unlock() | 417 | defer p.mu.Unlock() | ||
| 131 | - | if err := p.store.Delete([]byte(name)); err != nil { | 418 | + | if err := p.terminalLocked(); err != nil { |
| 132 | return err | 419 | return err | ||
| 133 | } | 420 | } | ||
| 134 | - | return p.store.Sync() | 421 | + | if err := p.store.Sync(); err != nil { |
| 422 | + | return p.fenceLocked(err) | |||
| 423 | + | } | |||
| 424 | + | return nil | |||
| 135 | } | 425 | } | ||
| 136 | 426 | ||||
| 137 | - | // LoadAll returns all persisted session records sorted by name. | 427 | + | func (p *Persister) Delete(name string) error { |
| 138 | - | func (p *Persister) LoadAll() ([]Record, error) { | 428 | + | return p.Apply(map[string]*domain.CatalogueRecord{name: nil}) |
| 139 | - | if p == nil || p.store == nil { | 429 | + | } |
| 140 | - | return nil, nil | 430 | + | func (p *Persister) LoadAll() ([]domain.CatalogueRecord, error) { return p.LoadCatalogue() } |
| 431 | + | func (p *Persister) LoadCatalogue() ([]domain.CatalogueRecord, error) { | |||
| 432 | + | if p == nil { | |||
| 433 | + | return []domain.CatalogueRecord{}, nil | |||
| 434 | + | } | |||
| 435 | + | if p.store == nil { | |||
| 436 | + | return nil, errPersistenceUnavailable | |||
| 141 | } | 437 | } | ||
| 142 | p.mu.Lock() | 438 | p.mu.Lock() | ||
| 143 | defer p.mu.Unlock() | 439 | defer p.mu.Unlock() | ||
| 440 | + | if err := p.terminalLocked(); err != nil { | |||
| 441 | + | return nil, err | |||
| 442 | + | } | |||
| 144 | data := make(map[string][]byte) | 443 | data := make(map[string][]byte) | ||
| 145 | - | p.store.Range(func(k, v []byte) bool { | 444 | + | p.store.Range(func(key, value []byte) bool { data[string(key)] = append([]byte(nil), value...); return true }) |
| 146 | - | data[string(k)] = append([]byte(nil), v...) | |||
| 147 | - | return true | |||
| 148 | - | }) | |||
| 149 | return decodeAll(data) | 445 | return decodeAll(data) | ||
| 150 | } | 446 | } | ||
| 151 | - | ||||
| 152 | - | // Close closes the underlying store. | |||
| 153 | func (p *Persister) Close() error { | 447 | func (p *Persister) Close() error { | ||
| 154 | if p == nil || p.store == nil { | 448 | if p == nil || p.store == nil { | ||
| 155 | return nil | 449 | return nil | ||
| 156 | } | 450 | } | ||
| 157 | p.mu.Lock() | 451 | p.mu.Lock() | ||
| 158 | defer p.mu.Unlock() | 452 | defer p.mu.Unlock() | ||
| 453 | + | if err := p.terminalLocked(); err != nil { | |||
| 454 | + | if closeErr := p.store.CloseWithoutSync(); closeErr != nil { | |||
| 455 | + | return errors.Join(err, closeErr) | |||
| 456 | + | } | |||
| 457 | + | return err | |||
| 458 | + | } | |||
| 159 | return p.store.Close() | 459 | return p.store.Close() | ||
| 160 | } | 460 | } | ||
| 161 | 461 | ||||
| 162 | - | func decodeAll(data map[string][]byte) ([]Record, error) { | 462 | + | func decodeAll(data map[string][]byte) ([]domain.CatalogueRecord, error) { |
| 163 | - | records := make([]Record, 0, len(data)) | 463 | + | records := make([]domain.CatalogueRecord, 0, len(data)) |
| 164 | for name, value := range data { | 464 | for name, value := range data { | ||
| 165 | - | r, err := decodeRecordValue(name, value) | 465 | + | record, err := decodeRecordValue(name, value) |
| 166 | if err != nil { | 466 | if err != nil { | ||
| 167 | - | slog.Warn("skipping malformed persisted session", "session", name, "err", err) | 467 | + | return nil, fmt.Errorf("persist: decode catalogue key %q: %w", name, err) |
| 168 | - | continue | |||
| 169 | } | 468 | } | ||
| 170 | - | records = append(records, r) | 469 | + | records = append(records, record) |
| 171 | } | 470 | } | ||
| 172 | sort.Slice(records, func(i, j int) bool { return records[i].Name < records[j].Name }) | 471 | sort.Slice(records, func(i, j int) bool { return records[i].Name < records[j].Name }) | ||
| 472 | + | if err := validateUniqueIncarnations(records); err != nil { | |||
| 473 | + | return nil, err | |||
| 474 | + | } | |||
| 173 | return records, nil | 475 | return records, nil | ||
| 174 | } | 476 | } | ||
| 477 | + | func validateUniqueIncarnations(records []domain.CatalogueRecord) error { | |||
| 478 | + | seen := make(map[domain.IncarnationID]string, len(records)) | |||
| 479 | + | for _, record := range records { | |||
| 480 | + | if err := record.Validate(); err != nil { | |||
| 481 | + | return err | |||
| 482 | + | } | |||
| 483 | + | if previous, ok := seen[record.IncarnationID]; ok { | |||
| 484 | + | return fmt.Errorf("persist: duplicate incarnation for %q and %q", previous, record.Name) | |||
| 485 | + | } | |||
| 486 | + | seen[record.IncarnationID] = record.Name | |||
| 487 | + | } | |||
| 488 | + | return nil | |||
| 489 | + | } | |||
M internal/persist/persist_test.go +412 -234
| @@ -2,301 +2,479 @@ package persist | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "errors" | 4 | "errors" | ||
| 5 | + | "os" | |||
| 5 | "path/filepath" | 6 | "path/filepath" | ||
| 6 | - | "sync" | |||
| 7 | "testing" | 7 | "testing" | ||
| 8 | - | "time" | |||
| 9 | 8 | ||||
| 10 | "github.com/stretchr/testify/mock" | 9 | "github.com/stretchr/testify/mock" | ||
| 11 | "github.com/stretchr/testify/require" | 10 | "github.com/stretchr/testify/require" | ||
| 12 | 11 | ||||
| 12 | + | "github.com/bnema/vev/internal/domain" | |||
| 13 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | 13 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | ||
| 14 | + | "github.com/bnema/vev/pkg/safedir" | |||
| 14 | ) | 15 | ) | ||
| 15 | 16 | ||||
| 16 | - | func privateDir(t *testing.T) string { | 17 | + | func privateDir(t *testing.T) string { t.Helper(); return filepath.Join(t.TempDir(), "vev") } |
| 17 | - | t.Helper() | 18 | + | func validRecord(name string, id byte) domain.CatalogueRecord { |
| 18 | - | return filepath.Join(t.TempDir(), "vev") | 19 | + | return domain.CatalogueRecord{Name: name, IncarnationID: domain.IncarnationID{id}} |
| 19 | } | 20 | } | ||
| 20 | 21 | ||||
| 21 | - | func TestRecordCodecRoundTrip(t *testing.T) { | 22 | + | func TestOpenOrCreate(t *testing.T) { |
| 22 | - | r := Record{Name: "work", Cwd: "/tmp/project", CreatedAt: 11, UpdatedAt: 22, LastUsedSeq: 33, TabNames: []string{"shell", "logs"}} | 23 | + | t.Parallel() |
| 23 | - | value, err := encodeRecordValue(r) | 24 | + | tests := []struct { |
| 24 | - | require.NoError(t, err) | 25 | + | name string |
| 25 | - | 26 | + | write []byte // nil means "no file at all" | |
| 26 | - | got, err := decodeRecordValue(r.Name, value) | 27 | + | wantNew bool |
| 27 | - | require.NoError(t, err) | 28 | + | wantErr error |
| 28 | - | require.Equal(t, r, got) | 29 | + | }{ |
| 30 | + | {name: "absent file starts empty", write: nil, wantNew: true}, | |||
| 31 | + | {name: "garbage fails closed", write: []byte("not a vev catalogue"), wantErr: ErrCatalogueUnreadable}, | |||
| 32 | + | {name: "legacy headerless record fails closed", write: []byte{0x08, 0x00, 0x00, 0x00, 0xde, 0xad, 0xbe, 0xef}, wantErr: ErrCatalogueUnreadable}, | |||
| 33 | + | } | |||
| 34 | + | for _, tt := range tests { | |||
| 35 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 36 | + | t.Parallel() | |||
| 37 | + | dir := privateDir(t) | |||
| 38 | + | if tt.write != nil { | |||
| 39 | + | require.NoError(t, safedir.EnsurePrivate(dir)) | |||
| 40 | + | require.NoError(t, os.WriteFile(StorePath(dir), tt.write, 0o600)) | |||
| 41 | + | } | |||
| 42 | + | got, err := OpenOrCreate(dir) | |||
| 43 | + | if tt.wantErr != nil { | |||
| 44 | + | require.ErrorIs(t, err, tt.wantErr) | |||
| 45 | + | after, readErr := os.ReadFile(StorePath(dir)) | |||
| 46 | + | require.NoError(t, readErr) | |||
| 47 | + | require.Equal(t, tt.write, after, "unreadable catalogue must remain untouched") | |||
| 48 | + | return | |||
| 49 | + | } | |||
| 50 | + | require.NoError(t, err) | |||
| 51 | + | t.Cleanup(func() { _ = got.Catalogue.Close() }) | |||
| 52 | + | require.Equal(t, tt.wantNew, got.NewInstall) | |||
| 53 | + | require.Empty(t, got.Records) | |||
| 54 | + | }) | |||
| 55 | + | } | |||
| 29 | } | 56 | } | ||
| 30 | 57 | ||||
| 31 | - | func TestRecordCodecOldRecordDefaultsLastUsed(t *testing.T) { | 58 | + | func TestOpenOrCreateRejectsNonCurrentDurableStateWithoutMutation(t *testing.T) { |
| 32 | - | old := []byte{0, 0, 0, 4, '/', 't', 'm', 'p'} | 59 | + | t.Parallel() |
| 33 | - | old = append(old, 0, 0, 0, 0, 0, 0, 0, 11) | 60 | + | tests := []struct { |
| 34 | - | old = append(old, 0, 0, 0, 0, 0, 0, 0, 22) | 61 | + | name string |
| 62 | + | files map[string][]byte | |||
| 63 | + | }{ | |||
| 64 | + | { | |||
| 65 | + | name: "old VEVK version", | |||
| 66 | + | files: map[string][]byte{ | |||
| 67 | + | filename: {'V', 'E', 'V', 'K', 0, 1}, | |||
| 68 | + | }, | |||
| 69 | + | }, | |||
| 70 | + | } | |||
| 71 | + | for _, tt := range tests { | |||
| 72 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 73 | + | t.Parallel() | |||
| 74 | + | dir := privateDir(t) | |||
| 75 | + | require.NoError(t, safedir.EnsurePrivate(dir)) | |||
| 76 | + | for name, data := range tt.files { | |||
| 77 | + | require.NoError(t, os.WriteFile(filepath.Join(dir, name), data, 0o600)) | |||
| 78 | + | } | |||
| 79 | + | ||||
| 80 | + | before := directoryBytes(t, dir) | |||
| 81 | + | _, err := OpenOrCreate(dir) | |||
| 82 | + | require.ErrorIs(t, err, ErrCatalogueUnreadable) | |||
| 83 | + | after := directoryBytes(t, dir) | |||
| 84 | + | for name, data := range before { | |||
| 85 | + | require.Equal(t, data, after[name], "failure must preserve durable file %q", name) | |||
| 86 | + | } | |||
| 87 | + | }) | |||
| 88 | + | } | |||
| 89 | + | } | |||
| 35 | 90 | ||||
| 36 | - | got, err := decodeRecordValue("work", old) | 91 | + | func directoryBytes(t *testing.T, dir string) map[string][]byte { |
| 92 | + | t.Helper() | |||
| 93 | + | entries, err := os.ReadDir(dir) | |||
| 37 | require.NoError(t, err) | 94 | require.NoError(t, err) | ||
| 38 | - | require.Equal(t, Record{Name: "work", Cwd: "/tmp", CreatedAt: 11, UpdatedAt: 22}, got) | 95 | + | files := make(map[string][]byte, len(entries)) |
| 96 | + | for _, entry := range entries { | |||
| 97 | + | require.False(t, entry.IsDir(), "unexpected directory %q", entry.Name()) | |||
| 98 | + | data, err := os.ReadFile(filepath.Join(dir, entry.Name())) | |||
| 99 | + | require.NoError(t, err) | |||
| 100 | + | files[entry.Name()] = data | |||
| 101 | + | } | |||
| 102 | + | return files | |||
| 39 | } | 103 | } | ||
| 40 | 104 | ||||
| 41 | - | func TestRecordCodecRejectsMalformedData(t *testing.T) { | 105 | + | func TestCatalogueDecodeAllFailsOnMalformedRecord(t *testing.T) { |
| 42 | - | _, err := encodeRecordValue(Record{Cwd: "/tmp"}) | 106 | + | good, err := encodeRecordValue(validRecord("good", 1)) |
| 43 | - | require.ErrorIs(t, err, errEmptyName) | 107 | + | require.NoError(t, err) |
| 108 | + | _, err = decodeAll(map[string][]byte{"bad": {0, 1}, "good": good}) | |||
| 109 | + | require.Error(t, err) | |||
| 110 | + | } | |||
| 44 | 111 | ||||
| 45 | - | badValues := [][]byte{ | 112 | + | func TestCatalogueMutationsUseOneSync(t *testing.T) { |
| 46 | - | nil, | 113 | + | store := portsmocks.NewMockStore(t) |
| 47 | - | {0, 0, 0}, | 114 | + | state := map[string][]byte{} |
| 48 | - | {0, 0, 0, 5, 'a'}, | 115 | + | store.EXPECT().Range(mock.Anything).Run(func(fn func([]byte, []byte) bool) { |
| 49 | - | {0, 0, 0, 1, 'a', 0}, | 116 | + | for key, value := range state { |
| 50 | - | {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, | 117 | + | if !fn([]byte(key), value) { |
| 51 | - | } | 118 | + | return |
| 52 | - | for _, value := range badValues { | 119 | + | } |
| 53 | - | _, err := decodeRecordValue("name", value) | 120 | + | } |
| 54 | - | require.ErrorIs(t, err, errMalformedRecord) | 121 | + | }).Once() |
| 55 | - | } | 122 | + | store.EXPECT().Get(mock.Anything).RunAndReturn(func(key []byte) ([]byte, bool) { |
| 123 | + | value, ok := state[string(key)] | |||
| 124 | + | return append([]byte(nil), value...), ok | |||
| 125 | + | }).Times(3) | |||
| 126 | + | store.EXPECT().Set(mock.Anything, mock.Anything).RunAndReturn(func(key, value []byte) error { | |||
| 127 | + | state[string(key)] = append([]byte(nil), value...) | |||
| 128 | + | return nil | |||
| 129 | + | }).Times(2) | |||
| 130 | + | store.EXPECT().Delete([]byte("one")).RunAndReturn(func(key []byte) error { | |||
| 131 | + | delete(state, string(key)) | |||
| 132 | + | return nil | |||
| 133 | + | }).Once() | |||
| 134 | + | store.EXPECT().Sync().Return(nil).Twice() | |||
| 135 | + | p := New(store) | |||
| 136 | + | one := validRecord("one", 1) | |||
| 137 | + | require.NoError(t, p.Replace("one", one)) | |||
| 138 | + | renamed := one | |||
| 139 | + | renamed.Name = "renamed" | |||
| 140 | + | renamed.Cwd = "/renamed" | |||
| 141 | + | require.NoError(t, p.Rename("one", renamed)) | |||
| 142 | + | require.NotContains(t, state, "one") | |||
| 143 | + | encoded, ok := state["renamed"] | |||
| 144 | + | require.True(t, ok) | |||
| 145 | + | got, err := decodeRecordValue("renamed", encoded) | |||
| 146 | + | require.NoError(t, err) | |||
| 147 | + | require.Equal(t, renamed, got) | |||
| 148 | + | } | |||
| 56 | 149 | ||||
| 57 | - | _, err = decodeRecordValue("", make([]byte, 20)) | 150 | + | func TestCatalogueMutationFailureDoesNotSync(t *testing.T) { |
| 58 | - | require.ErrorIs(t, err, errEmptyName) | 151 | + | store := portsmocks.NewMockStore(t) |
| 152 | + | setErr := errors.New("set failed") | |||
| 153 | + | store.EXPECT().Range(mock.Anything).Run(func(func([]byte, []byte) bool) {}) | |||
| 154 | + | store.EXPECT().Get([]byte("one")).Return(nil, false).Once() | |||
| 155 | + | store.EXPECT().Set(mock.Anything, mock.Anything).Return(setErr) | |||
| 156 | + | store.EXPECT().Delete([]byte("one")).Return(nil).Once() | |||
| 157 | + | p := New(store) | |||
| 158 | + | err := p.Replace("one", validRecord("one", 1)) | |||
| 159 | + | require.ErrorIs(t, err, setErr) | |||
| 59 | } | 160 | } | ||
| 60 | 161 | ||||
| 61 | - | func TestNilPersisterNoOps(t *testing.T) { | 162 | + | // TestCatalogueNilPersisterIsNoOp covers the ephemeral/optional-persistence |
| 163 | + | // sentinel: a nil *Persister. Every mutating and reading method must be a | |||
| 164 | + | // silent no-op so callers that never enable persistence don't need nil | |||
| 165 | + | // checks of their own. | |||
| 166 | + | func TestCatalogueNilPersisterIsNoOp(t *testing.T) { | |||
| 62 | var p *Persister | 167 | var p *Persister | ||
| 63 | - | require.NoError(t, p.Save(Record{Name: "a", Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1})) | 168 | + | require.NoError(t, p.Save(validRecord("one", 1))) |
| 64 | - | require.NoError(t, p.Touch("a", "/tmp/next", 2)) | 169 | + | require.NoError(t, p.Apply(map[string]*domain.CatalogueRecord{"one": nil})) |
| 65 | - | require.NoError(t, p.Delete("a")) | 170 | + | require.NoError(t, p.UpdateMetadata(validRecord("one", 1).MetadataUpdate())) |
| 171 | + | require.NoError(t, p.Delete("one")) | |||
| 66 | records, err := p.LoadAll() | 172 | records, err := p.LoadAll() | ||
| 67 | require.NoError(t, err) | 173 | require.NoError(t, err) | ||
| 68 | require.Empty(t, records) | 174 | require.Empty(t, records) | ||
| 69 | require.NoError(t, p.Close()) | 175 | require.NoError(t, p.Close()) | ||
| 70 | - | 176 | + | got, err := p.Records() | |
| 71 | - | p = New(nil) | |||
| 72 | - | require.NoError(t, p.Save(Record{Name: "a", Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1})) | |||
| 73 | - | require.NoError(t, p.Touch("a", "/tmp/next", 2)) | |||
| 74 | - | require.NoError(t, p.Delete("a")) | |||
| 75 | - | records, err = p.LoadAll() | |||
| 76 | require.NoError(t, err) | 177 | require.NoError(t, err) | ||
| 77 | - | require.Empty(t, records) | 178 | + | require.Empty(t, got) |
| 78 | - | require.NoError(t, p.Close()) | |||
| 79 | } | 179 | } | ||
| 80 | 180 | ||||
| 81 | - | func TestPersisterSaveTouchDeleteLoadAll(t *testing.T) { | 181 | + | // TestCatalogueUnavailableStoreErrors covers a non-nil Persister with no |
| 82 | - | dir := privateDir(t) | 182 | + | // backing store (e.g. New(nil)). Unlike the nil-persister sentinel above, |
| 83 | - | p, err := Open(dir) | 183 | + | // this is a Persister that was supposed to have a store: silently no-oping |
| 84 | - | require.NoError(t, err) | 184 | + | // here previously made recovery.Coordinator's checkpoint publish/retry/ |
| 85 | - | 185 | + | // discard paths believe a durable commit happened when nothing was written. | |
| 86 | - | require.NoError(t, p.Save(Record{Name: "b", Cwd: "/b", CreatedAt: 10, UpdatedAt: 10, LastUsedSeq: 40, TabNames: []string{"shell"}})) | 186 | + | // Every mutation and read-with-error must surface errPersistenceUnavailable, |
| 87 | - | require.NoError(t, p.Save(Record{Name: "a", Cwd: "/a", CreatedAt: 20, UpdatedAt: 20, LastUsedSeq: 50})) | 187 | + | // matching Create; only Records() (which ports.Catalogue gives no error |
| 88 | - | require.NoError(t, p.Touch("b", "/b/next", 30)) | 188 | + | // return for) degrades to an empty slice, with the failure logged instead. |
| 89 | - | require.NoError(t, p.TouchMRU("b", 60)) | 189 | + | func TestCatalogueUnavailableStoreErrors(t *testing.T) { |
| 90 | - | 190 | + | p := New(nil) | |
| 191 | + | require.ErrorIs(t, p.Save(validRecord("one", 1)), errPersistenceUnavailable) | |||
| 192 | + | require.ErrorIs(t, p.Apply(map[string]*domain.CatalogueRecord{"one": nil}), errPersistenceUnavailable) | |||
| 193 | + | require.ErrorIs(t, p.Rename("one", validRecord("two", 1)), errPersistenceUnavailable) | |||
| 194 | + | require.ErrorIs(t, p.UpdateMetadata(validRecord("one", 1).MetadataUpdate()), errPersistenceUnavailable) | |||
| 195 | + | require.ErrorIs(t, p.Delete("one"), errPersistenceUnavailable) | |||
| 196 | + | require.ErrorIs(t, p.Create(validRecord("one", 1)), errPersistenceUnavailable) | |||
| 91 | records, err := p.LoadAll() | 197 | records, err := p.LoadAll() | ||
| 92 | - | require.NoError(t, err) | 198 | + | require.ErrorIs(t, err, errPersistenceUnavailable) |
| 93 | - | require.Equal(t, []Record{ | 199 | + | require.Empty(t, records) |
| 94 | - | {Name: "a", Cwd: "/a", CreatedAt: 20, UpdatedAt: 20, LastUsedSeq: 50}, | 200 | + | got, err := p.Records() |
| 95 | - | {Name: "b", Cwd: "/b/next", CreatedAt: 10, UpdatedAt: 30, LastUsedSeq: 60, TabNames: []string{"shell"}}, | 201 | + | require.ErrorIs(t, err, errPersistenceUnavailable) |
| 96 | - | }, records) | 202 | + | require.Empty(t, got) |
| 97 | - | ||||
| 98 | - | require.NoError(t, p.Delete("a")) | |||
| 99 | - | records, err = p.LoadAll() | |||
| 100 | - | require.NoError(t, err) | |||
| 101 | - | require.Equal(t, []Record{{Name: "b", Cwd: "/b/next", CreatedAt: 10, UpdatedAt: 30, LastUsedSeq: 60, TabNames: []string{"shell"}}}, records) | |||
| 102 | require.NoError(t, p.Close()) | 203 | require.NoError(t, p.Close()) | ||
| 103 | } | 204 | } | ||
| 104 | 205 | ||||
| 105 | - | func TestDecodeAllSkipsMalformedRecords(t *testing.T) { | 206 | + | func TestCatalogueRecordsPropagatesMalformedRecord(t *testing.T) { |
| 106 | - | good, err := encodeRecordValue(Record{Name: "good", Cwd: "/ok", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3}) | 207 | + | store := portsmocks.NewMockStore(t) |
| 107 | - | require.NoError(t, err) | 208 | + | store.EXPECT().Range(mock.Anything).Run(func(fn func([]byte, []byte) bool) { |
| 108 | - | 209 | + | fn([]byte("bad"), []byte{0, 1}) | |
| 109 | - | records, err := decodeAll(map[string][]byte{ | |||
| 110 | - | "bad": []byte{0, 0, 0, 5, 'x'}, | |||
| 111 | - | "good": good, | |||
| 112 | }) | 210 | }) | ||
| 113 | - | require.NoError(t, err) | 211 | + | records, err := New(store).Records() |
| 114 | - | require.Equal(t, []Record{{Name: "good", Cwd: "/ok", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3}}, records) | 212 | + | require.Error(t, err) |
| 115 | - | } | |||
| 116 | - | ||||
| 117 | - | func TestLoadReadOnlyUsesReplay(t *testing.T) { | |||
| 118 | - | dir := privateDir(t) | |||
| 119 | - | p, err := Open(dir) | |||
| 120 | - | require.NoError(t, err) | |||
| 121 | - | require.NoError(t, p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3})) | |||
| 122 | - | require.NoError(t, p.Close()) | |||
| 123 | - | ||||
| 124 | - | records, err := LoadReadOnly(dir) | |||
| 125 | - | require.NoError(t, err) | |||
| 126 | - | require.Equal(t, []Record{{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 2, LastUsedSeq: 3}}, records) | |||
| 127 | - | ||||
| 128 | - | // Missing stores replay as empty rather than creating the WAL. | |||
| 129 | - | missing := filepath.Join(t.TempDir(), "missing") | |||
| 130 | - | records, err = LoadReadOnly(missing) | |||
| 131 | - | require.NoError(t, err) | |||
| 132 | require.Empty(t, records) | 213 | require.Empty(t, records) | ||
| 133 | } | 214 | } | ||
| 134 | 215 | ||||
| 135 | - | func TestSyncBehavior(t *testing.T) { | 216 | + | type failClosedStore struct { |
| 136 | - | store, state := newMockStore(t) | 217 | + | data map[string][]byte |
| 137 | - | p := New(store) | 218 | + | durable map[string][]byte |
| 138 | - | 219 | + | ||
| 139 | - | require.NoError(t, p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 1})) | 220 | + | setFailures int |
| 140 | - | require.Equal(t, 1, state.sets) | 221 | + | deleteFailures int |
| 141 | - | require.Equal(t, 1, state.syncs) | 222 | + | syncFailures int |
| 142 | - | 223 | + | restoreSetFailures int | |
| 143 | - | require.NoError(t, p.Touch("work", "/work/next", 2)) | 224 | + | restoreDeleteFailures int |
| 144 | - | require.Equal(t, 2, state.sets) | 225 | + | restoring bool |
| 145 | - | require.Equal(t, 1, state.syncs) | 226 | + | failure error |
| 146 | - | 227 | + | restoreFailure error | |
| 147 | - | require.NoError(t, p.TouchMRU("work", 3)) | 228 | + | |
| 148 | - | require.Equal(t, 3, state.sets) | 229 | + | closeFlushed bool |
| 149 | - | require.Equal(t, 1, state.syncs) | 230 | + | closeAborted bool |
| 150 | - | ||||
| 151 | - | require.NoError(t, p.Delete("work")) | |||
| 152 | - | require.Equal(t, 1, state.deletes) | |||
| 153 | - | require.Equal(t, 2, state.syncs) | |||
| 154 | } | 231 | } | ||
| 155 | 232 | ||||
| 156 | - | func TestPersisterSerializesReadModifyWriteWithStructuralSave(t *testing.T) { | 233 | + | func newFailClosedStore(data map[string][]byte) *failClosedStore { |
| 157 | - | store := newBlockingGetStore() | 234 | + | return &failClosedStore{ |
| 158 | - | p := New(store) | 235 | + | data: copyStoreData(data), |
| 159 | - | require.NoError(t, p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 1, TabNames: []string{"old"}})) | 236 | + | durable: copyStoreData(data), |
| 160 | - | store.blockNextGet() | 237 | + | failure: errors.New("store write failed"), |
| 161 | - | 238 | + | restoreFailure: errors.New("store restore failed"), | |
| 162 | - | touchDone := make(chan error, 1) | |||
| 163 | - | go func() { touchDone <- p.TouchMRU("work", 2) }() | |||
| 164 | - | <-store.getStarted | |||
| 165 | - | ||||
| 166 | - | saveDone := make(chan error, 1) | |||
| 167 | - | go func() { | |||
| 168 | - | saveDone <- p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 3, LastUsedSeq: 2, TabNames: []string{"new"}}) | |||
| 169 | - | }() | |||
| 170 | - | select { | |||
| 171 | - | case err := <-saveDone: | |||
| 172 | - | t.Fatalf("structural save interleaved with TouchMRU: %v", err) | |||
| 173 | - | case <-time.After(20 * time.Millisecond): | |||
| 174 | } | 239 | } | ||
| 175 | - | close(store.releaseGet) | |||
| 176 | - | require.NoError(t, <-touchDone) | |||
| 177 | - | require.NoError(t, <-saveDone) | |||
| 178 | - | ||||
| 179 | - | records, err := p.LoadAll() | |||
| 180 | - | require.NoError(t, err) | |||
| 181 | - | require.Equal(t, []string{"new"}, records[0].TabNames) | |||
| 182 | } | 240 | } | ||
| 183 | 241 | ||||
| 184 | - | func TestSavePropagatesSetErrorWithoutSync(t *testing.T) { | 242 | + | func (s *failClosedStore) Get(key []byte) ([]byte, bool) { |
| 185 | - | store, state := newMockStore(t) | 243 | + | value, ok := s.data[string(key)] |
| 186 | - | state.errSet = errors.New("set failed") | 244 | + | return append([]byte(nil), value...), ok |
| 187 | - | p := New(store) | |||
| 188 | - | ||||
| 189 | - | err := p.Save(Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 1}) | |||
| 190 | - | require.ErrorIs(t, err, state.errSet) | |||
| 191 | - | require.Zero(t, state.syncs) | |||
| 192 | - | } | |||
| 193 | - | ||||
| 194 | - | type mockStoreState struct { | |||
| 195 | - | data map[string][]byte | |||
| 196 | - | sets int | |||
| 197 | - | deletes int | |||
| 198 | - | syncs int | |||
| 199 | - | closed bool | |||
| 200 | - | errSet error | |||
| 201 | } | 245 | } | ||
| 202 | 246 | ||||
| 203 | - | type blockingGetStore struct { | 247 | + | func (s *failClosedStore) Set(key, value []byte) error { |
| 204 | - | mu sync.Mutex | 248 | + | s.data[string(key)] = append([]byte(nil), value...) |
| 205 | - | data map[string][]byte | 249 | + | if s.restoring && s.restoreSetFailures > 0 { |
| 206 | - | block bool | 250 | + | s.restoreSetFailures-- |
| 207 | - | getStarted chan struct{} | 251 | + | return s.restoreFailure |
| 208 | - | releaseGet chan struct{} | 252 | + | } |
| 209 | - | getStartOnce sync.Once | 253 | + | if s.setFailures > 0 { |
| 254 | + | s.setFailures-- | |||
| 255 | + | return s.failure | |||
| 256 | + | } | |||
| 257 | + | return nil | |||
| 210 | } | 258 | } | ||
| 211 | 259 | ||||
| 212 | - | func newBlockingGetStore() *blockingGetStore { | 260 | + | func (s *failClosedStore) Delete(key []byte) error { |
| 213 | - | return &blockingGetStore{data: make(map[string][]byte), getStarted: make(chan struct{}), releaseGet: make(chan struct{})} | 261 | + | delete(s.data, string(key)) |
| 262 | + | if s.restoring && s.restoreDeleteFailures > 0 { | |||
| 263 | + | s.restoreDeleteFailures-- | |||
| 264 | + | return s.restoreFailure | |||
| 265 | + | } | |||
| 266 | + | if s.deleteFailures > 0 { | |||
| 267 | + | s.deleteFailures-- | |||
| 268 | + | return s.failure | |||
| 269 | + | } | |||
| 270 | + | return nil | |||
| 214 | } | 271 | } | ||
| 215 | 272 | ||||
| 216 | - | func (s *blockingGetStore) blockNextGet() { | 273 | + | func (s *failClosedStore) Range(fn func([]byte, []byte) bool) { |
| 217 | - | s.mu.Lock() | 274 | + | for key, value := range s.data { |
| 218 | - | s.block = true | 275 | + | if !fn([]byte(key), append([]byte(nil), value...)) { |
| 219 | - | s.mu.Unlock() | 276 | + | return |
| 277 | + | } | |||
| 278 | + | } | |||
| 220 | } | 279 | } | ||
| 221 | 280 | ||||
| 222 | - | func (s *blockingGetStore) Get(key []byte) ([]byte, bool) { | 281 | + | func (s *failClosedStore) Sync() error { |
| 223 | - | s.mu.Lock() | 282 | + | if s.syncFailures > 0 { |
| 224 | - | v, ok := s.data[string(key)] | 283 | + | s.syncFailures-- |
| 225 | - | v = append([]byte(nil), v...) | 284 | + | s.restoring = true |
| 226 | - | block := s.block | 285 | + | return s.failure |
| 227 | - | if block { | |||
| 228 | - | s.block = false | |||
| 229 | - | } | |||
| 230 | - | s.mu.Unlock() | |||
| 231 | - | if block { | |||
| 232 | - | s.getStartOnce.Do(func() { close(s.getStarted) }) | |||
| 233 | - | <-s.releaseGet | |||
| 234 | } | 286 | } | ||
| 235 | - | return v, ok | 287 | + | s.durable = copyStoreData(s.data) |
| 288 | + | return nil | |||
| 236 | } | 289 | } | ||
| 237 | 290 | ||||
| 238 | - | func (s *blockingGetStore) Set(key, value []byte) error { | 291 | + | func (s *failClosedStore) Close() error { |
| 239 | - | s.mu.Lock() | 292 | + | s.closeFlushed = true |
| 240 | - | s.data[string(key)] = append([]byte(nil), value...) | 293 | + | s.durable = copyStoreData(s.data) |
| 241 | - | s.mu.Unlock() | |||
| 242 | return nil | 294 | return nil | ||
| 243 | } | 295 | } | ||
| 244 | 296 | ||||
| 245 | - | func (s *blockingGetStore) Delete(key []byte) error { | 297 | + | func (s *failClosedStore) CloseWithoutSync() error { |
| 246 | - | s.mu.Lock() | 298 | + | s.closeAborted = true |
| 247 | - | delete(s.data, string(key)) | |||
| 248 | - | s.mu.Unlock() | |||
| 249 | return nil | 299 | return nil | ||
| 250 | } | 300 | } | ||
| 251 | 301 | ||||
| 252 | - | func (s *blockingGetStore) Range(fn func(k, v []byte) bool) { | 302 | + | func copyStoreData(data map[string][]byte) map[string][]byte { |
| 253 | - | s.mu.Lock() | 303 | + | cloned := make(map[string][]byte, len(data)) |
| 254 | - | defer s.mu.Unlock() | 304 | + | for key, value := range data { |
| 255 | - | for key, value := range s.data { | 305 | + | cloned[key] = append([]byte(nil), value...) |
| 256 | - | if !fn([]byte(key), append([]byte(nil), value...)) { | |||
| 257 | - | return | |||
| 258 | - | } | |||
| 259 | } | 306 | } | ||
| 307 | + | return cloned | |||
| 260 | } | 308 | } | ||
| 261 | 309 | ||||
| 262 | - | func (*blockingGetStore) Sync() error { return nil } | 310 | + | func TestRejectedIdentityWriteFencesCatalogue(t *testing.T) { |
| 263 | - | func (*blockingGetStore) Close() error { return nil } | 311 | + | t.Parallel() |
| 312 | + | one, err := encodeRecordValue(validRecord("work", 1)) | |||
| 313 | + | require.NoError(t, err) | |||
| 264 | 314 | ||||
| 265 | - | func newMockStore(t *testing.T) (*portsmocks.MockStore, *mockStoreState) { | 315 | + | tests := []struct { |
| 266 | - | t.Helper() | 316 | + | name string |
| 267 | - | state := &mockStoreState{data: make(map[string][]byte)} | 317 | + | initial map[string][]byte |
| 268 | - | store := portsmocks.NewMockStore(t) | 318 | + | setFailures int |
| 269 | - | store.EXPECT().Get(mock.Anything).RunAndReturn(func(key []byte) ([]byte, bool) { | 319 | + | deleteFailures int |
| 270 | - | v, ok := state.data[string(key)] | 320 | + | syncFailures int |
| 271 | - | return append([]byte(nil), v...), ok | 321 | + | mutate func(*Persister) error |
| 272 | - | }).Maybe() | 322 | + | }{ |
| 273 | - | store.EXPECT().Set(mock.Anything, mock.Anything).RunAndReturn(func(key, val []byte) error { | 323 | + | { |
| 274 | - | state.sets++ | 324 | + | name: "create set failure", |
| 275 | - | if state.errSet != nil { | 325 | + | initial: map[string][]byte{}, |
| 276 | - | return state.errSet | 326 | + | setFailures: 1, |
| 277 | - | } | 327 | + | mutate: func(p *Persister) error { |
| 278 | - | state.data[string(key)] = append([]byte(nil), val...) | 328 | + | return p.Create(validRecord("new", 2)) |
| 279 | - | return nil | 329 | + | }, |
| 280 | - | }).Maybe() | 330 | + | }, |
| 281 | - | store.EXPECT().Delete(mock.Anything).RunAndReturn(func(key []byte) error { | 331 | + | { |
| 282 | - | state.deletes++ | 332 | + | name: "create sync failure", |
| 283 | - | delete(state.data, string(key)) | 333 | + | initial: map[string][]byte{}, |
| 284 | - | return nil | 334 | + | syncFailures: 1, |
| 285 | - | }).Maybe() | 335 | + | mutate: func(p *Persister) error { |
| 286 | - | store.EXPECT().Range(mock.Anything).Run(func(fn func(k, v []byte) bool) { | 336 | + | return p.Create(validRecord("new", 2)) |
| 287 | - | for k, v := range state.data { | 337 | + | }, |
| 288 | - | if !fn([]byte(k), append([]byte(nil), v...)) { | 338 | + | }, |
| 289 | - | return | 339 | + | { |
| 290 | - | } | 340 | + | name: "replace set failure", |
| 291 | - | } | 341 | + | initial: map[string][]byte{"work": one}, |
| 292 | - | }).Maybe() | 342 | + | setFailures: 1, |
| 293 | - | store.EXPECT().Sync().RunAndReturn(func() error { | 343 | + | mutate: func(p *Persister) error { |
| 294 | - | state.syncs++ | 344 | + | next := validRecord("work", 1) |
| 295 | - | return nil | 345 | + | next.Cwd = "/next" |
| 296 | - | }).Maybe() | 346 | + | return p.Replace("work", next) |
| 297 | - | store.EXPECT().Close().RunAndReturn(func() error { | 347 | + | }, |
| 298 | - | state.closed = true | 348 | + | }, |
| 299 | - | return nil | 349 | + | { |
| 300 | - | }).Maybe() | 350 | + | name: "replace sync failure", |
| 301 | - | return store, state | 351 | + | initial: map[string][]byte{"work": one}, |
| 352 | + | syncFailures: 1, | |||
| 353 | + | mutate: func(p *Persister) error { | |||
| 354 | + | next := validRecord("work", 1) | |||
| 355 | + | next.Cwd = "/next" | |||
| 356 | + | return p.Replace("work", next) | |||
| 357 | + | }, | |||
| 358 | + | }, | |||
| 359 | + | { | |||
| 360 | + | name: "rename set failure", | |||
| 361 | + | initial: map[string][]byte{"work": one}, | |||
| 362 | + | setFailures: 1, | |||
| 363 | + | mutate: func(p *Persister) error { | |||
| 364 | + | next := validRecord("renamed", 1) | |||
| 365 | + | return p.Rename("work", next) | |||
| 366 | + | }, | |||
| 367 | + | }, | |||
| 368 | + | { | |||
| 369 | + | name: "rename delete failure", | |||
| 370 | + | initial: map[string][]byte{"work": one}, | |||
| 371 | + | deleteFailures: 1, | |||
| 372 | + | mutate: func(p *Persister) error { | |||
| 373 | + | next := validRecord("renamed", 1) | |||
| 374 | + | return p.Rename("work", next) | |||
| 375 | + | }, | |||
| 376 | + | }, | |||
| 377 | + | { | |||
| 378 | + | name: "rename sync failure", | |||
| 379 | + | initial: map[string][]byte{"work": one}, | |||
| 380 | + | syncFailures: 1, | |||
| 381 | + | mutate: func(p *Persister) error { | |||
| 382 | + | next := validRecord("renamed", 1) | |||
| 383 | + | return p.Rename("work", next) | |||
| 384 | + | }, | |||
| 385 | + | }, | |||
| 386 | + | { | |||
| 387 | + | name: "delete delete failure", | |||
| 388 | + | initial: map[string][]byte{"work": one}, | |||
| 389 | + | deleteFailures: 1, | |||
| 390 | + | mutate: func(p *Persister) error { | |||
| 391 | + | return p.Delete("work") | |||
| 392 | + | }, | |||
| 393 | + | }, | |||
| 394 | + | { | |||
| 395 | + | name: "delete sync failure", | |||
| 396 | + | initial: map[string][]byte{"work": one}, | |||
| 397 | + | syncFailures: 1, | |||
| 398 | + | mutate: func(p *Persister) error { | |||
| 399 | + | return p.Delete("work") | |||
| 400 | + | }, | |||
| 401 | + | }, | |||
| 402 | + | } | |||
| 403 | + | ||||
| 404 | + | for _, tt := range tests { | |||
| 405 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 406 | + | t.Parallel() | |||
| 407 | + | store := newFailClosedStore(tt.initial) | |||
| 408 | + | store.setFailures = tt.setFailures | |||
| 409 | + | store.deleteFailures = tt.deleteFailures | |||
| 410 | + | store.syncFailures = tt.syncFailures | |||
| 411 | + | p := New(store) | |||
| 412 | + | ||||
| 413 | + | err := tt.mutate(p) | |||
| 414 | + | require.ErrorIs(t, err, ErrCatalogueDurability) | |||
| 415 | + | require.ErrorIs(t, err, store.failure) | |||
| 416 | + | require.Equal(t, tt.initial, store.data, "rejected identity keys must be restored in memory") | |||
| 417 | + | require.Equal(t, tt.initial, store.durable, "rejected identity write must not reach disk") | |||
| 418 | + | ||||
| 419 | + | _, _, recordErr := p.Record("work") | |||
| 420 | + | require.ErrorIs(t, recordErr, ErrCatalogueDurability) | |||
| 421 | + | require.ErrorIs(t, p.Sync(), ErrCatalogueDurability) | |||
| 422 | + | require.ErrorIs(t, p.Create(validRecord("later", 3)), ErrCatalogueDurability) | |||
| 423 | + | require.Equal(t, tt.initial, store.durable, "later Sync must be fenced") | |||
| 424 | + | ||||
| 425 | + | require.ErrorIs(t, p.Close(), ErrCatalogueDurability) | |||
| 426 | + | require.True(t, store.closeAborted, "Close must release without flushing rejected state") | |||
| 427 | + | require.False(t, store.closeFlushed) | |||
| 428 | + | require.Equal(t, tt.initial, store.durable, "Close must not make rejected state durable") | |||
| 429 | + | }) | |||
| 430 | + | } | |||
| 431 | + | } | |||
| 432 | + | ||||
| 433 | + | func TestFailedSyncRetainsRepeatedRollbackFailuresAndAborts(t *testing.T) { | |||
| 434 | + | t.Parallel() | |||
| 435 | + | encoded, err := encodeRecordValue(validRecord("work", 1)) | |||
| 436 | + | require.NoError(t, err) | |||
| 437 | + | store := newFailClosedStore(map[string][]byte{"work": encoded}) | |||
| 438 | + | store.syncFailures = 1 | |||
| 439 | + | store.restoreSetFailures = 1 | |||
| 440 | + | store.restoreDeleteFailures = 1 | |||
| 441 | + | p := New(store) | |||
| 442 | + | ||||
| 443 | + | err = p.Rename("work", validRecord("renamed", 1)) | |||
| 444 | + | require.ErrorIs(t, err, ErrCatalogueDurability) | |||
| 445 | + | require.ErrorIs(t, err, store.failure) | |||
| 446 | + | require.ErrorIs(t, err, store.restoreFailure) | |||
| 447 | + | require.Zero(t, store.restoreSetFailures, "rollback Set failure must be observed") | |||
| 448 | + | require.Zero(t, store.restoreDeleteFailures, "rollback Delete failure must be observed") | |||
| 449 | + | ||||
| 450 | + | terminal := p.Sync() | |||
| 451 | + | require.Same(t, err, terminal, "the fenced terminal error must remain stable") | |||
| 452 | + | require.ErrorIs(t, terminal, store.failure) | |||
| 453 | + | require.ErrorIs(t, terminal, store.restoreFailure) | |||
| 454 | + | require.ErrorIs(t, p.Close(), ErrCatalogueDurability) | |||
| 455 | + | require.True(t, store.closeAborted) | |||
| 456 | + | require.False(t, store.closeFlushed) | |||
| 457 | + | require.Equal(t, map[string][]byte{"work": encoded}, store.durable, | |||
| 458 | + | "aborting a fenced store must not persist dirty rejected state") | |||
| 459 | + | } | |||
| 460 | + | ||||
| 461 | + | func TestRejectedIdentityWriteRestoresBufferedMetadata(t *testing.T) { | |||
| 462 | + | t.Parallel() | |||
| 463 | + | encoded, err := encodeRecordValue(validRecord("work", 1)) | |||
| 464 | + | require.NoError(t, err) | |||
| 465 | + | store := newFailClosedStore(map[string][]byte{"work": encoded}) | |||
| 466 | + | p := New(store) | |||
| 467 | + | ||||
| 468 | + | cwd := "/buffered" | |||
| 469 | + | require.NoError(t, p.UpdateMetadata(domain.CatalogueMetadataUpdate{ | |||
| 470 | + | Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: &cwd, | |||
| 471 | + | })) | |||
| 472 | + | buffered := copyStoreData(store.data) | |||
| 473 | + | store.syncFailures = 1 | |||
| 474 | + | ||||
| 475 | + | require.ErrorIs(t, p.Rename("work", validRecord("renamed", 1)), ErrCatalogueDurability) | |||
| 476 | + | require.Equal(t, buffered, store.data, "rollback must preserve already-buffered metadata on the affected key") | |||
| 477 | + | require.Equal(t, map[string][]byte{"work": encoded}, store.durable) | |||
| 478 | + | require.ErrorIs(t, p.Close(), ErrCatalogueDurability) | |||
| 479 | + | require.Equal(t, map[string][]byte{"work": encoded}, store.durable) | |||
| 302 | } | 480 | } | ||
M internal/persist/record.go +181 -105
| @@ -5,122 +5,198 @@ import ( | |||||
| 5 | "errors" | 5 | "errors" | ||
| 6 | "fmt" | 6 | "fmt" | ||
| 7 | "math" | 7 | "math" | ||
| 8 | + | ||||
| 9 | + | "github.com/bnema/vev/internal/domain" | |||
| 8 | ) | 10 | ) | ||
| 9 | 11 | ||||
| 10 | - | // Record is the persisted metadata for a named session. | 12 | + | const catalogueRecordVersion uint16 = 3 |
| 11 | - | type Record struct { | |||
| 12 | - | Name string | |||
| 13 | - | Cwd string | |||
| 14 | - | CreatedAt int64 | |||
| 15 | - | UpdatedAt int64 | |||
| 16 | - | LastUsedSeq uint64 | |||
| 17 | - | TabNames []string | |||
| 18 | - | } | |||
| 19 | 13 | ||||
| 20 | - | var ( | 14 | + | var errMalformedRecord = errors.New("persist: malformed catalogue record") |
| 21 | - | errEmptyName = errors.New("persist: empty session name") | 15 | + | |
| 22 | - | errMalformedRecord = errors.New("persist: malformed record") | 16 | + | var catalogueMagic = [4]byte{'V', 'E', 'V', 'C'} |
| 23 | - | ) | 17 | + | |
| 18 | + | // Record is the durable catalogue record used by daemon code. | |||
| 19 | + | type Record = domain.CatalogueRecord | |||
| 24 | 20 | ||||
| 25 | - | func encodeRecordValue(r Record) ([]byte, error) { | 21 | + | func encodeRecordValue(record domain.CatalogueRecord) ([]byte, error) { |
| 26 | - | if r.Name == "" { | 22 | + | if err := record.Validate(); err != nil { |
| 27 | - | return nil, errEmptyName | 23 | + | return nil, fmt.Errorf("persist: invalid catalogue record: %w", err) |
| 24 | + | } | |||
| 25 | + | buf := make([]byte, 0, 128) | |||
| 26 | + | buf = append(buf, catalogueMagic[:]...) | |||
| 27 | + | buf = binary.BigEndian.AppendUint16(buf, catalogueRecordVersion) | |||
| 28 | + | buf = append(buf, record.IncarnationID[:]...) | |||
| 29 | + | var err error | |||
| 30 | + | buf, err = appendCheckedString(buf, record.Cwd) | |||
| 31 | + | if err != nil { | |||
| 32 | + | return nil, err | |||
| 33 | + | } | |||
| 34 | + | buf = binary.BigEndian.AppendUint64(buf, uint64(record.CreatedAt)) | |||
| 35 | + | buf = binary.BigEndian.AppendUint64(buf, uint64(record.UpdatedAt)) | |||
| 36 | + | buf = binary.BigEndian.AppendUint64(buf, record.LastUsedSeq) | |||
| 37 | + | if uint64(len(record.TabNames)) > math.MaxUint32 { | |||
| 38 | + | return nil, errors.New("persist: too many tab names") | |||
| 39 | + | } | |||
| 40 | + | buf = binary.BigEndian.AppendUint32(buf, uint32(len(record.TabNames))) | |||
| 41 | + | for _, name := range record.TabNames { | |||
| 42 | + | buf, err = appendCheckedString(buf, name) | |||
| 43 | + | if err != nil { | |||
| 44 | + | return nil, err | |||
| 45 | + | } | |||
| 28 | } | 46 | } | ||
| 29 | - | if uint64(len(r.Cwd)) > math.MaxUint32 { | 47 | + | buf = appendRef(buf, record.Committed) |
| 30 | - | return nil, fmt.Errorf("persist: cwd too large") | 48 | + | buf, err = appendCheckedString(buf, record.DegradedReason) |
| 49 | + | return buf, err | |||
| 50 | + | } | |||
| 51 | + | ||||
| 52 | + | func decodeRecordValue(name string, value []byte) (domain.CatalogueRecord, error) { | |||
| 53 | + | if err := domain.ValidateSessionName(name); err != nil { | |||
| 54 | + | return domain.CatalogueRecord{}, err | |||
| 55 | + | } | |||
| 56 | + | reader := valueReader{data: value} | |||
| 57 | + | magic, ok := reader.take(4) | |||
| 58 | + | if !ok || string(magic) != string(catalogueMagic[:]) { | |||
| 59 | + | return domain.CatalogueRecord{}, errMalformedRecord | |||
| 60 | + | } | |||
| 61 | + | version, ok := reader.u16() | |||
| 62 | + | if !ok || version != catalogueRecordVersion { | |||
| 63 | + | return domain.CatalogueRecord{}, errMalformedRecord | |||
| 64 | + | } | |||
| 65 | + | id, ok := reader.take(16) | |||
| 66 | + | if !ok { | |||
| 67 | + | return domain.CatalogueRecord{}, errMalformedRecord | |||
| 68 | + | } | |||
| 69 | + | record := domain.CatalogueRecord{Name: name} | |||
| 70 | + | copy(record.IncarnationID[:], id) | |||
| 71 | + | var valid bool | |||
| 72 | + | if record.Cwd, valid = reader.str(); !valid { | |||
| 73 | + | return domain.CatalogueRecord{}, errMalformedRecord | |||
| 74 | + | } | |||
| 75 | + | created, ok := reader.u64() | |||
| 76 | + | if !ok { | |||
| 77 | + | return domain.CatalogueRecord{}, errMalformedRecord | |||
| 78 | + | } | |||
| 79 | + | record.CreatedAt = int64(created) | |||
| 80 | + | updated, ok := reader.u64() | |||
| 81 | + | if !ok { | |||
| 82 | + | return domain.CatalogueRecord{}, errMalformedRecord | |||
| 83 | + | } | |||
| 84 | + | record.UpdatedAt = int64(updated) | |||
| 85 | + | if record.LastUsedSeq, ok = reader.u64(); !ok { | |||
| 86 | + | return domain.CatalogueRecord{}, errMalformedRecord | |||
| 87 | + | } | |||
| 88 | + | count, ok := reader.u32() | |||
| 89 | + | if !ok || uint64(count) > uint64(reader.remaining())/4 { | |||
| 90 | + | return domain.CatalogueRecord{}, errMalformedRecord | |||
| 91 | + | } | |||
| 92 | + | if count > 0 { | |||
| 93 | + | record.TabNames = make([]string, 0, int(count)) | |||
| 94 | + | } | |||
| 95 | + | for range count { | |||
| 96 | + | tab, ok := reader.str() | |||
| 97 | + | if !ok { | |||
| 98 | + | return domain.CatalogueRecord{}, errMalformedRecord | |||
| 99 | + | } | |||
| 100 | + | record.TabNames = append(record.TabNames, tab) | |||
| 31 | } | 101 | } | ||
| 32 | - | if uint64(len(r.TabNames)) > math.MaxUint32 { | 102 | + | if record.Committed, ok = reader.ref(); !ok { |
| 33 | - | return nil, fmt.Errorf("persist: too many tab names") | 103 | + | return domain.CatalogueRecord{}, errMalformedRecord |
| 34 | } | 104 | } | ||
| 35 | - | tabNamesSize := 4 | 105 | + | if record.DegradedReason, ok = reader.str(); !ok || reader.remaining() != 0 { |
| 36 | - | for _, name := range r.TabNames { | 106 | + | return domain.CatalogueRecord{}, errMalformedRecord |
| 37 | - | if uint64(len(name)) > math.MaxUint32 { | 107 | + | } |
| 38 | - | return nil, fmt.Errorf("persist: tab name too large") | 108 | + | if err := record.Validate(); err != nil { |
| 39 | - | } | 109 | + | return domain.CatalogueRecord{}, errors.Join(errMalformedRecord, fmt.Errorf("persist: invalid catalogue record: %w", err)) |
| 40 | - | tabNamesSize += 4 + len(name) | |||
| 41 | } | 110 | } | ||
| 111 | + | return record, nil | |||
| 112 | + | } | |||
| 42 | 113 | ||||
| 43 | - | buf := make([]byte, 4+len(r.Cwd)+8+8+8+tabNamesSize) | 114 | + | func appendCheckedString(buf []byte, value string) ([]byte, error) { |
| 44 | - | binary.BigEndian.PutUint32(buf[:4], uint32(len(r.Cwd))) | 115 | + | if uint64(len(value)) > math.MaxUint32 { |
| 45 | - | copy(buf[4:], r.Cwd) | 116 | + | return nil, errors.New("persist: string too large") |
| 46 | - | off := 4 + len(r.Cwd) | 117 | + | } |
| 47 | - | binary.BigEndian.PutUint64(buf[off:off+8], uint64(r.CreatedAt)) | 118 | + | buf = binary.BigEndian.AppendUint32(buf, uint32(len(value))) |
| 48 | - | binary.BigEndian.PutUint64(buf[off+8:off+16], uint64(r.UpdatedAt)) | 119 | + | return append(buf, value...), nil |
| 49 | - | binary.BigEndian.PutUint64(buf[off+16:off+24], r.LastUsedSeq) | 120 | + | } |
| 50 | - | off += 24 | 121 | + | func appendRef(buf []byte, ref *domain.CheckpointRef) []byte { |
| 51 | - | binary.BigEndian.PutUint32(buf[off:off+4], uint32(len(r.TabNames))) | 122 | + | if ref == nil { |
| 52 | - | off += 4 | 123 | + | return append(buf, 0) |
| 53 | - | for _, name := range r.TabNames { | 124 | + | } |
| 54 | - | binary.BigEndian.PutUint32(buf[off:off+4], uint32(len(name))) | 125 | + | buf = append(buf, 1) |
| 55 | - | off += 4 | 126 | + | buf = binary.BigEndian.AppendUint64(buf, ref.Generation) |
| 56 | - | copy(buf[off:], name) | 127 | + | return append(buf, ref.ManifestDigest[:]...) |
| 57 | - | off += len(name) | |||
| 58 | - | } | |||
| 59 | - | return buf, nil | |||
| 60 | } | 128 | } | ||
| 61 | 129 | ||||
| 62 | - | func decodeRecordValue(name string, value []byte) (Record, error) { | 130 | + | type valueReader struct { |
| 63 | - | if name == "" { | 131 | + | data []byte |
| 64 | - | return Record{}, errEmptyName | 132 | + | off int |
| 65 | - | } | 133 | + | } |
| 66 | - | if len(value) < 4 { | 134 | + | |
| 67 | - | return Record{}, errMalformedRecord | 135 | + | func (r *valueReader) remaining() int { return len(r.data) - r.off } |
| 68 | - | } | 136 | + | func (r *valueReader) take(n int) ([]byte, bool) { |
| 69 | - | cwdLen32 := binary.BigEndian.Uint32(value[:4]) | 137 | + | if n < 0 || n > r.remaining() { |
| 70 | - | remaining := len(value) - 4 | 138 | + | return nil, false |
| 71 | - | if uint64(cwdLen32) > uint64(remaining) { | 139 | + | } |
| 72 | - | return Record{}, errMalformedRecord | 140 | + | value := r.data[r.off : r.off+n] |
| 73 | - | } | 141 | + | r.off += n |
| 74 | - | cwdLen := int(cwdLen32) | 142 | + | return value, true |
| 75 | - | baseLen := 4 + cwdLen + 8 + 8 | 143 | + | } |
| 76 | - | if len(value) < baseLen { | 144 | + | func (r *valueReader) byte() (byte, bool) { |
| 77 | - | return Record{}, errMalformedRecord | 145 | + | value, ok := r.take(1) |
| 78 | - | } | 146 | + | if !ok { |
| 79 | - | off := 4 + cwdLen | 147 | + | return 0, false |
| 80 | - | r := Record{ | 148 | + | } |
| 81 | - | Name: name, | 149 | + | return value[0], true |
| 82 | - | Cwd: string(value[4:off]), | 150 | + | } |
| 83 | - | CreatedAt: int64(binary.BigEndian.Uint64(value[off : off+8])), | 151 | + | func (r *valueReader) u16() (uint16, bool) { |
| 84 | - | UpdatedAt: int64(binary.BigEndian.Uint64(value[off+8 : off+16])), | 152 | + | value, ok := r.take(2) |
| 85 | - | } | 153 | + | if !ok { |
| 86 | - | if len(value) == baseLen { | 154 | + | return 0, false |
| 87 | - | return r, nil | 155 | + | } |
| 88 | - | } | 156 | + | return binary.BigEndian.Uint16(value), true |
| 89 | - | if len(value) < baseLen+8 { | 157 | + | } |
| 90 | - | return Record{}, errMalformedRecord | 158 | + | func (r *valueReader) u32() (uint32, bool) { |
| 91 | - | } | 159 | + | value, ok := r.take(4) |
| 92 | - | if len(value) == baseLen+8 { | 160 | + | if !ok { |
| 93 | - | r.LastUsedSeq = binary.BigEndian.Uint64(value[off+16 : off+24]) | 161 | + | return 0, false |
| 94 | - | return r, nil | 162 | + | } |
| 95 | - | } | 163 | + | return binary.BigEndian.Uint32(value), true |
| 96 | - | r.LastUsedSeq = binary.BigEndian.Uint64(value[off+16 : off+24]) | 164 | + | } |
| 97 | - | off += 24 | 165 | + | func (r *valueReader) u64() (uint64, bool) { |
| 98 | - | if len(value)-off < 4 { | 166 | + | value, ok := r.take(8) |
| 99 | - | return Record{}, errMalformedRecord | 167 | + | if !ok { |
| 100 | - | } | 168 | + | return 0, false |
| 101 | - | tabCount32 := binary.BigEndian.Uint32(value[off : off+4]) | 169 | + | } |
| 102 | - | off += 4 | 170 | + | return binary.BigEndian.Uint64(value), true |
| 103 | - | if uint64(tabCount32) > uint64(len(value)-off)/4 { | 171 | + | } |
| 104 | - | return Record{}, errMalformedRecord | 172 | + | func (r *valueReader) str() (string, bool) { |
| 105 | - | } | 173 | + | size, ok := r.u32() |
| 106 | - | if tabCount32 > 0 { | 174 | + | if !ok || uint64(size) > uint64(r.remaining()) { |
| 107 | - | r.TabNames = make([]string, 0, int(tabCount32)) | 175 | + | return "", false |
| 108 | - | } | 176 | + | } |
| 109 | - | for range tabCount32 { | 177 | + | value, _ := r.take(int(size)) |
| 110 | - | if len(value)-off < 4 { | 178 | + | return string(value), true |
| 111 | - | return Record{}, errMalformedRecord | 179 | + | } |
| 112 | - | } | 180 | + | func (r *valueReader) ref() (*domain.CheckpointRef, bool) { |
| 113 | - | nameLen32 := binary.BigEndian.Uint32(value[off : off+4]) | 181 | + | present, ok := r.byte() |
| 114 | - | off += 4 | 182 | + | if !ok { |
| 115 | - | if uint64(nameLen32) > uint64(len(value)-off) { | 183 | + | return nil, false |
| 116 | - | return Record{}, errMalformedRecord | 184 | + | } |
| 117 | - | } | 185 | + | if present == 0 { |
| 118 | - | nameLen := int(nameLen32) | 186 | + | return nil, true |
| 119 | - | r.TabNames = append(r.TabNames, string(value[off:off+nameLen])) | 187 | + | } |
| 120 | - | off += nameLen | 188 | + | if present != 1 { |
| 189 | + | return nil, false | |||
| 190 | + | } | |||
| 191 | + | generation, ok := r.u64() | |||
| 192 | + | if !ok { | |||
| 193 | + | return nil, false | |||
| 121 | } | 194 | } | ||
| 122 | - | if off != len(value) { | 195 | + | digest, ok := r.take(32) |
| 123 | - | return Record{}, errMalformedRecord | 196 | + | if !ok { |
| 197 | + | return nil, false | |||
| 124 | } | 198 | } | ||
| 125 | - | return r, nil | 199 | + | ref := &domain.CheckpointRef{Generation: generation} |
| 200 | + | copy(ref.ManifestDigest[:], digest) | |||
| 201 | + | return ref, true | |||
| 126 | } | 202 | } | ||
M internal/ports/frame.go +1 -1
| @@ -1,7 +1,7 @@ | |||||
| 1 | package ports | 1 | package ports | ||
| 2 | 2 | ||||
| 3 | // ProtocolVersion is the current vev IPC wire protocol version. | 3 | // ProtocolVersion is the current vev IPC wire protocol version. | ||
| 4 | - | const ProtocolVersion uint16 = 18 | 4 | + | const ProtocolVersion uint16 = 19 |
| 5 | 5 | ||||
| 6 | // MsgType identifies the kind of payload carried by a Frame. | 6 | // MsgType identifies the kind of payload carried by a Frame. | ||
| 7 | type MsgType uint8 | 7 | type MsgType uint8 | ||
M internal/ports/frame_test.go +2 -2
| @@ -3,8 +3,8 @@ package ports | |||||
| 3 | import "testing" | 3 | import "testing" | ||
| 4 | 4 | ||||
| 5 | func TestProtocolVersion(t *testing.T) { | 5 | func TestProtocolVersion(t *testing.T) { | ||
| 6 | - | if ProtocolVersion != 18 { | 6 | + | if ProtocolVersion != 19 { |
| 7 | - | t.Fatalf("ProtocolVersion = %d, want 18", ProtocolVersion) | 7 | + | t.Fatalf("ProtocolVersion = %d, want 19", ProtocolVersion) |
| 8 | } | 8 | } | ||
| 9 | } | 9 | } | ||
| 10 | 10 | ||||
A internal/ports/mocks/recovery_mock.go +855 -0
| @@ -0,0 +1,855 @@ | |||||
| 1 | + | // Code generated by mockery; DO NOT EDIT. | |||
| 2 | + | // github.com/vektra/mockery | |||
| 3 | + | // template: testify | |||
| 4 | + | ||||
| 5 | + | package portsmocks | |||
| 6 | + | ||||
| 7 | + | import ( | |||
| 8 | + | "context" | |||
| 9 | + | ||||
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 11 | + | "github.com/bnema/vev/internal/ports" | |||
| 12 | + | mock "github.com/stretchr/testify/mock" | |||
| 13 | + | ) | |||
| 14 | + | ||||
| 15 | + | // NewMockCatalogue creates a new instance of MockCatalogue. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | |||
| 16 | + | // The first argument is typically a *testing.T value. | |||
| 17 | + | func NewMockCatalogue(t interface { | |||
| 18 | + | mock.TestingT | |||
| 19 | + | Cleanup(func()) | |||
| 20 | + | }) *MockCatalogue { | |||
| 21 | + | mock := &MockCatalogue{} | |||
| 22 | + | mock.Mock.Test(t) | |||
| 23 | + | ||||
| 24 | + | t.Cleanup(func() { mock.AssertExpectations(t) }) | |||
| 25 | + | ||||
| 26 | + | return mock | |||
| 27 | + | } | |||
| 28 | + | ||||
| 29 | + | // MockCatalogue is an autogenerated mock type for the Catalogue type | |||
| 30 | + | type MockCatalogue struct { | |||
| 31 | + | mock.Mock | |||
| 32 | + | } | |||
| 33 | + | ||||
| 34 | + | type MockCatalogue_Expecter struct { | |||
| 35 | + | mock *mock.Mock | |||
| 36 | + | } | |||
| 37 | + | ||||
| 38 | + | func (_m *MockCatalogue) EXPECT() *MockCatalogue_Expecter { | |||
| 39 | + | return &MockCatalogue_Expecter{mock: &_m.Mock} | |||
| 40 | + | } | |||
| 41 | + | ||||
| 42 | + | // Close provides a mock function for the type MockCatalogue | |||
| 43 | + | func (_mock *MockCatalogue) Close() error { | |||
| 44 | + | ret := _mock.Called() | |||
| 45 | + | ||||
| 46 | + | if len(ret) == 0 { | |||
| 47 | + | panic("no return value specified for Close") | |||
| 48 | + | } | |||
| 49 | + | ||||
| 50 | + | var r0 error | |||
| 51 | + | if returnFunc, ok := ret.Get(0).(func() error); ok { | |||
| 52 | + | r0 = returnFunc() | |||
| 53 | + | } else { | |||
| 54 | + | r0 = ret.Error(0) | |||
| 55 | + | } | |||
| 56 | + | return r0 | |||
| 57 | + | } | |||
| 58 | + | ||||
| 59 | + | // MockCatalogue_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' | |||
| 60 | + | type MockCatalogue_Close_Call struct { | |||
| 61 | + | *mock.Call | |||
| 62 | + | } | |||
| 63 | + | ||||
| 64 | + | // Close is a helper method to define mock.On call | |||
| 65 | + | func (_e *MockCatalogue_Expecter) Close() *MockCatalogue_Close_Call { | |||
| 66 | + | return &MockCatalogue_Close_Call{Call: _e.mock.On("Close")} | |||
| 67 | + | } | |||
| 68 | + | ||||
| 69 | + | func (_c *MockCatalogue_Close_Call) Run(run func()) *MockCatalogue_Close_Call { | |||
| 70 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 71 | + | run() | |||
| 72 | + | }) | |||
| 73 | + | return _c | |||
| 74 | + | } | |||
| 75 | + | ||||
| 76 | + | func (_c *MockCatalogue_Close_Call) Return(err error) *MockCatalogue_Close_Call { | |||
| 77 | + | _c.Call.Return(err) | |||
| 78 | + | return _c | |||
| 79 | + | } | |||
| 80 | + | ||||
| 81 | + | func (_c *MockCatalogue_Close_Call) RunAndReturn(run func() error) *MockCatalogue_Close_Call { | |||
| 82 | + | _c.Call.Return(run) | |||
| 83 | + | return _c | |||
| 84 | + | } | |||
| 85 | + | ||||
| 86 | + | // Create provides a mock function for the type MockCatalogue | |||
| 87 | + | func (_mock *MockCatalogue) Create(catalogueRecord domain.CatalogueRecord) error { | |||
| 88 | + | ret := _mock.Called(catalogueRecord) | |||
| 89 | + | ||||
| 90 | + | if len(ret) == 0 { | |||
| 91 | + | panic("no return value specified for Create") | |||
| 92 | + | } | |||
| 93 | + | ||||
| 94 | + | var r0 error | |||
| 95 | + | if returnFunc, ok := ret.Get(0).(func(domain.CatalogueRecord) error); ok { | |||
| 96 | + | r0 = returnFunc(catalogueRecord) | |||
| 97 | + | } else { | |||
| 98 | + | r0 = ret.Error(0) | |||
| 99 | + | } | |||
| 100 | + | return r0 | |||
| 101 | + | } | |||
| 102 | + | ||||
| 103 | + | // MockCatalogue_Create_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Create' | |||
| 104 | + | type MockCatalogue_Create_Call struct { | |||
| 105 | + | *mock.Call | |||
| 106 | + | } | |||
| 107 | + | ||||
| 108 | + | // Create is a helper method to define mock.On call | |||
| 109 | + | // - catalogueRecord domain.CatalogueRecord | |||
| 110 | + | func (_e *MockCatalogue_Expecter) Create(catalogueRecord any) *MockCatalogue_Create_Call { | |||
| 111 | + | return &MockCatalogue_Create_Call{Call: _e.mock.On("Create", catalogueRecord)} | |||
| 112 | + | } | |||
| 113 | + | ||||
| 114 | + | func (_c *MockCatalogue_Create_Call) Run(run func(catalogueRecord domain.CatalogueRecord)) *MockCatalogue_Create_Call { | |||
| 115 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 116 | + | var arg0 domain.CatalogueRecord | |||
| 117 | + | if args[0] != nil { | |||
| 118 | + | arg0 = args[0].(domain.CatalogueRecord) | |||
| 119 | + | } | |||
| 120 | + | run( | |||
| 121 | + | arg0, | |||
| 122 | + | ) | |||
| 123 | + | }) | |||
| 124 | + | return _c | |||
| 125 | + | } | |||
| 126 | + | ||||
| 127 | + | func (_c *MockCatalogue_Create_Call) Return(err error) *MockCatalogue_Create_Call { | |||
| 128 | + | _c.Call.Return(err) | |||
| 129 | + | return _c | |||
| 130 | + | } | |||
| 131 | + | ||||
| 132 | + | func (_c *MockCatalogue_Create_Call) RunAndReturn(run func(catalogueRecord domain.CatalogueRecord) error) *MockCatalogue_Create_Call { | |||
| 133 | + | _c.Call.Return(run) | |||
| 134 | + | return _c | |||
| 135 | + | } | |||
| 136 | + | ||||
| 137 | + | // Delete provides a mock function for the type MockCatalogue | |||
| 138 | + | func (_mock *MockCatalogue) Delete(s string) error { | |||
| 139 | + | ret := _mock.Called(s) | |||
| 140 | + | ||||
| 141 | + | if len(ret) == 0 { | |||
| 142 | + | panic("no return value specified for Delete") | |||
| 143 | + | } | |||
| 144 | + | ||||
| 145 | + | var r0 error | |||
| 146 | + | if returnFunc, ok := ret.Get(0).(func(string) error); ok { | |||
| 147 | + | r0 = returnFunc(s) | |||
| 148 | + | } else { | |||
| 149 | + | r0 = ret.Error(0) | |||
| 150 | + | } | |||
| 151 | + | return r0 | |||
| 152 | + | } | |||
| 153 | + | ||||
| 154 | + | // MockCatalogue_Delete_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Delete' | |||
| 155 | + | type MockCatalogue_Delete_Call struct { | |||
| 156 | + | *mock.Call | |||
| 157 | + | } | |||
| 158 | + | ||||
| 159 | + | // Delete is a helper method to define mock.On call | |||
| 160 | + | // - s string | |||
| 161 | + | func (_e *MockCatalogue_Expecter) Delete(s any) *MockCatalogue_Delete_Call { | |||
| 162 | + | return &MockCatalogue_Delete_Call{Call: _e.mock.On("Delete", s)} | |||
| 163 | + | } | |||
| 164 | + | ||||
| 165 | + | func (_c *MockCatalogue_Delete_Call) Run(run func(s string)) *MockCatalogue_Delete_Call { | |||
| 166 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 167 | + | var arg0 string | |||
| 168 | + | if args[0] != nil { | |||
| 169 | + | arg0 = args[0].(string) | |||
| 170 | + | } | |||
| 171 | + | run( | |||
| 172 | + | arg0, | |||
| 173 | + | ) | |||
| 174 | + | }) | |||
| 175 | + | return _c | |||
| 176 | + | } | |||
| 177 | + | ||||
| 178 | + | func (_c *MockCatalogue_Delete_Call) Return(err error) *MockCatalogue_Delete_Call { | |||
| 179 | + | _c.Call.Return(err) | |||
| 180 | + | return _c | |||
| 181 | + | } | |||
| 182 | + | ||||
| 183 | + | func (_c *MockCatalogue_Delete_Call) RunAndReturn(run func(s string) error) *MockCatalogue_Delete_Call { | |||
| 184 | + | _c.Call.Return(run) | |||
| 185 | + | return _c | |||
| 186 | + | } | |||
| 187 | + | ||||
| 188 | + | // Record provides a mock function for the type MockCatalogue | |||
| 189 | + | func (_mock *MockCatalogue) Record(s string) (domain.CatalogueRecord, bool, error) { | |||
| 190 | + | ret := _mock.Called(s) | |||
| 191 | + | ||||
| 192 | + | if len(ret) == 0 { | |||
| 193 | + | panic("no return value specified for Record") | |||
| 194 | + | } | |||
| 195 | + | ||||
| 196 | + | var r0 domain.CatalogueRecord | |||
| 197 | + | var r1 bool | |||
| 198 | + | var r2 error | |||
| 199 | + | if returnFunc, ok := ret.Get(0).(func(string) (domain.CatalogueRecord, bool, error)); ok { | |||
| 200 | + | return returnFunc(s) | |||
| 201 | + | } | |||
| 202 | + | if returnFunc, ok := ret.Get(0).(func(string) domain.CatalogueRecord); ok { | |||
| 203 | + | r0 = returnFunc(s) | |||
| 204 | + | } else { | |||
| 205 | + | r0 = ret.Get(0).(domain.CatalogueRecord) | |||
| 206 | + | } | |||
| 207 | + | if returnFunc, ok := ret.Get(1).(func(string) bool); ok { | |||
| 208 | + | r1 = returnFunc(s) | |||
| 209 | + | } else { | |||
| 210 | + | r1 = ret.Get(1).(bool) | |||
| 211 | + | } | |||
| 212 | + | if returnFunc, ok := ret.Get(2).(func(string) error); ok { | |||
| 213 | + | r2 = returnFunc(s) | |||
| 214 | + | } else { | |||
| 215 | + | r2 = ret.Error(2) | |||
| 216 | + | } | |||
| 217 | + | return r0, r1, r2 | |||
| 218 | + | } | |||
| 219 | + | ||||
| 220 | + | // MockCatalogue_Record_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Record' | |||
| 221 | + | type MockCatalogue_Record_Call struct { | |||
| 222 | + | *mock.Call | |||
| 223 | + | } | |||
| 224 | + | ||||
| 225 | + | // Record is a helper method to define mock.On call | |||
| 226 | + | // - s string | |||
| 227 | + | func (_e *MockCatalogue_Expecter) Record(s any) *MockCatalogue_Record_Call { | |||
| 228 | + | return &MockCatalogue_Record_Call{Call: _e.mock.On("Record", s)} | |||
| 229 | + | } | |||
| 230 | + | ||||
| 231 | + | func (_c *MockCatalogue_Record_Call) Run(run func(s string)) *MockCatalogue_Record_Call { | |||
| 232 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 233 | + | var arg0 string | |||
| 234 | + | if args[0] != nil { | |||
| 235 | + | arg0 = args[0].(string) | |||
| 236 | + | } | |||
| 237 | + | run( | |||
| 238 | + | arg0, | |||
| 239 | + | ) | |||
| 240 | + | }) | |||
| 241 | + | return _c | |||
| 242 | + | } | |||
| 243 | + | ||||
| 244 | + | func (_c *MockCatalogue_Record_Call) Return(catalogueRecord domain.CatalogueRecord, b bool, err error) *MockCatalogue_Record_Call { | |||
| 245 | + | _c.Call.Return(catalogueRecord, b, err) | |||
| 246 | + | return _c | |||
| 247 | + | } | |||
| 248 | + | ||||
| 249 | + | func (_c *MockCatalogue_Record_Call) RunAndReturn(run func(s string) (domain.CatalogueRecord, bool, error)) *MockCatalogue_Record_Call { | |||
| 250 | + | _c.Call.Return(run) | |||
| 251 | + | return _c | |||
| 252 | + | } | |||
| 253 | + | ||||
| 254 | + | // Records provides a mock function for the type MockCatalogue | |||
| 255 | + | func (_mock *MockCatalogue) Records() ([]domain.CatalogueRecord, error) { | |||
| 256 | + | ret := _mock.Called() | |||
| 257 | + | ||||
| 258 | + | if len(ret) == 0 { | |||
| 259 | + | panic("no return value specified for Records") | |||
| 260 | + | } | |||
| 261 | + | ||||
| 262 | + | var r0 []domain.CatalogueRecord | |||
| 263 | + | var r1 error | |||
| 264 | + | if returnFunc, ok := ret.Get(0).(func() ([]domain.CatalogueRecord, error)); ok { | |||
| 265 | + | return returnFunc() | |||
| 266 | + | } | |||
| 267 | + | if returnFunc, ok := ret.Get(0).(func() []domain.CatalogueRecord); ok { | |||
| 268 | + | r0 = returnFunc() | |||
| 269 | + | } else { | |||
| 270 | + | if ret.Get(0) != nil { | |||
| 271 | + | r0 = ret.Get(0).([]domain.CatalogueRecord) | |||
| 272 | + | } | |||
| 273 | + | } | |||
| 274 | + | if returnFunc, ok := ret.Get(1).(func() error); ok { | |||
| 275 | + | r1 = returnFunc() | |||
| 276 | + | } else { | |||
| 277 | + | r1 = ret.Error(1) | |||
| 278 | + | } | |||
| 279 | + | return r0, r1 | |||
| 280 | + | } | |||
| 281 | + | ||||
| 282 | + | // MockCatalogue_Records_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Records' | |||
| 283 | + | type MockCatalogue_Records_Call struct { | |||
| 284 | + | *mock.Call | |||
| 285 | + | } | |||
| 286 | + | ||||
| 287 | + | // Records is a helper method to define mock.On call | |||
| 288 | + | func (_e *MockCatalogue_Expecter) Records() *MockCatalogue_Records_Call { | |||
| 289 | + | return &MockCatalogue_Records_Call{Call: _e.mock.On("Records")} | |||
| 290 | + | } | |||
| 291 | + | ||||
| 292 | + | func (_c *MockCatalogue_Records_Call) Run(run func()) *MockCatalogue_Records_Call { | |||
| 293 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 294 | + | run() | |||
| 295 | + | }) | |||
| 296 | + | return _c | |||
| 297 | + | } | |||
| 298 | + | ||||
| 299 | + | func (_c *MockCatalogue_Records_Call) Return(catalogueRecords []domain.CatalogueRecord, err error) *MockCatalogue_Records_Call { | |||
| 300 | + | _c.Call.Return(catalogueRecords, err) | |||
| 301 | + | return _c | |||
| 302 | + | } | |||
| 303 | + | ||||
| 304 | + | func (_c *MockCatalogue_Records_Call) RunAndReturn(run func() ([]domain.CatalogueRecord, error)) *MockCatalogue_Records_Call { | |||
| 305 | + | _c.Call.Return(run) | |||
| 306 | + | return _c | |||
| 307 | + | } | |||
| 308 | + | ||||
| 309 | + | // Rename provides a mock function for the type MockCatalogue | |||
| 310 | + | func (_mock *MockCatalogue) Rename(s string, catalogueRecord domain.CatalogueRecord) error { | |||
| 311 | + | ret := _mock.Called(s, catalogueRecord) | |||
| 312 | + | ||||
| 313 | + | if len(ret) == 0 { | |||
| 314 | + | panic("no return value specified for Rename") | |||
| 315 | + | } | |||
| 316 | + | ||||
| 317 | + | var r0 error | |||
| 318 | + | if returnFunc, ok := ret.Get(0).(func(string, domain.CatalogueRecord) error); ok { | |||
| 319 | + | r0 = returnFunc(s, catalogueRecord) | |||
| 320 | + | } else { | |||
| 321 | + | r0 = ret.Error(0) | |||
| 322 | + | } | |||
| 323 | + | return r0 | |||
| 324 | + | } | |||
| 325 | + | ||||
| 326 | + | // MockCatalogue_Rename_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Rename' | |||
| 327 | + | type MockCatalogue_Rename_Call struct { | |||
| 328 | + | *mock.Call | |||
| 329 | + | } | |||
| 330 | + | ||||
| 331 | + | // Rename is a helper method to define mock.On call | |||
| 332 | + | // - s string | |||
| 333 | + | // - catalogueRecord domain.CatalogueRecord | |||
| 334 | + | func (_e *MockCatalogue_Expecter) Rename(s any, catalogueRecord any) *MockCatalogue_Rename_Call { | |||
| 335 | + | return &MockCatalogue_Rename_Call{Call: _e.mock.On("Rename", s, catalogueRecord)} | |||
| 336 | + | } | |||
| 337 | + | ||||
| 338 | + | func (_c *MockCatalogue_Rename_Call) Run(run func(s string, catalogueRecord domain.CatalogueRecord)) *MockCatalogue_Rename_Call { | |||
| 339 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 340 | + | var arg0 string | |||
| 341 | + | if args[0] != nil { | |||
| 342 | + | arg0 = args[0].(string) | |||
| 343 | + | } | |||
| 344 | + | var arg1 domain.CatalogueRecord | |||
| 345 | + | if args[1] != nil { | |||
| 346 | + | arg1 = args[1].(domain.CatalogueRecord) | |||
| 347 | + | } | |||
| 348 | + | run( | |||
| 349 | + | arg0, | |||
| 350 | + | arg1, | |||
| 351 | + | ) | |||
| 352 | + | }) | |||
| 353 | + | return _c | |||
| 354 | + | } | |||
| 355 | + | ||||
| 356 | + | func (_c *MockCatalogue_Rename_Call) Return(err error) *MockCatalogue_Rename_Call { | |||
| 357 | + | _c.Call.Return(err) | |||
| 358 | + | return _c | |||
| 359 | + | } | |||
| 360 | + | ||||
| 361 | + | func (_c *MockCatalogue_Rename_Call) RunAndReturn(run func(s string, catalogueRecord domain.CatalogueRecord) error) *MockCatalogue_Rename_Call { | |||
| 362 | + | _c.Call.Return(run) | |||
| 363 | + | return _c | |||
| 364 | + | } | |||
| 365 | + | ||||
| 366 | + | // Replace provides a mock function for the type MockCatalogue | |||
| 367 | + | func (_mock *MockCatalogue) Replace(s string, catalogueRecord domain.CatalogueRecord) error { | |||
| 368 | + | ret := _mock.Called(s, catalogueRecord) | |||
| 369 | + | ||||
| 370 | + | if len(ret) == 0 { | |||
| 371 | + | panic("no return value specified for Replace") | |||
| 372 | + | } | |||
| 373 | + | ||||
| 374 | + | var r0 error | |||
| 375 | + | if returnFunc, ok := ret.Get(0).(func(string, domain.CatalogueRecord) error); ok { | |||
| 376 | + | r0 = returnFunc(s, catalogueRecord) | |||
| 377 | + | } else { | |||
| 378 | + | r0 = ret.Error(0) | |||
| 379 | + | } | |||
| 380 | + | return r0 | |||
| 381 | + | } | |||
| 382 | + | ||||
| 383 | + | // MockCatalogue_Replace_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Replace' | |||
| 384 | + | type MockCatalogue_Replace_Call struct { | |||
| 385 | + | *mock.Call | |||
| 386 | + | } | |||
| 387 | + | ||||
| 388 | + | // Replace is a helper method to define mock.On call | |||
| 389 | + | // - s string | |||
| 390 | + | // - catalogueRecord domain.CatalogueRecord | |||
| 391 | + | func (_e *MockCatalogue_Expecter) Replace(s any, catalogueRecord any) *MockCatalogue_Replace_Call { | |||
| 392 | + | return &MockCatalogue_Replace_Call{Call: _e.mock.On("Replace", s, catalogueRecord)} | |||
| 393 | + | } | |||
| 394 | + | ||||
| 395 | + | func (_c *MockCatalogue_Replace_Call) Run(run func(s string, catalogueRecord domain.CatalogueRecord)) *MockCatalogue_Replace_Call { | |||
| 396 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 397 | + | var arg0 string | |||
| 398 | + | if args[0] != nil { | |||
| 399 | + | arg0 = args[0].(string) | |||
| 400 | + | } | |||
| 401 | + | var arg1 domain.CatalogueRecord | |||
| 402 | + | if args[1] != nil { | |||
| 403 | + | arg1 = args[1].(domain.CatalogueRecord) | |||
| 404 | + | } | |||
| 405 | + | run( | |||
| 406 | + | arg0, | |||
| 407 | + | arg1, | |||
| 408 | + | ) | |||
| 409 | + | }) | |||
| 410 | + | return _c | |||
| 411 | + | } | |||
| 412 | + | ||||
| 413 | + | func (_c *MockCatalogue_Replace_Call) Return(err error) *MockCatalogue_Replace_Call { | |||
| 414 | + | _c.Call.Return(err) | |||
| 415 | + | return _c | |||
| 416 | + | } | |||
| 417 | + | ||||
| 418 | + | func (_c *MockCatalogue_Replace_Call) RunAndReturn(run func(s string, catalogueRecord domain.CatalogueRecord) error) *MockCatalogue_Replace_Call { | |||
| 419 | + | _c.Call.Return(run) | |||
| 420 | + | return _c | |||
| 421 | + | } | |||
| 422 | + | ||||
| 423 | + | // Sync provides a mock function for the type MockCatalogue | |||
| 424 | + | func (_mock *MockCatalogue) Sync() error { | |||
| 425 | + | ret := _mock.Called() | |||
| 426 | + | ||||
| 427 | + | if len(ret) == 0 { | |||
| 428 | + | panic("no return value specified for Sync") | |||
| 429 | + | } | |||
| 430 | + | ||||
| 431 | + | var r0 error | |||
| 432 | + | if returnFunc, ok := ret.Get(0).(func() error); ok { | |||
| 433 | + | r0 = returnFunc() | |||
| 434 | + | } else { | |||
| 435 | + | r0 = ret.Error(0) | |||
| 436 | + | } | |||
| 437 | + | return r0 | |||
| 438 | + | } | |||
| 439 | + | ||||
| 440 | + | // MockCatalogue_Sync_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Sync' | |||
| 441 | + | type MockCatalogue_Sync_Call struct { | |||
| 442 | + | *mock.Call | |||
| 443 | + | } | |||
| 444 | + | ||||
| 445 | + | // Sync is a helper method to define mock.On call | |||
| 446 | + | func (_e *MockCatalogue_Expecter) Sync() *MockCatalogue_Sync_Call { | |||
| 447 | + | return &MockCatalogue_Sync_Call{Call: _e.mock.On("Sync")} | |||
| 448 | + | } | |||
| 449 | + | ||||
| 450 | + | func (_c *MockCatalogue_Sync_Call) Run(run func()) *MockCatalogue_Sync_Call { | |||
| 451 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 452 | + | run() | |||
| 453 | + | }) | |||
| 454 | + | return _c | |||
| 455 | + | } | |||
| 456 | + | ||||
| 457 | + | func (_c *MockCatalogue_Sync_Call) Return(err error) *MockCatalogue_Sync_Call { | |||
| 458 | + | _c.Call.Return(err) | |||
| 459 | + | return _c | |||
| 460 | + | } | |||
| 461 | + | ||||
| 462 | + | func (_c *MockCatalogue_Sync_Call) RunAndReturn(run func() error) *MockCatalogue_Sync_Call { | |||
| 463 | + | _c.Call.Return(run) | |||
| 464 | + | return _c | |||
| 465 | + | } | |||
| 466 | + | ||||
| 467 | + | // UpdateMetadata provides a mock function for the type MockCatalogue | |||
| 468 | + | func (_mock *MockCatalogue) UpdateMetadata(catalogueMetadataUpdate domain.CatalogueMetadataUpdate) error { | |||
| 469 | + | ret := _mock.Called(catalogueMetadataUpdate) | |||
| 470 | + | ||||
| 471 | + | if len(ret) == 0 { | |||
| 472 | + | panic("no return value specified for UpdateMetadata") | |||
| 473 | + | } | |||
| 474 | + | ||||
| 475 | + | var r0 error | |||
| 476 | + | if returnFunc, ok := ret.Get(0).(func(domain.CatalogueMetadataUpdate) error); ok { | |||
| 477 | + | r0 = returnFunc(catalogueMetadataUpdate) | |||
| 478 | + | } else { | |||
| 479 | + | r0 = ret.Error(0) | |||
| 480 | + | } | |||
| 481 | + | return r0 | |||
| 482 | + | } | |||
| 483 | + | ||||
| 484 | + | // MockCatalogue_UpdateMetadata_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateMetadata' | |||
| 485 | + | type MockCatalogue_UpdateMetadata_Call struct { | |||
| 486 | + | *mock.Call | |||
| 487 | + | } | |||
| 488 | + | ||||
| 489 | + | // UpdateMetadata is a helper method to define mock.On call | |||
| 490 | + | // - catalogueMetadataUpdate domain.CatalogueMetadataUpdate | |||
| 491 | + | func (_e *MockCatalogue_Expecter) UpdateMetadata(catalogueMetadataUpdate any) *MockCatalogue_UpdateMetadata_Call { | |||
| 492 | + | return &MockCatalogue_UpdateMetadata_Call{Call: _e.mock.On("UpdateMetadata", catalogueMetadataUpdate)} | |||
| 493 | + | } | |||
| 494 | + | ||||
| 495 | + | func (_c *MockCatalogue_UpdateMetadata_Call) Run(run func(catalogueMetadataUpdate domain.CatalogueMetadataUpdate)) *MockCatalogue_UpdateMetadata_Call { | |||
| 496 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 497 | + | var arg0 domain.CatalogueMetadataUpdate | |||
| 498 | + | if args[0] != nil { | |||
| 499 | + | arg0 = args[0].(domain.CatalogueMetadataUpdate) | |||
| 500 | + | } | |||
| 501 | + | run( | |||
| 502 | + | arg0, | |||
| 503 | + | ) | |||
| 504 | + | }) | |||
| 505 | + | return _c | |||
| 506 | + | } | |||
| 507 | + | ||||
| 508 | + | func (_c *MockCatalogue_UpdateMetadata_Call) Return(err error) *MockCatalogue_UpdateMetadata_Call { | |||
| 509 | + | _c.Call.Return(err) | |||
| 510 | + | return _c | |||
| 511 | + | } | |||
| 512 | + | ||||
| 513 | + | func (_c *MockCatalogue_UpdateMetadata_Call) RunAndReturn(run func(catalogueMetadataUpdate domain.CatalogueMetadataUpdate) error) *MockCatalogue_UpdateMetadata_Call { | |||
| 514 | + | _c.Call.Return(run) | |||
| 515 | + | return _c | |||
| 516 | + | } | |||
| 517 | + | ||||
| 518 | + | // NewMockSnapshotRepository creates a new instance of MockSnapshotRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | |||
| 519 | + | // The first argument is typically a *testing.T value. | |||
| 520 | + | func NewMockSnapshotRepository(t interface { | |||
| 521 | + | mock.TestingT | |||
| 522 | + | Cleanup(func()) | |||
| 523 | + | }) *MockSnapshotRepository { | |||
| 524 | + | mock := &MockSnapshotRepository{} | |||
| 525 | + | mock.Mock.Test(t) | |||
| 526 | + | ||||
| 527 | + | t.Cleanup(func() { mock.AssertExpectations(t) }) | |||
| 528 | + | ||||
| 529 | + | return mock | |||
| 530 | + | } | |||
| 531 | + | ||||
| 532 | + | // MockSnapshotRepository is an autogenerated mock type for the SnapshotRepository type | |||
| 533 | + | type MockSnapshotRepository struct { | |||
| 534 | + | mock.Mock | |||
| 535 | + | } | |||
| 536 | + | ||||
| 537 | + | type MockSnapshotRepository_Expecter struct { | |||
| 538 | + | mock *mock.Mock | |||
| 539 | + | } | |||
| 540 | + | ||||
| 541 | + | func (_m *MockSnapshotRepository) EXPECT() *MockSnapshotRepository_Expecter { | |||
| 542 | + | return &MockSnapshotRepository_Expecter{mock: &_m.Mock} | |||
| 543 | + | } | |||
| 544 | + | ||||
| 545 | + | // CollectGarbage provides a mock function for the type MockSnapshotRepository | |||
| 546 | + | func (_mock *MockSnapshotRepository) CollectGarbage(context1 context.Context, incarnationIDToCheckpointRef map[domain.IncarnationID]domain.CheckpointRef) error { | |||
| 547 | + | ret := _mock.Called(context1, incarnationIDToCheckpointRef) | |||
| 548 | + | ||||
| 549 | + | if len(ret) == 0 { | |||
| 550 | + | panic("no return value specified for CollectGarbage") | |||
| 551 | + | } | |||
| 552 | + | ||||
| 553 | + | var r0 error | |||
| 554 | + | if returnFunc, ok := ret.Get(0).(func(context.Context, map[domain.IncarnationID]domain.CheckpointRef) error); ok { | |||
| 555 | + | r0 = returnFunc(context1, incarnationIDToCheckpointRef) | |||
| 556 | + | } else { | |||
| 557 | + | r0 = ret.Error(0) | |||
| 558 | + | } | |||
| 559 | + | return r0 | |||
| 560 | + | } | |||
| 561 | + | ||||
| 562 | + | // MockSnapshotRepository_CollectGarbage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CollectGarbage' | |||
| 563 | + | type MockSnapshotRepository_CollectGarbage_Call struct { | |||
| 564 | + | *mock.Call | |||
| 565 | + | } | |||
| 566 | + | ||||
| 567 | + | // CollectGarbage is a helper method to define mock.On call | |||
| 568 | + | // - context1 context.Context | |||
| 569 | + | // - incarnationIDToCheckpointRef map[domain.IncarnationID]domain.CheckpointRef | |||
| 570 | + | func (_e *MockSnapshotRepository_Expecter) CollectGarbage(context1 any, incarnationIDToCheckpointRef any) *MockSnapshotRepository_CollectGarbage_Call { | |||
| 571 | + | return &MockSnapshotRepository_CollectGarbage_Call{Call: _e.mock.On("CollectGarbage", context1, incarnationIDToCheckpointRef)} | |||
| 572 | + | } | |||
| 573 | + | ||||
| 574 | + | func (_c *MockSnapshotRepository_CollectGarbage_Call) Run(run func(context1 context.Context, incarnationIDToCheckpointRef map[domain.IncarnationID]domain.CheckpointRef)) *MockSnapshotRepository_CollectGarbage_Call { | |||
| 575 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 576 | + | var arg0 context.Context | |||
| 577 | + | if args[0] != nil { | |||
| 578 | + | arg0 = args[0].(context.Context) | |||
| 579 | + | } | |||
| 580 | + | var arg1 map[domain.IncarnationID]domain.CheckpointRef | |||
| 581 | + | if args[1] != nil { | |||
| 582 | + | arg1 = args[1].(map[domain.IncarnationID]domain.CheckpointRef) | |||
| 583 | + | } | |||
| 584 | + | run( | |||
| 585 | + | arg0, | |||
| 586 | + | arg1, | |||
| 587 | + | ) | |||
| 588 | + | }) | |||
| 589 | + | return _c | |||
| 590 | + | } | |||
| 591 | + | ||||
| 592 | + | func (_c *MockSnapshotRepository_CollectGarbage_Call) Return(err error) *MockSnapshotRepository_CollectGarbage_Call { | |||
| 593 | + | _c.Call.Return(err) | |||
| 594 | + | return _c | |||
| 595 | + | } | |||
| 596 | + | ||||
| 597 | + | func (_c *MockSnapshotRepository_CollectGarbage_Call) RunAndReturn(run func(context1 context.Context, incarnationIDToCheckpointRef map[domain.IncarnationID]domain.CheckpointRef) error) *MockSnapshotRepository_CollectGarbage_Call { | |||
| 598 | + | _c.Call.Return(run) | |||
| 599 | + | return _c | |||
| 600 | + | } | |||
| 601 | + | ||||
| 602 | + | // DeleteIncarnation provides a mock function for the type MockSnapshotRepository | |||
| 603 | + | func (_mock *MockSnapshotRepository) DeleteIncarnation(context1 context.Context, incarnationID domain.IncarnationID) error { | |||
| 604 | + | ret := _mock.Called(context1, incarnationID) | |||
| 605 | + | ||||
| 606 | + | if len(ret) == 0 { | |||
| 607 | + | panic("no return value specified for DeleteIncarnation") | |||
| 608 | + | } | |||
| 609 | + | ||||
| 610 | + | var r0 error | |||
| 611 | + | if returnFunc, ok := ret.Get(0).(func(context.Context, domain.IncarnationID) error); ok { | |||
| 612 | + | r0 = returnFunc(context1, incarnationID) | |||
| 613 | + | } else { | |||
| 614 | + | r0 = ret.Error(0) | |||
| 615 | + | } | |||
| 616 | + | return r0 | |||
| 617 | + | } | |||
| 618 | + | ||||
| 619 | + | // MockSnapshotRepository_DeleteIncarnation_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteIncarnation' | |||
| 620 | + | type MockSnapshotRepository_DeleteIncarnation_Call struct { | |||
| 621 | + | *mock.Call | |||
| 622 | + | } | |||
| 623 | + | ||||
| 624 | + | // DeleteIncarnation is a helper method to define mock.On call | |||
| 625 | + | // - context1 context.Context | |||
| 626 | + | // - incarnationID domain.IncarnationID | |||
| 627 | + | func (_e *MockSnapshotRepository_Expecter) DeleteIncarnation(context1 any, incarnationID any) *MockSnapshotRepository_DeleteIncarnation_Call { | |||
| 628 | + | return &MockSnapshotRepository_DeleteIncarnation_Call{Call: _e.mock.On("DeleteIncarnation", context1, incarnationID)} | |||
| 629 | + | } | |||
| 630 | + | ||||
| 631 | + | func (_c *MockSnapshotRepository_DeleteIncarnation_Call) Run(run func(context1 context.Context, incarnationID domain.IncarnationID)) *MockSnapshotRepository_DeleteIncarnation_Call { | |||
| 632 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 633 | + | var arg0 context.Context | |||
| 634 | + | if args[0] != nil { | |||
| 635 | + | arg0 = args[0].(context.Context) | |||
| 636 | + | } | |||
| 637 | + | var arg1 domain.IncarnationID | |||
| 638 | + | if args[1] != nil { | |||
| 639 | + | arg1 = args[1].(domain.IncarnationID) | |||
| 640 | + | } | |||
| 641 | + | run( | |||
| 642 | + | arg0, | |||
| 643 | + | arg1, | |||
| 644 | + | ) | |||
| 645 | + | }) | |||
| 646 | + | return _c | |||
| 647 | + | } | |||
| 648 | + | ||||
| 649 | + | func (_c *MockSnapshotRepository_DeleteIncarnation_Call) Return(err error) *MockSnapshotRepository_DeleteIncarnation_Call { | |||
| 650 | + | _c.Call.Return(err) | |||
| 651 | + | return _c | |||
| 652 | + | } | |||
| 653 | + | ||||
| 654 | + | func (_c *MockSnapshotRepository_DeleteIncarnation_Call) RunAndReturn(run func(context1 context.Context, incarnationID domain.IncarnationID) error) *MockSnapshotRepository_DeleteIncarnation_Call { | |||
| 655 | + | _c.Call.Return(run) | |||
| 656 | + | return _c | |||
| 657 | + | } | |||
| 658 | + | ||||
| 659 | + | // LoadCheckpoint provides a mock function for the type MockSnapshotRepository | |||
| 660 | + | func (_mock *MockSnapshotRepository) LoadCheckpoint(context1 context.Context, incarnationID domain.IncarnationID, s string, v ports.CheckpointRef) (ports.SnapshotGeneration, error) { | |||
| 661 | + | ret := _mock.Called(context1, incarnationID, s, v) | |||
| 662 | + | ||||
| 663 | + | if len(ret) == 0 { | |||
| 664 | + | panic("no return value specified for LoadCheckpoint") | |||
| 665 | + | } | |||
| 666 | + | ||||
| 667 | + | var r0 ports.SnapshotGeneration | |||
| 668 | + | var r1 error | |||
| 669 | + | if returnFunc, ok := ret.Get(0).(func(context.Context, domain.IncarnationID, string, ports.CheckpointRef) (ports.SnapshotGeneration, error)); ok { | |||
| 670 | + | return returnFunc(context1, incarnationID, s, v) | |||
| 671 | + | } | |||
| 672 | + | if returnFunc, ok := ret.Get(0).(func(context.Context, domain.IncarnationID, string, ports.CheckpointRef) ports.SnapshotGeneration); ok { | |||
| 673 | + | r0 = returnFunc(context1, incarnationID, s, v) | |||
| 674 | + | } else { | |||
| 675 | + | r0 = ret.Get(0).(ports.SnapshotGeneration) | |||
| 676 | + | } | |||
| 677 | + | if returnFunc, ok := ret.Get(1).(func(context.Context, domain.IncarnationID, string, ports.CheckpointRef) error); ok { | |||
| 678 | + | r1 = returnFunc(context1, incarnationID, s, v) | |||
| 679 | + | } else { | |||
| 680 | + | r1 = ret.Error(1) | |||
| 681 | + | } | |||
| 682 | + | return r0, r1 | |||
| 683 | + | } | |||
| 684 | + | ||||
| 685 | + | // MockSnapshotRepository_LoadCheckpoint_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LoadCheckpoint' | |||
| 686 | + | type MockSnapshotRepository_LoadCheckpoint_Call struct { | |||
| 687 | + | *mock.Call | |||
| 688 | + | } | |||
| 689 | + | ||||
| 690 | + | // LoadCheckpoint is a helper method to define mock.On call | |||
| 691 | + | // - context1 context.Context | |||
| 692 | + | // - incarnationID domain.IncarnationID | |||
| 693 | + | // - s string | |||
| 694 | + | // - v ports.CheckpointRef | |||
| 695 | + | func (_e *MockSnapshotRepository_Expecter) LoadCheckpoint(context1 any, incarnationID any, s any, v any) *MockSnapshotRepository_LoadCheckpoint_Call { | |||
| 696 | + | return &MockSnapshotRepository_LoadCheckpoint_Call{Call: _e.mock.On("LoadCheckpoint", context1, incarnationID, s, v)} | |||
| 697 | + | } | |||
| 698 | + | ||||
| 699 | + | func (_c *MockSnapshotRepository_LoadCheckpoint_Call) Run(run func(context1 context.Context, incarnationID domain.IncarnationID, s string, v ports.CheckpointRef)) *MockSnapshotRepository_LoadCheckpoint_Call { | |||
| 700 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 701 | + | var arg0 context.Context | |||
| 702 | + | if args[0] != nil { | |||
| 703 | + | arg0 = args[0].(context.Context) | |||
| 704 | + | } | |||
| 705 | + | var arg1 domain.IncarnationID | |||
| 706 | + | if args[1] != nil { | |||
| 707 | + | arg1 = args[1].(domain.IncarnationID) | |||
| 708 | + | } | |||
| 709 | + | var arg2 string | |||
| 710 | + | if args[2] != nil { | |||
| 711 | + | arg2 = args[2].(string) | |||
| 712 | + | } | |||
| 713 | + | var arg3 ports.CheckpointRef | |||
| 714 | + | if args[3] != nil { | |||
| 715 | + | arg3 = args[3].(ports.CheckpointRef) | |||
| 716 | + | } | |||
| 717 | + | run( | |||
| 718 | + | arg0, | |||
| 719 | + | arg1, | |||
| 720 | + | arg2, | |||
| 721 | + | arg3, | |||
| 722 | + | ) | |||
| 723 | + | }) | |||
| 724 | + | return _c | |||
| 725 | + | } | |||
| 726 | + | ||||
| 727 | + | func (_c *MockSnapshotRepository_LoadCheckpoint_Call) Return(snapshotGeneration ports.SnapshotGeneration, err error) *MockSnapshotRepository_LoadCheckpoint_Call { | |||
| 728 | + | _c.Call.Return(snapshotGeneration, err) | |||
| 729 | + | return _c | |||
| 730 | + | } | |||
| 731 | + | ||||
| 732 | + | func (_c *MockSnapshotRepository_LoadCheckpoint_Call) RunAndReturn(run func(context1 context.Context, incarnationID domain.IncarnationID, s string, v ports.CheckpointRef) (ports.SnapshotGeneration, error)) *MockSnapshotRepository_LoadCheckpoint_Call { | |||
| 733 | + | _c.Call.Return(run) | |||
| 734 | + | return _c | |||
| 735 | + | } | |||
| 736 | + | ||||
| 737 | + | // Publish provides a mock function for the type MockSnapshotRepository | |||
| 738 | + | func (_mock *MockSnapshotRepository) Publish(context1 context.Context, snapshotPublication ports.SnapshotPublication) error { | |||
| 739 | + | ret := _mock.Called(context1, snapshotPublication) | |||
| 740 | + | ||||
| 741 | + | if len(ret) == 0 { | |||
| 742 | + | panic("no return value specified for Publish") | |||
| 743 | + | } | |||
| 744 | + | ||||
| 745 | + | var r0 error | |||
| 746 | + | if returnFunc, ok := ret.Get(0).(func(context.Context, ports.SnapshotPublication) error); ok { | |||
| 747 | + | r0 = returnFunc(context1, snapshotPublication) | |||
| 748 | + | } else { | |||
| 749 | + | r0 = ret.Error(0) | |||
| 750 | + | } | |||
| 751 | + | return r0 | |||
| 752 | + | } | |||
| 753 | + | ||||
| 754 | + | // MockSnapshotRepository_Publish_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Publish' | |||
| 755 | + | type MockSnapshotRepository_Publish_Call struct { | |||
| 756 | + | *mock.Call | |||
| 757 | + | } | |||
| 758 | + | ||||
| 759 | + | // Publish is a helper method to define mock.On call | |||
| 760 | + | // - context1 context.Context | |||
| 761 | + | // - snapshotPublication ports.SnapshotPublication | |||
| 762 | + | func (_e *MockSnapshotRepository_Expecter) Publish(context1 any, snapshotPublication any) *MockSnapshotRepository_Publish_Call { | |||
| 763 | + | return &MockSnapshotRepository_Publish_Call{Call: _e.mock.On("Publish", context1, snapshotPublication)} | |||
| 764 | + | } | |||
| 765 | + | ||||
| 766 | + | func (_c *MockSnapshotRepository_Publish_Call) Run(run func(context1 context.Context, snapshotPublication ports.SnapshotPublication)) *MockSnapshotRepository_Publish_Call { | |||
| 767 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 768 | + | var arg0 context.Context | |||
| 769 | + | if args[0] != nil { | |||
| 770 | + | arg0 = args[0].(context.Context) | |||
| 771 | + | } | |||
| 772 | + | var arg1 ports.SnapshotPublication | |||
| 773 | + | if args[1] != nil { | |||
| 774 | + | arg1 = args[1].(ports.SnapshotPublication) | |||
| 775 | + | } | |||
| 776 | + | run( | |||
| 777 | + | arg0, | |||
| 778 | + | arg1, | |||
| 779 | + | ) | |||
| 780 | + | }) | |||
| 781 | + | return _c | |||
| 782 | + | } | |||
| 783 | + | ||||
| 784 | + | func (_c *MockSnapshotRepository_Publish_Call) Return(err error) *MockSnapshotRepository_Publish_Call { | |||
| 785 | + | _c.Call.Return(err) | |||
| 786 | + | return _c | |||
| 787 | + | } | |||
| 788 | + | ||||
| 789 | + | func (_c *MockSnapshotRepository_Publish_Call) RunAndReturn(run func(context1 context.Context, snapshotPublication ports.SnapshotPublication) error) *MockSnapshotRepository_Publish_Call { | |||
| 790 | + | _c.Call.Return(run) | |||
| 791 | + | return _c | |||
| 792 | + | } | |||
| 793 | + | ||||
| 794 | + | // ReconcileCheckpoint provides a mock function for the type MockSnapshotRepository | |||
| 795 | + | func (_mock *MockSnapshotRepository) ReconcileCheckpoint(context1 context.Context, incarnationID domain.IncarnationID, v ports.CheckpointRef) error { | |||
| 796 | + | ret := _mock.Called(context1, incarnationID, v) | |||
| 797 | + | ||||
| 798 | + | if len(ret) == 0 { | |||
| 799 | + | panic("no return value specified for ReconcileCheckpoint") | |||
| 800 | + | } | |||
| 801 | + | ||||
| 802 | + | var r0 error | |||
| 803 | + | if returnFunc, ok := ret.Get(0).(func(context.Context, domain.IncarnationID, ports.CheckpointRef) error); ok { | |||
| 804 | + | r0 = returnFunc(context1, incarnationID, v) | |||
| 805 | + | } else { | |||
| 806 | + | r0 = ret.Error(0) | |||
| 807 | + | } | |||
| 808 | + | return r0 | |||
| 809 | + | } | |||
| 810 | + | ||||
| 811 | + | // MockSnapshotRepository_ReconcileCheckpoint_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReconcileCheckpoint' | |||
| 812 | + | type MockSnapshotRepository_ReconcileCheckpoint_Call struct { | |||
| 813 | + | *mock.Call | |||
| 814 | + | } | |||
| 815 | + | ||||
| 816 | + | // ReconcileCheckpoint is a helper method to define mock.On call | |||
| 817 | + | // - context1 context.Context | |||
| 818 | + | // - incarnationID domain.IncarnationID | |||
| 819 | + | // - v ports.CheckpointRef | |||
| 820 | + | func (_e *MockSnapshotRepository_Expecter) ReconcileCheckpoint(context1 any, incarnationID any, v any) *MockSnapshotRepository_ReconcileCheckpoint_Call { | |||
| 821 | + | return &MockSnapshotRepository_ReconcileCheckpoint_Call{Call: _e.mock.On("ReconcileCheckpoint", context1, incarnationID, v)} | |||
| 822 | + | } | |||
| 823 | + | ||||
| 824 | + | func (_c *MockSnapshotRepository_ReconcileCheckpoint_Call) Run(run func(context1 context.Context, incarnationID domain.IncarnationID, v ports.CheckpointRef)) *MockSnapshotRepository_ReconcileCheckpoint_Call { | |||
| 825 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 826 | + | var arg0 context.Context | |||
| 827 | + | if args[0] != nil { | |||
| 828 | + | arg0 = args[0].(context.Context) | |||
| 829 | + | } | |||
| 830 | + | var arg1 domain.IncarnationID | |||
| 831 | + | if args[1] != nil { | |||
| 832 | + | arg1 = args[1].(domain.IncarnationID) | |||
| 833 | + | } | |||
| 834 | + | var arg2 ports.CheckpointRef | |||
| 835 | + | if args[2] != nil { | |||
| 836 | + | arg2 = args[2].(ports.CheckpointRef) | |||
| 837 | + | } | |||
| 838 | + | run( | |||
| 839 | + | arg0, | |||
| 840 | + | arg1, | |||
| 841 | + | arg2, | |||
| 842 | + | ) | |||
| 843 | + | }) | |||
| 844 | + | return _c | |||
| 845 | + | } | |||
| 846 | + | ||||
| 847 | + | func (_c *MockSnapshotRepository_ReconcileCheckpoint_Call) Return(err error) *MockSnapshotRepository_ReconcileCheckpoint_Call { | |||
| 848 | + | _c.Call.Return(err) | |||
| 849 | + | return _c | |||
| 850 | + | } | |||
| 851 | + | ||||
| 852 | + | func (_c *MockSnapshotRepository_ReconcileCheckpoint_Call) RunAndReturn(run func(context1 context.Context, incarnationID domain.IncarnationID, v ports.CheckpointRef) error) *MockSnapshotRepository_ReconcileCheckpoint_Call { | |||
| 853 | + | _c.Call.Return(run) | |||
| 854 | + | return _c | |||
| 855 | + | } | |||
M internal/ports/mocks/store_mock.go +44 -640
| @@ -5,9 +5,6 @@ | |||||
| 5 | package portsmocks | 5 | package portsmocks | ||
| 6 | 6 | ||||
| 7 | import ( | 7 | import ( | ||
| 8 | - | "context" | |||
| 9 | - | ||||
| 10 | - | "github.com/bnema/vev/internal/ports" | |||
| 11 | mock "github.com/stretchr/testify/mock" | 8 | mock "github.com/stretchr/testify/mock" | ||
| 12 | ) | 9 | ) | ||
| 13 | 10 | ||||
| @@ -82,6 +79,50 @@ func (_c *MockStore_Close_Call) RunAndReturn(run func() error) *MockStore_Close_ | |||||
| 82 | return _c | 79 | return _c | ||
| 83 | } | 80 | } | ||
| 84 | 81 | ||||
| 82 | + | // CloseWithoutSync provides a mock function for the type MockStore | |||
| 83 | + | func (_mock *MockStore) CloseWithoutSync() error { | |||
| 84 | + | ret := _mock.Called() | |||
| 85 | + | ||||
| 86 | + | if len(ret) == 0 { | |||
| 87 | + | panic("no return value specified for CloseWithoutSync") | |||
| 88 | + | } | |||
| 89 | + | ||||
| 90 | + | var r0 error | |||
| 91 | + | if returnFunc, ok := ret.Get(0).(func() error); ok { | |||
| 92 | + | r0 = returnFunc() | |||
| 93 | + | } else { | |||
| 94 | + | r0 = ret.Error(0) | |||
| 95 | + | } | |||
| 96 | + | return r0 | |||
| 97 | + | } | |||
| 98 | + | ||||
| 99 | + | // MockStore_CloseWithoutSync_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CloseWithoutSync' | |||
| 100 | + | type MockStore_CloseWithoutSync_Call struct { | |||
| 101 | + | *mock.Call | |||
| 102 | + | } | |||
| 103 | + | ||||
| 104 | + | // CloseWithoutSync is a helper method to define mock.On call | |||
| 105 | + | func (_e *MockStore_Expecter) CloseWithoutSync() *MockStore_CloseWithoutSync_Call { | |||
| 106 | + | return &MockStore_CloseWithoutSync_Call{Call: _e.mock.On("CloseWithoutSync")} | |||
| 107 | + | } | |||
| 108 | + | ||||
| 109 | + | func (_c *MockStore_CloseWithoutSync_Call) Run(run func()) *MockStore_CloseWithoutSync_Call { | |||
| 110 | + | _c.Call.Run(func(args mock.Arguments) { | |||
| 111 | + | run() | |||
| 112 | + | }) | |||
| 113 | + | return _c | |||
| 114 | + | } | |||
| 115 | + | ||||
| 116 | + | func (_c *MockStore_CloseWithoutSync_Call) Return(err error) *MockStore_CloseWithoutSync_Call { | |||
| 117 | + | _c.Call.Return(err) | |||
| 118 | + | return _c | |||
| 119 | + | } | |||
| 120 | + | ||||
| 121 | + | func (_c *MockStore_CloseWithoutSync_Call) RunAndReturn(run func() error) *MockStore_CloseWithoutSync_Call { | |||
| 122 | + | _c.Call.Return(run) | |||
| 123 | + | return _c | |||
| 124 | + | } | |||
| 125 | + | ||||
| 85 | // Delete provides a mock function for the type MockStore | 126 | // Delete provides a mock function for the type MockStore | ||
| 86 | func (_mock *MockStore) Delete(key []byte) error { | 127 | func (_mock *MockStore) Delete(key []byte) error { | ||
| 87 | ret := _mock.Called(key) | 128 | ret := _mock.Called(key) | ||
| @@ -335,640 +376,3 @@ func (_c *MockStore_Sync_Call) RunAndReturn(run func() error) *MockStore_Sync_Ca | |||||
| 335 | _c.Call.Return(run) | 376 | _c.Call.Return(run) | ||
| 336 | return _c | 377 | return _c | ||
| 337 | } | 378 | } | ||
| 338 | - | ||||
| 339 | - | // NewMockSnapshotRepository creates a new instance of MockSnapshotRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | |||
| 340 | - | // The first argument is typically a *testing.T value. | |||
| 341 | - | func NewMockSnapshotRepository(t interface { | |||
| 342 | - | mock.TestingT | |||
| 343 | - | Cleanup(func()) | |||
| 344 | - | }) *MockSnapshotRepository { | |||
| 345 | - | mock := &MockSnapshotRepository{} | |||
| 346 | - | mock.Mock.Test(t) | |||
| 347 | - | ||||
| 348 | - | t.Cleanup(func() { mock.AssertExpectations(t) }) | |||
| 349 | - | ||||
| 350 | - | return mock | |||
| 351 | - | } | |||
| 352 | - | ||||
| 353 | - | // MockSnapshotRepository is an autogenerated mock type for the SnapshotRepository type | |||
| 354 | - | type MockSnapshotRepository struct { | |||
| 355 | - | mock.Mock | |||
| 356 | - | } | |||
| 357 | - | ||||
| 358 | - | type MockSnapshotRepository_Expecter struct { | |||
| 359 | - | mock *mock.Mock | |||
| 360 | - | } | |||
| 361 | - | ||||
| 362 | - | func (_m *MockSnapshotRepository) EXPECT() *MockSnapshotRepository_Expecter { | |||
| 363 | - | return &MockSnapshotRepository_Expecter{mock: &_m.Mock} | |||
| 364 | - | } | |||
| 365 | - | ||||
| 366 | - | // Delete provides a mock function for the type MockSnapshotRepository | |||
| 367 | - | func (_mock *MockSnapshotRepository) Delete(context1 context.Context, s string) error { | |||
| 368 | - | ret := _mock.Called(context1, s) | |||
| 369 | - | ||||
| 370 | - | if len(ret) == 0 { | |||
| 371 | - | panic("no return value specified for Delete") | |||
| 372 | - | } | |||
| 373 | - | ||||
| 374 | - | var r0 error | |||
| 375 | - | if returnFunc, ok := ret.Get(0).(func(context.Context, string) error); ok { | |||
| 376 | - | r0 = returnFunc(context1, s) | |||
| 377 | - | } else { | |||
| 378 | - | r0 = ret.Error(0) | |||
| 379 | - | } | |||
| 380 | - | return r0 | |||
| 381 | - | } | |||
| 382 | - | ||||
| 383 | - | // MockSnapshotRepository_Delete_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Delete' | |||
| 384 | - | type MockSnapshotRepository_Delete_Call struct { | |||
| 385 | - | *mock.Call | |||
| 386 | - | } | |||
| 387 | - | ||||
| 388 | - | // Delete is a helper method to define mock.On call | |||
| 389 | - | // - context1 context.Context | |||
| 390 | - | // - s string | |||
| 391 | - | func (_e *MockSnapshotRepository_Expecter) Delete(context1 any, s any) *MockSnapshotRepository_Delete_Call { | |||
| 392 | - | return &MockSnapshotRepository_Delete_Call{Call: _e.mock.On("Delete", context1, s)} | |||
| 393 | - | } | |||
| 394 | - | ||||
| 395 | - | func (_c *MockSnapshotRepository_Delete_Call) Run(run func(context1 context.Context, s string)) *MockSnapshotRepository_Delete_Call { | |||
| 396 | - | _c.Call.Run(func(args mock.Arguments) { | |||
| 397 | - | var arg0 context.Context | |||
| 398 | - | if args[0] != nil { | |||
| 399 | - | arg0 = args[0].(context.Context) | |||
| 400 | - | } | |||
| 401 | - | var arg1 string | |||
| 402 | - | if args[1] != nil { | |||
| 403 | - | arg1 = args[1].(string) | |||
| 404 | - | } | |||
| 405 | - | run( | |||
| 406 | - | arg0, | |||
| 407 | - | arg1, | |||
| 408 | - | ) | |||
| 409 | - | }) | |||
| 410 | - | return _c | |||
| 411 | - | } | |||
| 412 | - | ||||
| 413 | - | func (_c *MockSnapshotRepository_Delete_Call) Return(err error) *MockSnapshotRepository_Delete_Call { | |||
| 414 | - | _c.Call.Return(err) | |||
| 415 | - | return _c | |||
| 416 | - | } | |||
| 417 | - | ||||
| 418 | - | func (_c *MockSnapshotRepository_Delete_Call) RunAndReturn(run func(context1 context.Context, s string) error) *MockSnapshotRepository_Delete_Call { | |||
| 419 | - | _c.Call.Return(run) | |||
| 420 | - | return _c | |||
| 421 | - | } | |||
| 422 | - | ||||
| 423 | - | // DeleteTombstone provides a mock function for the type MockSnapshotRepository | |||
| 424 | - | func (_mock *MockSnapshotRepository) DeleteTombstone(context1 context.Context, s string) error { | |||
| 425 | - | ret := _mock.Called(context1, s) | |||
| 426 | - | ||||
| 427 | - | if len(ret) == 0 { | |||
| 428 | - | panic("no return value specified for DeleteTombstone") | |||
| 429 | - | } | |||
| 430 | - | ||||
| 431 | - | var r0 error | |||
| 432 | - | if returnFunc, ok := ret.Get(0).(func(context.Context, string) error); ok { | |||
| 433 | - | r0 = returnFunc(context1, s) | |||
| 434 | - | } else { | |||
| 435 | - | r0 = ret.Error(0) | |||
| 436 | - | } | |||
| 437 | - | return r0 | |||
| 438 | - | } | |||
| 439 | - | ||||
| 440 | - | // MockSnapshotRepository_DeleteTombstone_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteTombstone' | |||
| 441 | - | type MockSnapshotRepository_DeleteTombstone_Call struct { | |||
| 442 | - | *mock.Call | |||
| 443 | - | } | |||
| 444 | - | ||||
| 445 | - | // DeleteTombstone is a helper method to define mock.On call | |||
| 446 | - | // - context1 context.Context | |||
| 447 | - | // - s string | |||
| 448 | - | func (_e *MockSnapshotRepository_Expecter) DeleteTombstone(context1 any, s any) *MockSnapshotRepository_DeleteTombstone_Call { | |||
| 449 | - | return &MockSnapshotRepository_DeleteTombstone_Call{Call: _e.mock.On("DeleteTombstone", context1, s)} | |||
| 450 | - | } | |||
| 451 | - | ||||
| 452 | - | func (_c *MockSnapshotRepository_DeleteTombstone_Call) Run(run func(context1 context.Context, s string)) *MockSnapshotRepository_DeleteTombstone_Call { | |||
| 453 | - | _c.Call.Run(func(args mock.Arguments) { | |||
| 454 | - | var arg0 context.Context | |||
| 455 | - | if args[0] != nil { | |||
| 456 | - | arg0 = args[0].(context.Context) | |||
| 457 | - | } | |||
| 458 | - | var arg1 string | |||
| 459 | - | if args[1] != nil { | |||
| 460 | - | arg1 = args[1].(string) | |||
| 461 | - | } | |||
| 462 | - | run( | |||
| 463 | - | arg0, | |||
| 464 | - | arg1, | |||
| 465 | - | ) | |||
| 466 | - | }) | |||
| 467 | - | return _c | |||
| 468 | - | } | |||
| 469 | - | ||||
| 470 | - | func (_c *MockSnapshotRepository_DeleteTombstone_Call) Return(err error) *MockSnapshotRepository_DeleteTombstone_Call { | |||
| 471 | - | _c.Call.Return(err) | |||
| 472 | - | return _c | |||
| 473 | - | } | |||
| 474 | - | ||||
| 475 | - | func (_c *MockSnapshotRepository_DeleteTombstone_Call) RunAndReturn(run func(context1 context.Context, s string) error) *MockSnapshotRepository_DeleteTombstone_Call { | |||
| 476 | - | _c.Call.Return(run) | |||
| 477 | - | return _c | |||
| 478 | - | } | |||
| 479 | - | ||||
| 480 | - | // List provides a mock function for the type MockSnapshotRepository | |||
| 481 | - | func (_mock *MockSnapshotRepository) List(context1 context.Context) ([]string, error) { | |||
| 482 | - | ret := _mock.Called(context1) | |||
| 483 | - | ||||
| 484 | - | if len(ret) == 0 { | |||
| 485 | - | panic("no return value specified for List") | |||
| 486 | - | } | |||
| 487 | - | ||||
| 488 | - | var r0 []string | |||
| 489 | - | var r1 error | |||
| 490 | - | if returnFunc, ok := ret.Get(0).(func(context.Context) ([]string, error)); ok { | |||
| 491 | - | return returnFunc(context1) | |||
| 492 | - | } | |||
| 493 | - | if returnFunc, ok := ret.Get(0).(func(context.Context) []string); ok { | |||
| 494 | - | r0 = returnFunc(context1) | |||
| 495 | - | } else { | |||
| 496 | - | if ret.Get(0) != nil { | |||
| 497 | - | r0 = ret.Get(0).([]string) | |||
| 498 | - | } | |||
| 499 | - | } | |||
| 500 | - | if returnFunc, ok := ret.Get(1).(func(context.Context) error); ok { | |||
| 501 | - | r1 = returnFunc(context1) | |||
| 502 | - | } else { | |||
| 503 | - | r1 = ret.Error(1) | |||
| 504 | - | } | |||
| 505 | - | return r0, r1 | |||
| 506 | - | } | |||
| 507 | - | ||||
| 508 | - | // MockSnapshotRepository_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List' | |||
| 509 | - | type MockSnapshotRepository_List_Call struct { | |||
| 510 | - | *mock.Call | |||
| 511 | - | } | |||
| 512 | - | ||||
| 513 | - | // List is a helper method to define mock.On call | |||
| 514 | - | // - context1 context.Context | |||
| 515 | - | func (_e *MockSnapshotRepository_Expecter) List(context1 any) *MockSnapshotRepository_List_Call { | |||
| 516 | - | return &MockSnapshotRepository_List_Call{Call: _e.mock.On("List", context1)} | |||
| 517 | - | } | |||
| 518 | - | ||||
| 519 | - | func (_c *MockSnapshotRepository_List_Call) Run(run func(context1 context.Context)) *MockSnapshotRepository_List_Call { | |||
| 520 | - | _c.Call.Run(func(args mock.Arguments) { | |||
| 521 | - | var arg0 context.Context | |||
| 522 | - | if args[0] != nil { | |||
| 523 | - | arg0 = args[0].(context.Context) | |||
| 524 | - | } | |||
| 525 | - | run( | |||
| 526 | - | arg0, | |||
| 527 | - | ) | |||
| 528 | - | }) | |||
| 529 | - | return _c | |||
| 530 | - | } | |||
| 531 | - | ||||
| 532 | - | func (_c *MockSnapshotRepository_List_Call) Return(strings []string, err error) *MockSnapshotRepository_List_Call { | |||
| 533 | - | _c.Call.Return(strings, err) | |||
| 534 | - | return _c | |||
| 535 | - | } | |||
| 536 | - | ||||
| 537 | - | func (_c *MockSnapshotRepository_List_Call) RunAndReturn(run func(context1 context.Context) ([]string, error)) *MockSnapshotRepository_List_Call { | |||
| 538 | - | _c.Call.Return(run) | |||
| 539 | - | return _c | |||
| 540 | - | } | |||
| 541 | - | ||||
| 542 | - | // Load provides a mock function for the type MockSnapshotRepository | |||
| 543 | - | func (_mock *MockSnapshotRepository) Load(context1 context.Context, s string) (ports.SnapshotGeneration, error) { | |||
| 544 | - | ret := _mock.Called(context1, s) | |||
| 545 | - | ||||
| 546 | - | if len(ret) == 0 { | |||
| 547 | - | panic("no return value specified for Load") | |||
| 548 | - | } | |||
| 549 | - | ||||
| 550 | - | var r0 ports.SnapshotGeneration | |||
| 551 | - | var r1 error | |||
| 552 | - | if returnFunc, ok := ret.Get(0).(func(context.Context, string) (ports.SnapshotGeneration, error)); ok { | |||
| 553 | - | return returnFunc(context1, s) | |||
| 554 | - | } | |||
| 555 | - | if returnFunc, ok := ret.Get(0).(func(context.Context, string) ports.SnapshotGeneration); ok { | |||
| 556 | - | r0 = returnFunc(context1, s) | |||
| 557 | - | } else { | |||
| 558 | - | r0 = ret.Get(0).(ports.SnapshotGeneration) | |||
| 559 | - | } | |||
| 560 | - | if returnFunc, ok := ret.Get(1).(func(context.Context, string) error); ok { | |||
| 561 | - | r1 = returnFunc(context1, s) | |||
| 562 | - | } else { | |||
| 563 | - | r1 = ret.Error(1) | |||
| 564 | - | } | |||
| 565 | - | return r0, r1 | |||
| 566 | - | } | |||
| 567 | - | ||||
| 568 | - | // MockSnapshotRepository_Load_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Load' | |||
| 569 | - | type MockSnapshotRepository_Load_Call struct { | |||
| 570 | - | *mock.Call | |||
| 571 | - | } | |||
| 572 | - | ||||
| 573 | - | // Load is a helper method to define mock.On call | |||
| 574 | - | // - context1 context.Context | |||
| 575 | - | // - s string | |||
| 576 | - | func (_e *MockSnapshotRepository_Expecter) Load(context1 any, s any) *MockSnapshotRepository_Load_Call { | |||
| 577 | - | return &MockSnapshotRepository_Load_Call{Call: _e.mock.On("Load", context1, s)} | |||
| 578 | - | } | |||
| 579 | - | ||||
| 580 | - | func (_c *MockSnapshotRepository_Load_Call) Run(run func(context1 context.Context, s string)) *MockSnapshotRepository_Load_Call { | |||
| 581 | - | _c.Call.Run(func(args mock.Arguments) { | |||
| 582 | - | var arg0 context.Context | |||
| 583 | - | if args[0] != nil { | |||
| 584 | - | arg0 = args[0].(context.Context) | |||
| 585 | - | } | |||
| 586 | - | var arg1 string | |||
| 587 | - | if args[1] != nil { | |||
| 588 | - | arg1 = args[1].(string) | |||
| 589 | - | } | |||
| 590 | - | run( | |||
| 591 | - | arg0, | |||
| 592 | - | arg1, | |||
| 593 | - | ) | |||
| 594 | - | }) | |||
| 595 | - | return _c | |||
| 596 | - | } | |||
| 597 | - | ||||
| 598 | - | func (_c *MockSnapshotRepository_Load_Call) Return(snapshotGeneration ports.SnapshotGeneration, err error) *MockSnapshotRepository_Load_Call { | |||
| 599 | - | _c.Call.Return(snapshotGeneration, err) | |||
| 600 | - | return _c | |||
| 601 | - | } | |||
| 602 | - | ||||
| 603 | - | func (_c *MockSnapshotRepository_Load_Call) RunAndReturn(run func(context1 context.Context, s string) (ports.SnapshotGeneration, error)) *MockSnapshotRepository_Load_Call { | |||
| 604 | - | _c.Call.Return(run) | |||
| 605 | - | return _c | |||
| 606 | - | } | |||
| 607 | - | ||||
| 608 | - | // Maintain provides a mock function for the type MockSnapshotRepository | |||
| 609 | - | func (_mock *MockSnapshotRepository) Maintain(context1 context.Context) error { | |||
| 610 | - | ret := _mock.Called(context1) | |||
| 611 | - | ||||
| 612 | - | if len(ret) == 0 { | |||
| 613 | - | panic("no return value specified for Maintain") | |||
| 614 | - | } | |||
| 615 | - | ||||
| 616 | - | var r0 error | |||
| 617 | - | if returnFunc, ok := ret.Get(0).(func(context.Context) error); ok { | |||
| 618 | - | r0 = returnFunc(context1) | |||
| 619 | - | } else { | |||
| 620 | - | r0 = ret.Error(0) | |||
| 621 | - | } | |||
| 622 | - | return r0 | |||
| 623 | - | } | |||
| 624 | - | ||||
| 625 | - | // MockSnapshotRepository_Maintain_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Maintain' | |||
| 626 | - | type MockSnapshotRepository_Maintain_Call struct { | |||
| 627 | - | *mock.Call | |||
| 628 | - | } | |||
| 629 | - | ||||
| 630 | - | // Maintain is a helper method to define mock.On call | |||
| 631 | - | // - context1 context.Context | |||
| 632 | - | func (_e *MockSnapshotRepository_Expecter) Maintain(context1 any) *MockSnapshotRepository_Maintain_Call { | |||
| 633 | - | return &MockSnapshotRepository_Maintain_Call{Call: _e.mock.On("Maintain", context1)} | |||
| 634 | - | } | |||
| 635 | - | ||||
| 636 | - | func (_c *MockSnapshotRepository_Maintain_Call) Run(run func(context1 context.Context)) *MockSnapshotRepository_Maintain_Call { | |||
| 637 | - | _c.Call.Run(func(args mock.Arguments) { | |||
| 638 | - | var arg0 context.Context | |||
| 639 | - | if args[0] != nil { | |||
| 640 | - | arg0 = args[0].(context.Context) | |||
| 641 | - | } | |||
| 642 | - | run( | |||
| 643 | - | arg0, | |||
| 644 | - | ) | |||
| 645 | - | }) | |||
| 646 | - | return _c | |||
| 647 | - | } | |||
| 648 | - | ||||
| 649 | - | func (_c *MockSnapshotRepository_Maintain_Call) Return(err error) *MockSnapshotRepository_Maintain_Call { | |||
| 650 | - | _c.Call.Return(err) | |||
| 651 | - | return _c | |||
| 652 | - | } | |||
| 653 | - | ||||
| 654 | - | func (_c *MockSnapshotRepository_Maintain_Call) RunAndReturn(run func(context1 context.Context) error) *MockSnapshotRepository_Maintain_Call { | |||
| 655 | - | _c.Call.Return(run) | |||
| 656 | - | return _c | |||
| 657 | - | } | |||
| 658 | - | ||||
| 659 | - | // Publish provides a mock function for the type MockSnapshotRepository | |||
| 660 | - | func (_mock *MockSnapshotRepository) Publish(context1 context.Context, snapshotPublication ports.SnapshotPublication) error { | |||
| 661 | - | ret := _mock.Called(context1, snapshotPublication) | |||
| 662 | - | ||||
| 663 | - | if len(ret) == 0 { | |||
| 664 | - | panic("no return value specified for Publish") | |||
| 665 | - | } | |||
| 666 | - | ||||
| 667 | - | var r0 error | |||
| 668 | - | if returnFunc, ok := ret.Get(0).(func(context.Context, ports.SnapshotPublication) error); ok { | |||
| 669 | - | r0 = returnFunc(context1, snapshotPublication) | |||
| 670 | - | } else { | |||
| 671 | - | r0 = ret.Error(0) | |||
| 672 | - | } | |||
| 673 | - | return r0 | |||
| 674 | - | } | |||
| 675 | - | ||||
| 676 | - | // MockSnapshotRepository_Publish_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Publish' | |||
| 677 | - | type MockSnapshotRepository_Publish_Call struct { | |||
| 678 | - | *mock.Call | |||
| 679 | - | } | |||
| 680 | - | ||||
| 681 | - | // Publish is a helper method to define mock.On call | |||
| 682 | - | // - context1 context.Context | |||
| 683 | - | // - snapshotPublication ports.SnapshotPublication | |||
| 684 | - | func (_e *MockSnapshotRepository_Expecter) Publish(context1 any, snapshotPublication any) *MockSnapshotRepository_Publish_Call { | |||
| 685 | - | return &MockSnapshotRepository_Publish_Call{Call: _e.mock.On("Publish", context1, snapshotPublication)} | |||
| 686 | - | } | |||
| 687 | - | ||||
| 688 | - | func (_c *MockSnapshotRepository_Publish_Call) Run(run func(context1 context.Context, snapshotPublication ports.SnapshotPublication)) *MockSnapshotRepository_Publish_Call { | |||
| 689 | - | _c.Call.Run(func(args mock.Arguments) { | |||
| 690 | - | var arg0 context.Context | |||
| 691 | - | if args[0] != nil { | |||
| 692 | - | arg0 = args[0].(context.Context) | |||
| 693 | - | } | |||
| 694 | - | var arg1 ports.SnapshotPublication | |||
| 695 | - | if args[1] != nil { | |||
| 696 | - | arg1 = args[1].(ports.SnapshotPublication) | |||
| 697 | - | } | |||
| 698 | - | run( | |||
| 699 | - | arg0, | |||
| 700 | - | arg1, | |||
| 701 | - | ) | |||
| 702 | - | }) | |||
| 703 | - | return _c | |||
| 704 | - | } | |||
| 705 | - | ||||
| 706 | - | func (_c *MockSnapshotRepository_Publish_Call) Return(err error) *MockSnapshotRepository_Publish_Call { | |||
| 707 | - | _c.Call.Return(err) | |||
| 708 | - | return _c | |||
| 709 | - | } | |||
| 710 | - | ||||
| 711 | - | func (_c *MockSnapshotRepository_Publish_Call) RunAndReturn(run func(context1 context.Context, snapshotPublication ports.SnapshotPublication) error) *MockSnapshotRepository_Publish_Call { | |||
| 712 | - | _c.Call.Return(run) | |||
| 713 | - | return _c | |||
| 714 | - | } | |||
| 715 | - | ||||
| 716 | - | // Tombstone provides a mock function for the type MockSnapshotRepository | |||
| 717 | - | func (_mock *MockSnapshotRepository) Tombstone(context1 context.Context, s string) error { | |||
| 718 | - | ret := _mock.Called(context1, s) | |||
| 719 | - | ||||
| 720 | - | if len(ret) == 0 { | |||
| 721 | - | panic("no return value specified for Tombstone") | |||
| 722 | - | } | |||
| 723 | - | ||||
| 724 | - | var r0 error | |||
| 725 | - | if returnFunc, ok := ret.Get(0).(func(context.Context, string) error); ok { | |||
| 726 | - | r0 = returnFunc(context1, s) | |||
| 727 | - | } else { | |||
| 728 | - | r0 = ret.Error(0) | |||
| 729 | - | } | |||
| 730 | - | return r0 | |||
| 731 | - | } | |||
| 732 | - | ||||
| 733 | - | // MockSnapshotRepository_Tombstone_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Tombstone' | |||
| 734 | - | type MockSnapshotRepository_Tombstone_Call struct { | |||
| 735 | - | *mock.Call | |||
| 736 | - | } | |||
| 737 | - | ||||
| 738 | - | // Tombstone is a helper method to define mock.On call | |||
| 739 | - | // - context1 context.Context | |||
| 740 | - | // - s string | |||
| 741 | - | func (_e *MockSnapshotRepository_Expecter) Tombstone(context1 any, s any) *MockSnapshotRepository_Tombstone_Call { | |||
| 742 | - | return &MockSnapshotRepository_Tombstone_Call{Call: _e.mock.On("Tombstone", context1, s)} | |||
| 743 | - | } | |||
| 744 | - | ||||
| 745 | - | func (_c *MockSnapshotRepository_Tombstone_Call) Run(run func(context1 context.Context, s string)) *MockSnapshotRepository_Tombstone_Call { | |||
| 746 | - | _c.Call.Run(func(args mock.Arguments) { | |||
| 747 | - | var arg0 context.Context | |||
| 748 | - | if args[0] != nil { | |||
| 749 | - | arg0 = args[0].(context.Context) | |||
| 750 | - | } | |||
| 751 | - | var arg1 string | |||
| 752 | - | if args[1] != nil { | |||
| 753 | - | arg1 = args[1].(string) | |||
| 754 | - | } | |||
| 755 | - | run( | |||
| 756 | - | arg0, | |||
| 757 | - | arg1, | |||
| 758 | - | ) | |||
| 759 | - | }) | |||
| 760 | - | return _c | |||
| 761 | - | } | |||
| 762 | - | ||||
| 763 | - | func (_c *MockSnapshotRepository_Tombstone_Call) Return(err error) *MockSnapshotRepository_Tombstone_Call { | |||
| 764 | - | _c.Call.Return(err) | |||
| 765 | - | return _c | |||
| 766 | - | } | |||
| 767 | - | ||||
| 768 | - | func (_c *MockSnapshotRepository_Tombstone_Call) RunAndReturn(run func(context1 context.Context, s string) error) *MockSnapshotRepository_Tombstone_Call { | |||
| 769 | - | _c.Call.Return(run) | |||
| 770 | - | return _c | |||
| 771 | - | } | |||
| 772 | - | ||||
| 773 | - | // NewMockLegacySnapshotSource creates a new instance of MockLegacySnapshotSource. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. | |||
| 774 | - | // The first argument is typically a *testing.T value. | |||
| 775 | - | func NewMockLegacySnapshotSource(t interface { | |||
| 776 | - | mock.TestingT | |||
| 777 | - | Cleanup(func()) | |||
| 778 | - | }) *MockLegacySnapshotSource { | |||
| 779 | - | mock := &MockLegacySnapshotSource{} | |||
| 780 | - | mock.Mock.Test(t) | |||
| 781 | - | ||||
| 782 | - | t.Cleanup(func() { mock.AssertExpectations(t) }) | |||
| 783 | - | ||||
| 784 | - | return mock | |||
| 785 | - | } | |||
| 786 | - | ||||
| 787 | - | // MockLegacySnapshotSource is an autogenerated mock type for the LegacySnapshotSource type | |||
| 788 | - | type MockLegacySnapshotSource struct { | |||
| 789 | - | mock.Mock | |||
| 790 | - | } | |||
| 791 | - | ||||
| 792 | - | type MockLegacySnapshotSource_Expecter struct { | |||
| 793 | - | mock *mock.Mock | |||
| 794 | - | } | |||
| 795 | - | ||||
| 796 | - | func (_m *MockLegacySnapshotSource) EXPECT() *MockLegacySnapshotSource_Expecter { | |||
| 797 | - | return &MockLegacySnapshotSource_Expecter{mock: &_m.Mock} | |||
| 798 | - | } | |||
| 799 | - | ||||
| 800 | - | // DeleteLegacy provides a mock function for the type MockLegacySnapshotSource | |||
| 801 | - | func (_mock *MockLegacySnapshotSource) DeleteLegacy(context1 context.Context, s string) error { | |||
| 802 | - | ret := _mock.Called(context1, s) | |||
| 803 | - | ||||
| 804 | - | if len(ret) == 0 { | |||
| 805 | - | panic("no return value specified for DeleteLegacy") | |||
| 806 | - | } | |||
| 807 | - | ||||
| 808 | - | var r0 error | |||
| 809 | - | if returnFunc, ok := ret.Get(0).(func(context.Context, string) error); ok { | |||
| 810 | - | r0 = returnFunc(context1, s) | |||
| 811 | - | } else { | |||
| 812 | - | r0 = ret.Error(0) | |||
| 813 | - | } | |||
| 814 | - | return r0 | |||
| 815 | - | } | |||
| 816 | - | ||||
| 817 | - | // MockLegacySnapshotSource_DeleteLegacy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteLegacy' | |||
| 818 | - | type MockLegacySnapshotSource_DeleteLegacy_Call struct { | |||
| 819 | - | *mock.Call | |||
| 820 | - | } | |||
| 821 | - | ||||
| 822 | - | // DeleteLegacy is a helper method to define mock.On call | |||
| 823 | - | // - context1 context.Context | |||
| 824 | - | // - s string | |||
| 825 | - | func (_e *MockLegacySnapshotSource_Expecter) DeleteLegacy(context1 any, s any) *MockLegacySnapshotSource_DeleteLegacy_Call { | |||
| 826 | - | return &MockLegacySnapshotSource_DeleteLegacy_Call{Call: _e.mock.On("DeleteLegacy", context1, s)} | |||
| 827 | - | } | |||
| 828 | - | ||||
| 829 | - | func (_c *MockLegacySnapshotSource_DeleteLegacy_Call) Run(run func(context1 context.Context, s string)) *MockLegacySnapshotSource_DeleteLegacy_Call { | |||
| 830 | - | _c.Call.Run(func(args mock.Arguments) { | |||
| 831 | - | var arg0 context.Context | |||
| 832 | - | if args[0] != nil { | |||
| 833 | - | arg0 = args[0].(context.Context) | |||
| 834 | - | } | |||
| 835 | - | var arg1 string | |||
| 836 | - | if args[1] != nil { | |||
| 837 | - | arg1 = args[1].(string) | |||
| 838 | - | } | |||
| 839 | - | run( | |||
| 840 | - | arg0, | |||
| 841 | - | arg1, | |||
| 842 | - | ) | |||
| 843 | - | }) | |||
| 844 | - | return _c | |||
| 845 | - | } | |||
| 846 | - | ||||
| 847 | - | func (_c *MockLegacySnapshotSource_DeleteLegacy_Call) Return(err error) *MockLegacySnapshotSource_DeleteLegacy_Call { | |||
| 848 | - | _c.Call.Return(err) | |||
| 849 | - | return _c | |||
| 850 | - | } | |||
| 851 | - | ||||
| 852 | - | func (_c *MockLegacySnapshotSource_DeleteLegacy_Call) RunAndReturn(run func(context1 context.Context, s string) error) *MockLegacySnapshotSource_DeleteLegacy_Call { | |||
| 853 | - | _c.Call.Return(run) | |||
| 854 | - | return _c | |||
| 855 | - | } | |||
| 856 | - | ||||
| 857 | - | // DeleteVerifiedLegacy provides a mock function for the type MockLegacySnapshotSource | |||
| 858 | - | func (_mock *MockLegacySnapshotSource) DeleteVerifiedLegacy(context1 context.Context, legacySnapshot ports.LegacySnapshot) error { | |||
| 859 | - | ret := _mock.Called(context1, legacySnapshot) | |||
| 860 | - | ||||
| 861 | - | if len(ret) == 0 { | |||
| 862 | - | panic("no return value specified for DeleteVerifiedLegacy") | |||
| 863 | - | } | |||
| 864 | - | ||||
| 865 | - | var r0 error | |||
| 866 | - | if returnFunc, ok := ret.Get(0).(func(context.Context, ports.LegacySnapshot) error); ok { | |||
| 867 | - | r0 = returnFunc(context1, legacySnapshot) | |||
| 868 | - | } else { | |||
| 869 | - | r0 = ret.Error(0) | |||
| 870 | - | } | |||
| 871 | - | return r0 | |||
| 872 | - | } | |||
| 873 | - | ||||
| 874 | - | // MockLegacySnapshotSource_DeleteVerifiedLegacy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteVerifiedLegacy' | |||
| 875 | - | type MockLegacySnapshotSource_DeleteVerifiedLegacy_Call struct { | |||
| 876 | - | *mock.Call | |||
| 877 | - | } | |||
| 878 | - | ||||
| 879 | - | // DeleteVerifiedLegacy is a helper method to define mock.On call | |||
| 880 | - | // - context1 context.Context | |||
| 881 | - | // - legacySnapshot ports.LegacySnapshot | |||
| 882 | - | func (_e *MockLegacySnapshotSource_Expecter) DeleteVerifiedLegacy(context1 any, legacySnapshot any) *MockLegacySnapshotSource_DeleteVerifiedLegacy_Call { | |||
| 883 | - | return &MockLegacySnapshotSource_DeleteVerifiedLegacy_Call{Call: _e.mock.On("DeleteVerifiedLegacy", context1, legacySnapshot)} | |||
| 884 | - | } | |||
| 885 | - | ||||
| 886 | - | func (_c *MockLegacySnapshotSource_DeleteVerifiedLegacy_Call) Run(run func(context1 context.Context, legacySnapshot ports.LegacySnapshot)) *MockLegacySnapshotSource_DeleteVerifiedLegacy_Call { | |||
| 887 | - | _c.Call.Run(func(args mock.Arguments) { | |||
| 888 | - | var arg0 context.Context | |||
| 889 | - | if args[0] != nil { | |||
| 890 | - | arg0 = args[0].(context.Context) | |||
| 891 | - | } | |||
| 892 | - | var arg1 ports.LegacySnapshot | |||
| 893 | - | if args[1] != nil { | |||
| 894 | - | arg1 = args[1].(ports.LegacySnapshot) | |||
| 895 | - | } | |||
| 896 | - | run( | |||
| 897 | - | arg0, | |||
| 898 | - | arg1, | |||
| 899 | - | ) | |||
| 900 | - | }) | |||
| 901 | - | return _c | |||
| 902 | - | } | |||
| 903 | - | ||||
| 904 | - | func (_c *MockLegacySnapshotSource_DeleteVerifiedLegacy_Call) Return(err error) *MockLegacySnapshotSource_DeleteVerifiedLegacy_Call { | |||
| 905 | - | _c.Call.Return(err) | |||
| 906 | - | return _c | |||
| 907 | - | } | |||
| 908 | - | ||||
| 909 | - | func (_c *MockLegacySnapshotSource_DeleteVerifiedLegacy_Call) RunAndReturn(run func(context1 context.Context, legacySnapshot ports.LegacySnapshot) error) *MockLegacySnapshotSource_DeleteVerifiedLegacy_Call { | |||
| 910 | - | _c.Call.Return(run) | |||
| 911 | - | return _c | |||
| 912 | - | } | |||
| 913 | - | ||||
| 914 | - | // LoadLegacy provides a mock function for the type MockLegacySnapshotSource | |||
| 915 | - | func (_mock *MockLegacySnapshotSource) LoadLegacy(context1 context.Context) ([]ports.LegacySnapshot, error) { | |||
| 916 | - | ret := _mock.Called(context1) | |||
| 917 | - | ||||
| 918 | - | if len(ret) == 0 { | |||
| 919 | - | panic("no return value specified for LoadLegacy") | |||
| 920 | - | } | |||
| 921 | - | ||||
| 922 | - | var r0 []ports.LegacySnapshot | |||
| 923 | - | var r1 error | |||
| 924 | - | if returnFunc, ok := ret.Get(0).(func(context.Context) ([]ports.LegacySnapshot, error)); ok { | |||
| 925 | - | return returnFunc(context1) | |||
| 926 | - | } | |||
| 927 | - | if returnFunc, ok := ret.Get(0).(func(context.Context) []ports.LegacySnapshot); ok { | |||
| 928 | - | r0 = returnFunc(context1) | |||
| 929 | - | } else { | |||
| 930 | - | if ret.Get(0) != nil { | |||
| 931 | - | r0 = ret.Get(0).([]ports.LegacySnapshot) | |||
| 932 | - | } | |||
| 933 | - | } | |||
| 934 | - | if returnFunc, ok := ret.Get(1).(func(context.Context) error); ok { | |||
| 935 | - | r1 = returnFunc(context1) | |||
| 936 | - | } else { | |||
| 937 | - | r1 = ret.Error(1) | |||
| 938 | - | } | |||
| 939 | - | return r0, r1 | |||
| 940 | - | } | |||
| 941 | - | ||||
| 942 | - | // MockLegacySnapshotSource_LoadLegacy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LoadLegacy' | |||
| 943 | - | type MockLegacySnapshotSource_LoadLegacy_Call struct { | |||
| 944 | - | *mock.Call | |||
| 945 | - | } | |||
| 946 | - | ||||
| 947 | - | // LoadLegacy is a helper method to define mock.On call | |||
| 948 | - | // - context1 context.Context | |||
| 949 | - | func (_e *MockLegacySnapshotSource_Expecter) LoadLegacy(context1 any) *MockLegacySnapshotSource_LoadLegacy_Call { | |||
| 950 | - | return &MockLegacySnapshotSource_LoadLegacy_Call{Call: _e.mock.On("LoadLegacy", context1)} | |||
| 951 | - | } | |||
| 952 | - | ||||
| 953 | - | func (_c *MockLegacySnapshotSource_LoadLegacy_Call) Run(run func(context1 context.Context)) *MockLegacySnapshotSource_LoadLegacy_Call { | |||
| 954 | - | _c.Call.Run(func(args mock.Arguments) { | |||
| 955 | - | var arg0 context.Context | |||
| 956 | - | if args[0] != nil { | |||
| 957 | - | arg0 = args[0].(context.Context) | |||
| 958 | - | } | |||
| 959 | - | run( | |||
| 960 | - | arg0, | |||
| 961 | - | ) | |||
| 962 | - | }) | |||
| 963 | - | return _c | |||
| 964 | - | } | |||
| 965 | - | ||||
| 966 | - | func (_c *MockLegacySnapshotSource_LoadLegacy_Call) Return(legacySnapshots []ports.LegacySnapshot, err error) *MockLegacySnapshotSource_LoadLegacy_Call { | |||
| 967 | - | _c.Call.Return(legacySnapshots, err) | |||
| 968 | - | return _c | |||
| 969 | - | } | |||
| 970 | - | ||||
| 971 | - | func (_c *MockLegacySnapshotSource_LoadLegacy_Call) RunAndReturn(run func(context1 context.Context) ([]ports.LegacySnapshot, error)) *MockLegacySnapshotSource_LoadLegacy_Call { | |||
| 972 | - | _c.Call.Return(run) | |||
| 973 | - | return _c | |||
| 974 | - | } | |||
A internal/ports/recovery.go +48 -0
| @@ -0,0 +1,48 @@ | |||||
| 1 | + | package ports | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | ||||
| 6 | + | "github.com/bnema/vev/internal/domain" | |||
| 7 | + | ) | |||
| 8 | + | ||||
| 9 | + | type CheckpointRef = domain.CheckpointRef | |||
| 10 | + | ||||
| 11 | + | type SnapshotPublication struct { | |||
| 12 | + | IncarnationID domain.IncarnationID | |||
| 13 | + | Name string | |||
| 14 | + | Generation uint64 | |||
| 15 | + | ParentCheckpoint *domain.CheckpointRef | |||
| 16 | + | Manifest []byte | |||
| 17 | + | Objects []SnapshotObject | |||
| 18 | + | } | |||
| 19 | + | ||||
| 20 | + | type SnapshotGeneration struct { | |||
| 21 | + | IncarnationID domain.IncarnationID | |||
| 22 | + | Name string | |||
| 23 | + | Generation uint64 | |||
| 24 | + | ParentCheckpoint *domain.CheckpointRef | |||
| 25 | + | Manifest []byte | |||
| 26 | + | Objects map[SnapshotDigest][]byte | |||
| 27 | + | } | |||
| 28 | + | ||||
| 29 | + | type Catalogue interface { | |||
| 30 | + | Records() ([]domain.CatalogueRecord, error) | |||
| 31 | + | Record(string) (domain.CatalogueRecord, bool, error) | |||
| 32 | + | Create(domain.CatalogueRecord) error | |||
| 33 | + | UpdateMetadata(domain.CatalogueMetadataUpdate) error | |||
| 34 | + | Replace(string, domain.CatalogueRecord) error | |||
| 35 | + | Rename(string, domain.CatalogueRecord) error | |||
| 36 | + | Delete(string) error | |||
| 37 | + | // Sync makes buffered metadata durable. Identity operations sync internally. | |||
| 38 | + | Sync() error | |||
| 39 | + | Close() error | |||
| 40 | + | } | |||
| 41 | + | ||||
| 42 | + | type SnapshotRepository interface { | |||
| 43 | + | Publish(context.Context, SnapshotPublication) error | |||
| 44 | + | LoadCheckpoint(context.Context, domain.IncarnationID, string, CheckpointRef) (SnapshotGeneration, error) | |||
| 45 | + | ReconcileCheckpoint(context.Context, domain.IncarnationID, CheckpointRef) error | |||
| 46 | + | DeleteIncarnation(context.Context, domain.IncarnationID) error | |||
| 47 | + | CollectGarbage(context.Context, map[domain.IncarnationID]domain.CheckpointRef) error | |||
| 48 | + | } | |||
M internal/ports/store.go +4 -57
| @@ -1,9 +1,6 @@ | |||||
| 1 | package ports | 1 | package ports | ||
| 2 | 2 | ||||
| 3 | - | import ( | 3 | + | import "crypto/sha256" |
| 4 | - | "context" | |||
| 5 | - | "crypto/sha256" | |||
| 6 | - | ) | |||
| 7 | 4 | ||||
| 8 | // Store is a small byte-key/value persistence port. | 5 | // Store is a small byte-key/value persistence port. | ||
| 9 | // | 6 | // | ||
| @@ -19,6 +16,9 @@ type Store interface { | |||||
| 19 | // or mutate either slice after Range returns. | 16 | // or mutate either slice after Range returns. | ||
| 20 | Range(fn func(k, v []byte) bool) | 17 | Range(fn func(k, v []byte) bool) | ||
| 21 | Sync() error | 18 | Sync() error | ||
| 19 | + | // CloseWithoutSync releases the store without persisting buffered writes. | |||
| 20 | + | // Callers use it after a rejected durable mutation. | |||
| 21 | + | CloseWithoutSync() error | |||
| 22 | Close() error | 22 | Close() error | ||
| 23 | } | 23 | } | ||
| 24 | 24 | ||||
| @@ -29,56 +29,3 @@ type SnapshotObject struct { | |||||
| 29 | Digest SnapshotDigest | 29 | Digest SnapshotDigest | ||
| 30 | Data []byte | 30 | Data []byte | ||
| 31 | } | 31 | } | ||
| 32 | - | ||||
| 33 | - | // SnapshotPublication atomically publishes a complete VEVM manifest and any | |||
| 34 | - | // newly reachable content-addressed VEVO objects for one named generation. | |||
| 35 | - | type SnapshotPublication struct { | |||
| 36 | - | Name string | |||
| 37 | - | Generation uint64 | |||
| 38 | - | Manifest []byte | |||
| 39 | - | Objects []SnapshotObject | |||
| 40 | - | } | |||
| 41 | - | ||||
| 42 | - | // SnapshotGeneration is the caller-owned material needed to restore one | |||
| 43 | - | // named generation. Fallback indicates that the repository selected an older | |||
| 44 | - | // valid generation after the requested/current generation was unavailable. | |||
| 45 | - | type SnapshotGeneration struct { | |||
| 46 | - | Name string | |||
| 47 | - | Generation uint64 | |||
| 48 | - | Manifest []byte | |||
| 49 | - | Objects map[SnapshotDigest][]byte | |||
| 50 | - | Fallback bool | |||
| 51 | - | } | |||
| 52 | - | ||||
| 53 | - | // SnapshotRepository persists content-addressed incremental snapshot | |||
| 54 | - | // generations. All returned bytes are owned by the caller. | |||
| 55 | - | type SnapshotRepository interface { | |||
| 56 | - | Publish(context.Context, SnapshotPublication) error | |||
| 57 | - | List(context.Context) ([]string, error) | |||
| 58 | - | Load(context.Context, string) (SnapshotGeneration, error) | |||
| 59 | - | Delete(context.Context, string) error | |||
| 60 | - | // Tombstone fences a named-session purge before either incremental or | |||
| 61 | - | // legacy data is deleted. It must remain durable until DeleteTombstone. | |||
| 62 | - | Tombstone(context.Context, string) error | |||
| 63 | - | DeleteTombstone(context.Context, string) error | |||
| 64 | - | Maintain(context.Context) error | |||
| 65 | - | } | |||
| 66 | - | ||||
| 67 | - | // LegacySnapshot is the pre-incremental named-session blob retained only for | |||
| 68 | - | // one-way migration. Data is caller-owned. | |||
| 69 | - | type LegacySnapshot struct { | |||
| 70 | - | Name string | |||
| 71 | - | Data []byte | |||
| 72 | - | } | |||
| 73 | - | ||||
| 74 | - | // LegacySnapshotSource exposes the v3 bridge separately from the new write | |||
| 75 | - | // contract. | |||
| 76 | - | type LegacySnapshotSource interface { | |||
| 77 | - | LoadLegacy(context.Context) ([]LegacySnapshot, error) | |||
| 78 | - | // DeleteVerifiedLegacy removes precisely the legacy blob that was verified | |||
| 79 | - | // after import. Implementations must persist its identity before unlinking. | |||
| 80 | - | DeleteVerifiedLegacy(context.Context, LegacySnapshot) error | |||
| 81 | - | // DeleteLegacy is reserved for explicit session purges, which intentionally | |||
| 82 | - | // delete by name rather than as part of an import receipt. | |||
| 83 | - | DeleteLegacy(context.Context, string) error | |||
| 84 | - | } | |||
M internal/ports/wire.go +16 -8
| @@ -190,14 +190,23 @@ type Kill struct { | |||||
| 190 | All bool | 190 | All bool | ||
| 191 | } | 191 | } | ||
| 192 | 192 | ||||
| 193 | + | // SessionState is the client-visible lifecycle state of a session. | |||
| 194 | + | type SessionState uint8 | |||
| 195 | + | ||||
| 196 | + | const ( | |||
| 197 | + | SessionRunning SessionState = iota | |||
| 198 | + | SessionStopped | |||
| 199 | + | SessionBroken | |||
| 200 | + | ) | |||
| 201 | + | ||||
| 193 | // SessionInfo describes one session in a Sessions listing. | 202 | // SessionInfo describes one session in a Sessions listing. | ||
| 194 | type SessionInfo struct { | 203 | type SessionInfo struct { | ||
| 195 | SessionID string | 204 | SessionID string | ||
| 196 | Name string | 205 | Name string | ||
| 206 | + | State SessionState | |||
| 197 | Ephemeral bool | 207 | Ephemeral bool | ||
| 198 | Tabs uint16 | 208 | Tabs uint16 | ||
| 199 | Attached bool | 209 | Attached bool | ||
| 200 | - | Stopped bool | |||
| 201 | } | 210 | } | ||
| 202 | 211 | ||||
| 203 | // Sessions is the daemon's reply to a List, enumerating live sessions. | 212 | // Sessions is the daemon's reply to a List, enumerating live sessions. | ||
| @@ -875,11 +884,7 @@ func MarshalSessions(m Sessions) []byte { | |||||
| 875 | } else { | 884 | } else { | ||
| 876 | w.putUint8(0) | 885 | w.putUint8(0) | ||
| 877 | } | 886 | } | ||
| 878 | - | if s.Stopped { | 887 | + | w.putUint8(uint8(s.State)) |
| 879 | - | w.putUint8(1) | |||
| 880 | - | } else { | |||
| 881 | - | w.putUint8(0) | |||
| 882 | - | } | |||
| 883 | } | 888 | } | ||
| 884 | return w.b | 889 | return w.b | ||
| 885 | } | 890 | } | ||
| @@ -913,11 +918,14 @@ func UnmarshalSessions(b []byte) (Sessions, error) { | |||||
| 913 | return Sessions{}, err | 918 | return Sessions{}, err | ||
| 914 | } | 919 | } | ||
| 915 | s.Attached = att != 0 | 920 | s.Attached = att != 0 | ||
| 916 | - | stopped, err := r.getUint8() | 921 | + | state, err := r.getUint8() |
| 917 | if err != nil { | 922 | if err != nil { | ||
| 918 | return Sessions{}, err | 923 | return Sessions{}, err | ||
| 919 | } | 924 | } | ||
| 920 | - | s.Stopped = stopped != 0 | 925 | + | s.State = SessionState(state) |
| 926 | + | if s.State > SessionBroken { | |||
| 927 | + | return Sessions{}, errors.New("ports: invalid session state") | |||
| 928 | + | } | |||
| 921 | sessions = append(sessions, s) | 929 | sessions = append(sessions, s) | ||
| 922 | } | 930 | } | ||
| 923 | if err := r.done(); err != nil { | 931 | if err := r.done(); err != nil { | ||
M internal/ports/wire_test.go +35 -6
| @@ -122,7 +122,7 @@ func TestHelloEnvironmentCodec(t *testing.T) { | |||||
| 122 | Env: []string{"A=B", "XY=123"}, | 122 | Env: []string{"A=B", "XY=123"}, | ||
| 123 | }) | 123 | }) | ||
| 124 | want := []byte{ | 124 | want := []byte{ | ||
| 125 | - | 0x00, 0x12, 0x00, // version, intent | 125 | + | 0x00, 0x13, 0x00, // version, intent |
| 126 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, | 126 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, | ||
| 127 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // client ID | 127 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // client ID | ||
| 128 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // resume token | 128 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // resume token | ||
| @@ -336,7 +336,7 @@ func TestThemeGenerationClearedWireGoldenPreservesProtocolVersion(t *testing.T) | |||||
| 336 | } | 336 | } | ||
| 337 | want := append([]byte{0x0f, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x00}, make([]byte, 48)...) | 337 | want := append([]byte{0x0f, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x00}, make([]byte, 48)...) | ||
| 338 | require.Equal(t, want, MarshalTheme(cleared)) | 338 | require.Equal(t, want, MarshalTheme(cleared)) | ||
| 339 | - | require.Equal(t, uint16(18), ProtocolVersion) | 339 | + | require.Equal(t, uint16(19), ProtocolVersion) |
| 340 | } | 340 | } | ||
| 341 | 341 | ||||
| 342 | func TestResizeGoldenAndRoundTrip(t *testing.T) { | 342 | func TestResizeGoldenAndRoundTrip(t *testing.T) { | ||
| @@ -461,7 +461,7 @@ func TestCommandRequestGoldenAndRoundTrip(t *testing.T) { | |||||
| 461 | name: "minimal", | 461 | name: "minimal", | ||
| 462 | msg: CommandRequest{Version: ProtocolVersion, Slug: "split-right"}, | 462 | msg: CommandRequest{Version: ProtocolVersion, Slug: "split-right"}, | ||
| 463 | want: []byte{ | 463 | want: []byte{ | ||
| 464 | - | 0x00, 0x12, | 464 | + | 0x00, 0x13, |
| 465 | 0x00, | 465 | 0x00, | ||
| 466 | 0x00, 0x0b, 's', 'p', 'l', 'i', 't', '-', 'r', 'i', 'g', 'h', 't', | 466 | 0x00, 0x0b, 's', 'p', 'l', 'i', 't', '-', 'r', 'i', 'g', 'h', 't', | ||
| 467 | 0x00, 0x00, | 467 | 0x00, 0x00, | ||
| @@ -482,7 +482,7 @@ func TestCommandRequestGoldenAndRoundTrip(t *testing.T) { | |||||
| 482 | JSON: true, | 482 | JSON: true, | ||
| 483 | }, | 483 | }, | ||
| 484 | want: []byte{ | 484 | want: []byte{ | ||
| 485 | - | 0x00, 0x12, | 485 | + | 0x00, 0x13, |
| 486 | 0x01, | 486 | 0x01, | ||
| 487 | 0x00, 0x05, 't', 'o', 'a', 's', 't', | 487 | 0x00, 0x05, 't', 'o', 'a', 's', 't', | ||
| 488 | 0x00, 0x03, | 488 | 0x00, 0x03, | ||
| @@ -741,6 +741,35 @@ func TestKillGoldenAndRoundTrip(t *testing.T) { | |||||
| 741 | assertTrailingGarbageFails(t, append(append([]byte(nil), legacy...), 0x00), UnmarshalKill) | 741 | assertTrailingGarbageFails(t, append(append([]byte(nil), legacy...), 0x00), UnmarshalKill) | ||
| 742 | } | 742 | } | ||
| 743 | 743 | ||||
| 744 | + | func TestSessionInfoRecoveryState(t *testing.T) { | |||
| 745 | + | tests := []struct { | |||
| 746 | + | name string | |||
| 747 | + | state SessionState | |||
| 748 | + | want []byte | |||
| 749 | + | }{ | |||
| 750 | + | {name: "running", state: SessionRunning, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, | |||
| 751 | + | {name: "stopped", state: SessionStopped, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1}}, | |||
| 752 | + | {name: "broken", state: SessionBroken, want: []byte{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2}}, | |||
| 753 | + | } | |||
| 754 | + | for _, tt := range tests { | |||
| 755 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 756 | + | payload := MarshalSessions(Sessions{Sessions: []SessionInfo{{State: tt.state}}}) | |||
| 757 | + | require.Equal(t, tt.want, payload) | |||
| 758 | + | got, err := UnmarshalSessions(payload) | |||
| 759 | + | require.NoError(t, err) | |||
| 760 | + | require.Equal(t, tt.state, got.Sessions[0].State) | |||
| 761 | + | assertAllPrefixesFail(t, payload, UnmarshalSessions) | |||
| 762 | + | assertTrailingGarbageFails(t, payload, UnmarshalSessions) | |||
| 763 | + | }) | |||
| 764 | + | } | |||
| 765 | + | for _, state := range []byte{3, 255} { | |||
| 766 | + | payload := append([]byte(nil), tests[0].want...) | |||
| 767 | + | payload[len(payload)-1] = state | |||
| 768 | + | _, err := UnmarshalSessions(payload) | |||
| 769 | + | require.Error(t, err, "state %d", state) | |||
| 770 | + | } | |||
| 771 | + | } | |||
| 772 | + | ||||
| 744 | func TestSessionsGoldenAndRoundTrip(t *testing.T) { | 773 | func TestSessionsGoldenAndRoundTrip(t *testing.T) { | ||
| 745 | tests := []struct { | 774 | tests := []struct { | ||
| 746 | name string | 775 | name string | ||
| @@ -755,8 +784,8 @@ func TestSessionsGoldenAndRoundTrip(t *testing.T) { | |||||
| 755 | { | 784 | { | ||
| 756 | name: "two", | 785 | name: "two", | ||
| 757 | msg: Sessions{Sessions: []SessionInfo{ | 786 | msg: Sessions{Sessions: []SessionInfo{ | ||
| 758 | - | {SessionID: "0", Name: "0", Ephemeral: true, Tabs: 1, Attached: false}, | 787 | + | {SessionID: "0", Name: "0", State: SessionRunning, Ephemeral: true, Tabs: 1, Attached: false}, |
| 759 | - | {SessionID: "work", Name: "proj", Ephemeral: false, Tabs: 5, Attached: true, Stopped: true}, | 788 | + | {SessionID: "work", Name: "proj", State: SessionStopped, Ephemeral: false, Tabs: 5, Attached: true}, |
| 760 | }}, | 789 | }}, | ||
| 761 | want: []byte{ | 790 | want: []byte{ | ||
| 762 | 0x00, 0x02, | 791 | 0x00, 0x02, | ||
M internal/usecase/command/command.go +13 -3
| @@ -8,9 +8,9 @@ import ( | |||||
| 8 | // ErrInvalidArguments reports arguments that do not have the exact command format. | 8 | // ErrInvalidArguments reports arguments that do not have the exact command format. | ||
| 9 | var ErrInvalidArguments = errors.New("invalid command arguments") | 9 | var ErrInvalidArguments = errors.New("invalid command arguments") | ||
| 10 | 10 | ||||
| 11 | - | // ParsePositiveDecimal accepts exactly one base-10 positive decimal value. | 11 | + | // ParsePositiveUint64 accepts exactly one base-10 positive decimal value. |
| 12 | // It deliberately rejects signs, whitespace, zero, and non-canonical forms. | 12 | // It deliberately rejects signs, whitespace, zero, and non-canonical forms. | ||
| 13 | - | func ParsePositiveDecimal(args []string) (int, error) { | 13 | + | func ParsePositiveUint64(args []string) (uint64, error) { |
| 14 | if len(args) != 1 || args[0] == "" || (len(args[0]) > 1 && args[0][0] == '0') { | 14 | if len(args) != 1 || args[0] == "" || (len(args[0]) > 1 && args[0][0] == '0') { | ||
| 15 | return 0, ErrInvalidArguments | 15 | return 0, ErrInvalidArguments | ||
| 16 | } | 16 | } | ||
| @@ -19,13 +19,22 @@ func ParsePositiveDecimal(args []string) (int, error) { | |||||
| 19 | return 0, ErrInvalidArguments | 19 | return 0, ErrInvalidArguments | ||
| 20 | } | 20 | } | ||
| 21 | } | 21 | } | ||
| 22 | - | n, err := strconv.Atoi(args[0]) | 22 | + | n, err := strconv.ParseUint(args[0], 10, 64) |
| 23 | if err != nil || n < 1 { | 23 | if err != nil || n < 1 { | ||
| 24 | return 0, ErrInvalidArguments | 24 | return 0, ErrInvalidArguments | ||
| 25 | } | 25 | } | ||
| 26 | return n, nil | 26 | return n, nil | ||
| 27 | } | 27 | } | ||
| 28 | 28 | ||||
| 29 | + | // ParsePositiveDecimal parses a positive decimal that must fit the platform int. | |||
| 30 | + | func ParsePositiveDecimal(args []string) (int, error) { | |||
| 31 | + | n, err := ParsePositiveUint64(args) | |||
| 32 | + | if err != nil || n > uint64(^uint(0)>>1) { | |||
| 33 | + | return 0, ErrInvalidArguments | |||
| 34 | + | } | |||
| 35 | + | return int(n), nil | |||
| 36 | + | } | |||
| 37 | + | ||||
| 29 | // Context is the set of application actions available to commands. | 38 | // Context is the set of application actions available to commands. | ||
| 30 | type Context interface { | 39 | type Context interface { | ||
| 31 | CreateTab() error | 40 | CreateTab() error | ||
| @@ -124,6 +133,7 @@ type ControlContext interface { | |||||
| 124 | RenameSessionTo(name string) error | 133 | RenameSessionTo(name string) error | ||
| 125 | RenameTabTo(name string) error | 134 | RenameTabTo(name string) error | ||
| 126 | Toast(severity, message string) error | 135 | Toast(severity, message string) error | ||
| 136 | + | SessionRecovery(action string) (string, error) | |||
| 127 | ListSessions(json bool) (string, error) | 137 | ListSessions(json bool) (string, error) | ||
| 128 | ListTabs(json bool) (string, error) | 138 | ListTabs(json bool) (string, error) | ||
| 129 | ListPanes(json bool) (string, error) | 139 | ListPanes(json bool) (string, error) | ||
M internal/usecase/command/command_test.go +34 -0
| @@ -5,6 +5,40 @@ import ( | |||||
| 5 | "testing" | 5 | "testing" | ||
| 6 | ) | 6 | ) | ||
| 7 | 7 | ||||
| 8 | + | func TestParsePositiveUint64(t *testing.T) { | |||
| 9 | + | for _, tt := range []struct { | |||
| 10 | + | name string | |||
| 11 | + | args []string | |||
| 12 | + | want uint64 | |||
| 13 | + | invalid bool | |||
| 14 | + | }{ | |||
| 15 | + | {name: "maximum", args: []string{"18446744073709551615"}, want: ^uint64(0)}, | |||
| 16 | + | {name: "overflow", args: []string{"18446744073709551616"}, invalid: true}, | |||
| 17 | + | {name: "zero", args: []string{"0"}, invalid: true}, | |||
| 18 | + | {name: "leading zero", args: []string{"01"}, invalid: true}, | |||
| 19 | + | {name: "empty string", args: []string{""}, invalid: true}, | |||
| 20 | + | {name: "positive sign", args: []string{"+1"}, invalid: true}, | |||
| 21 | + | {name: "negative sign", args: []string{"-1"}, invalid: true}, | |||
| 22 | + | {name: "whitespace", args: []string{"1 2"}, invalid: true}, | |||
| 23 | + | {name: "non-digit", args: []string{"1x"}, invalid: true}, | |||
| 24 | + | {name: "empty arguments", invalid: true}, | |||
| 25 | + | {name: "multiple arguments", args: []string{"1", "2"}, invalid: true}, | |||
| 26 | + | } { | |||
| 27 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 28 | + | got, err := ParsePositiveUint64(tt.args) | |||
| 29 | + | if tt.invalid { | |||
| 30 | + | if !errors.Is(err, ErrInvalidArguments) { | |||
| 31 | + | t.Fatalf("error = %v", err) | |||
| 32 | + | } | |||
| 33 | + | return | |||
| 34 | + | } | |||
| 35 | + | if err != nil || got != tt.want { | |||
| 36 | + | t.Fatalf("ParsePositiveUint64() = %d, %v", got, err) | |||
| 37 | + | } | |||
| 38 | + | }) | |||
| 39 | + | } | |||
| 40 | + | } | |||
| 41 | + | ||||
| 8 | func TestParsePositiveDecimal(t *testing.T) { | 42 | func TestParsePositiveDecimal(t *testing.T) { | ||
| 9 | for _, tt := range []struct { | 43 | for _, tt := range []struct { | ||
| 10 | name string | 44 | name string | ||
M internal/usecase/command/registry.go +16 -0
| @@ -47,6 +47,7 @@ func Registry() []Command { | |||||
| 47 | listCommand("list-sessions", "List sessions", "List sessions with active markers", TargetNone, func(ctx ControlContext, json bool) (string, error) { return ctx.ListSessions(json) }), | 47 | listCommand("list-sessions", "List sessions", "List sessions with active markers", TargetNone, func(ctx ControlContext, json bool) (string, error) { return ctx.ListSessions(json) }), | ||
| 48 | listCommand("list-tabs", "List tabs", "List tabs in the target session", TargetSession, func(ctx ControlContext, json bool) (string, error) { return ctx.ListTabs(json) }), | 48 | listCommand("list-tabs", "List tabs", "List tabs in the target session", TargetSession, func(ctx ControlContext, json bool) (string, error) { return ctx.ListTabs(json) }), | ||
| 49 | listCommand("list-panes", "List panes", "List panes in the target tab", TargetTab, func(ctx ControlContext, json bool) (string, error) { return ctx.ListPanes(json) }), | 49 | listCommand("list-panes", "List panes", "List panes in the target tab", TargetTab, func(ctx ControlContext, json bool) (string, error) { return ctx.ListPanes(json) }), | ||
| 50 | + | sessionRecoveryCommand(), | |||
| 50 | } | 51 | } | ||
| 51 | return commands | 52 | return commands | ||
| 52 | } | 53 | } | ||
| @@ -89,6 +90,21 @@ func toastCommand() Command { | |||||
| 89 | }} | 90 | }} | ||
| 90 | } | 91 | } | ||
| 91 | 92 | ||||
| 93 | + | func sessionRecoveryCommand() Command { | |||
| 94 | + | return Command{ | |||
| 95 | + | Slug: "session-recovery", Name: "Session recovery", Desc: "Discard a broken durable session's persisted state", | |||
| 96 | + | Usage: "session-recovery discard", | |||
| 97 | + | Scriptable: true, Target: TargetNone, | |||
| 98 | + | Control: func(ctx ControlContext, args []string, _ ControlOptions) (ControlResult, error) { | |||
| 99 | + | if len(args) != 1 || args[0] != "discard" { | |||
| 100 | + | return ControlResult{}, ErrInvalidArguments | |||
| 101 | + | } | |||
| 102 | + | output, err := ctx.SessionRecovery("discard") | |||
| 103 | + | return ControlResult{Output: output}, err | |||
| 104 | + | }, | |||
| 105 | + | } | |||
| 106 | + | } | |||
| 107 | + | ||||
| 92 | func listCommand(slug, name, desc string, target TargetKind, list func(ControlContext, bool) (string, error)) Command { | 108 | func listCommand(slug, name, desc string, target TargetKind, list func(ControlContext, bool) (string, error)) Command { | ||
| 93 | return Command{Slug: slug, Name: name, Desc: desc, Usage: slug + " [--json]", Scriptable: true, Target: target, Control: func(ctx ControlContext, args []string, opts ControlOptions) (ControlResult, error) { | 109 | return Command{Slug: slug, Name: name, Desc: desc, Usage: slug + " [--json]", Scriptable: true, Target: target, Control: func(ctx ControlContext, args []string, opts ControlOptions) (ControlResult, error) { | ||
| 94 | if len(args) != 0 { | 110 | if len(args) != 0 { | ||
M internal/usecase/command/registry_test.go +36 -1
| @@ -105,6 +105,35 @@ func TestPaletteRegistryHidesAPIOnly(t *testing.T) { | |||||
| 105 | } | 105 | } | ||
| 106 | } | 106 | } | ||
| 107 | 107 | ||||
| 108 | + | func TestSessionRecoveryCommand(t *testing.T) { | |||
| 109 | + | cmd, ok := BySlug("session-recovery") | |||
| 110 | + | if !ok || !cmd.Scriptable || cmd.Target != TargetNone { | |||
| 111 | + | t.Fatalf("session-recovery command = %#v, ok=%v", cmd, ok) | |||
| 112 | + | } | |||
| 113 | + | tests := []struct { | |||
| 114 | + | args []string | |||
| 115 | + | wantCall string | |||
| 116 | + | wantErr error | |||
| 117 | + | }{ | |||
| 118 | + | {args: []string{"discard"}, wantCall: "session-recovery:discard"}, | |||
| 119 | + | {args: []string{"retry"}, wantErr: ErrInvalidArguments}, | |||
| 120 | + | {args: []string{"restore", "7"}, wantErr: ErrInvalidArguments}, | |||
| 121 | + | {args: []string{"export", "/tmp/export"}, wantErr: ErrInvalidArguments}, | |||
| 122 | + | {args: nil, wantErr: ErrInvalidArguments}, | |||
| 123 | + | } | |||
| 124 | + | for _, test := range tests { | |||
| 125 | + | ctx := &controlSpy{recoveryOutput: "recovery output"} | |||
| 126 | + | got, err := cmd.Control(ctx, test.args, ControlOptions{}) | |||
| 127 | + | wantOutput := "" | |||
| 128 | + | if test.wantCall != "" { | |||
| 129 | + | wantOutput = ctx.recoveryOutput | |||
| 130 | + | } | |||
| 131 | + | if !errors.Is(err, test.wantErr) || ctx.call != test.wantCall || got.Output != wantOutput { | |||
| 132 | + | t.Errorf("args %v: call/output/error = %q/%q/%v, want %q/%q/%v", test.args, ctx.call, got.Output, err, test.wantCall, wantOutput, test.wantErr) | |||
| 133 | + | } | |||
| 134 | + | } | |||
| 135 | + | } | |||
| 136 | + | ||||
| 108 | func TestControlHandlersValidateAndDelegate(t *testing.T) { | 137 | func TestControlHandlersValidateAndDelegate(t *testing.T) { | ||
| 109 | tests := []struct { | 138 | tests := []struct { | ||
| 110 | name string | 139 | name string | ||
| @@ -149,7 +178,10 @@ func TestControlHandlersValidateAndDelegate(t *testing.T) { | |||||
| 149 | } | 178 | } | ||
| 150 | } | 179 | } | ||
| 151 | 180 | ||||
| 152 | - | type controlSpy struct{ call string } | 181 | + | type controlSpy struct { |
| 182 | + | call string | |||
| 183 | + | recoveryOutput string | |||
| 184 | + | } | |||
| 153 | 185 | ||||
| 154 | func (s *controlSpy) record(call string) error { s.call = call; return nil } | 186 | func (s *controlSpy) record(call string) error { s.call = call; return nil } | ||
| 155 | func (s *controlSpy) CreateTab() error { return s.record("new-tab") } | 187 | func (s *controlSpy) CreateTab() error { return s.record("new-tab") } | ||
| @@ -178,6 +210,9 @@ func (s *controlSpy) RenameTabTo(v string) error { return s.record("renam | |||||
| 178 | func (s *controlSpy) Toast(level, message string) error { | 210 | func (s *controlSpy) Toast(level, message string) error { | ||
| 179 | return s.record("toast:" + level + ":" + message) | 211 | return s.record("toast:" + level + ":" + message) | ||
| 180 | } | 212 | } | ||
| 213 | + | func (s *controlSpy) SessionRecovery(action string) (string, error) { | |||
| 214 | + | return s.recoveryOutput, s.record("session-recovery:" + action) | |||
| 215 | + | } | |||
| 181 | func (s *controlSpy) ListSessions(json bool) (string, error) { | 216 | func (s *controlSpy) ListSessions(json bool) (string, error) { | ||
| 182 | _ = s.record("list-sessions:" + strconv.FormatBool(json)) | 217 | _ = s.record("list-sessions:" + strconv.FormatBool(json)) | ||
| 183 | return "sessions", nil | 218 | return "sessions", nil | ||
A internal/usecase/daemon/coderabbit_regression_test.go +160 -0
| @@ -0,0 +1,160 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "bytes" | |||
| 5 | + | "context" | |||
| 6 | + | "errors" | |||
| 7 | + | "sync" | |||
| 8 | + | "syscall" | |||
| 9 | + | "testing" | |||
| 10 | + | ||||
| 11 | + | "github.com/stretchr/testify/require" | |||
| 12 | + | ||||
| 13 | + | "github.com/bnema/vev/internal/domain" | |||
| 14 | + | "github.com/bnema/vev/internal/ports" | |||
| 15 | + | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | |||
| 16 | + | ) | |||
| 17 | + | ||||
| 18 | + | func TestRestoreTransientLoadFailureBecomesDiscardable(t *testing.T) { | |||
| 19 | + | record := durableRecoveryRecord(0) | |||
| 20 | + | repository := &durableRecoveryRepository{ | |||
| 21 | + | errors: map[string]error{record.Name: syscall.ENOSPC}, | |||
| 22 | + | loads: make(map[string]int), | |||
| 23 | + | repairs: make(map[string]domain.CheckpointRef), | |||
| 24 | + | } | |||
| 25 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |||
| 26 | + | ||||
| 27 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |||
| 28 | + | ||||
| 29 | + | persisted, ok, err := catalogue.Record(record.Name) | |||
| 30 | + | require.NoError(t, err) | |||
| 31 | + | require.True(t, ok) | |||
| 32 | + | require.Equal(t, "checkpoint load failed", persisted.DegradedReason) | |||
| 33 | + | d.mu.Lock() | |||
| 34 | + | entry := d.stopped[record.Name] | |||
| 35 | + | d.mu.Unlock() | |||
| 36 | + | require.Equal(t, ports.SessionBroken, entry.state) | |||
| 37 | + | require.Equal(t, persisted, entry.record) | |||
| 38 | + | require.Equal(t, ports.SessionBroken, listSessions(t, d).Sessions[0].State) | |||
| 39 | + | select { | |||
| 40 | + | case <-entry.restoreDone: | |||
| 41 | + | default: | |||
| 42 | + | t.Fatal("restore completion was not signaled") | |||
| 43 | + | } | |||
| 44 | + | } | |||
| 45 | + | ||||
| 46 | + | type blockingDiscardRecoveryRepository struct { | |||
| 47 | + | *durableRecoveryRepository | |||
| 48 | + | deleteEntered chan struct{} | |||
| 49 | + | releaseDelete chan struct{} | |||
| 50 | + | } | |||
| 51 | + | ||||
| 52 | + | func (r *blockingDiscardRecoveryRepository) DeleteIncarnation(context.Context, domain.IncarnationID) error { | |||
| 53 | + | close(r.deleteEntered) | |||
| 54 | + | <-r.releaseDelete | |||
| 55 | + | return nil | |||
| 56 | + | } | |||
| 57 | + | ||||
| 58 | + | func TestRestoreFailureCannotOverwriteDiscardReplacement(t *testing.T) { | |||
| 59 | + | record := durableRecoveryRecord(0) | |||
| 60 | + | record.DegradedReason = "previous restore failure" | |||
| 61 | + | repository := &blockingDiscardRecoveryRepository{ | |||
| 62 | + | durableRecoveryRepository: &durableRecoveryRepository{}, | |||
| 63 | + | deleteEntered: make(chan struct{}), | |||
| 64 | + | releaseDelete: make(chan struct{}), | |||
| 65 | + | } | |||
| 66 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |||
| 67 | + | d.recovery = recoveryusecase.NewCoordinator(catalogue, repository, bytes.NewReader(bytes.Repeat([]byte{7}, 16))) | |||
| 68 | + | ||||
| 69 | + | discarded := make(chan error, 1) | |||
| 70 | + | go func() { | |||
| 71 | + | _, err := (controlExec{ctx: t.Context(), d: d, recoveryName: record.Name}).SessionRecovery("discard") | |||
| 72 | + | discarded <- err | |||
| 73 | + | }() | |||
| 74 | + | <-repository.deleteEntered // The fresh record is now authoritative behind Discard's mutation fence. | |||
| 75 | + | ||||
| 76 | + | finished := make(chan struct{}) | |||
| 77 | + | go func() { | |||
| 78 | + | d.finishRecordRestore(record, errors.New("late restore failure"), make(chan struct{})) | |||
| 79 | + | close(finished) | |||
| 80 | + | }() | |||
| 81 | + | ||||
| 82 | + | close(repository.releaseDelete) | |||
| 83 | + | require.NoError(t, <-discarded) | |||
| 84 | + | <-finished | |||
| 85 | + | ||||
| 86 | + | persisted, ok, err := catalogue.Record(record.Name) | |||
| 87 | + | require.NoError(t, err) | |||
| 88 | + | require.True(t, ok) | |||
| 89 | + | require.NotEqual(t, record.IncarnationID, persisted.IncarnationID) | |||
| 90 | + | require.Nil(t, persisted.Committed) | |||
| 91 | + | require.Empty(t, persisted.DegradedReason) | |||
| 92 | + | d.mu.Lock() | |||
| 93 | + | entry := d.stopped[record.Name] | |||
| 94 | + | d.mu.Unlock() | |||
| 95 | + | require.Equal(t, persisted, entry.record) | |||
| 96 | + | require.Equal(t, ports.SessionStopped, entry.state) | |||
| 97 | + | } | |||
| 98 | + | ||||
| 99 | + | func TestFinishRecordRestoreAlwaysClosesProvidedBarrier(t *testing.T) { | |||
| 100 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 101 | + | done := make(chan struct{}) | |||
| 102 | + | ||||
| 103 | + | d.finishRecordRestore(domain.CatalogueRecord{Name: "removed"}, nil, done) | |||
| 104 | + | ||||
| 105 | + | select { | |||
| 106 | + | case <-done: | |||
| 107 | + | default: | |||
| 108 | + | t.Fatal("restore completion was not signaled") | |||
| 109 | + | } | |||
| 110 | + | } | |||
| 111 | + | ||||
| 112 | + | type blockingRecordCatalogue struct { | |||
| 113 | + | ports.Catalogue | |||
| 114 | + | record domain.CatalogueRecord | |||
| 115 | + | entered chan struct{} | |||
| 116 | + | release chan struct{} | |||
| 117 | + | } | |||
| 118 | + | ||||
| 119 | + | func (c *blockingRecordCatalogue) Record(string) (domain.CatalogueRecord, bool, error) { | |||
| 120 | + | close(c.entered) | |||
| 121 | + | <-c.release | |||
| 122 | + | return c.record, true, nil | |||
| 123 | + | } | |||
| 124 | + | ||||
| 125 | + | func TestCreateSessionRechecksShutdownAfterCatalogueRead(t *testing.T) { | |||
| 126 | + | record := durableRecoveryRecord(0) | |||
| 127 | + | catalogue := &blockingRecordCatalogue{record: record, entered: make(chan struct{}), release: make(chan struct{})} | |||
| 128 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 129 | + | d.catalogue = catalogue | |||
| 130 | + | d.persistEnabled = true | |||
| 131 | + | d.stopped[record.Name] = stoppedSessionFromRecord(record, ports.SessionStopped, make(chan struct{})) | |||
| 132 | + | ||||
| 133 | + | result := make(chan error, 1) | |||
| 134 | + | go func() { | |||
| 135 | + | _, err := createSessionForTest(d, record.Name, false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, nil) | |||
| 136 | + | result <- err | |||
| 137 | + | }() | |||
| 138 | + | <-catalogue.entered | |||
| 139 | + | d.mu.Lock() | |||
| 140 | + | d.closing = true | |||
| 141 | + | d.mu.Unlock() | |||
| 142 | + | close(catalogue.release) | |||
| 143 | + | ||||
| 144 | + | err := <-result | |||
| 145 | + | require.ErrorContains(t, err, "daemon is shutting down") | |||
| 146 | + | d.mu.Lock() | |||
| 147 | + | defer d.mu.Unlock() | |||
| 148 | + | require.Empty(t, d.sessions) | |||
| 149 | + | } | |||
| 150 | + | ||||
| 151 | + | func TestSnapshotStopContextCancelIsIdempotent(t *testing.T) { | |||
| 152 | + | deadline := &snapshotShutdownDeadline{done: make(chan struct{})} | |||
| 153 | + | _, cancel := snapshotStopContext(deadline) | |||
| 154 | + | ||||
| 155 | + | var wg sync.WaitGroup | |||
| 156 | + | for range 8 { | |||
| 157 | + | wg.Go(cancel) | |||
| 158 | + | } | |||
| 159 | + | wg.Wait() | |||
| 160 | + | } | |||
M internal/usecase/daemon/control.go +38 -17
| @@ -1,9 +1,11 @@ | |||||
| 1 | package daemon | 1 | package daemon | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | + | "context" | |||
| 4 | "encoding/json" | 5 | "encoding/json" | ||
| 5 | "errors" | 6 | "errors" | ||
| 6 | "fmt" | 7 | "fmt" | ||
| 8 | + | "slices" | |||
| 7 | "sort" | 9 | "sort" | ||
| 8 | "strings" | 10 | "strings" | ||
| 9 | 11 | ||||
| @@ -27,14 +29,14 @@ func (d *Daemon) handleCommand(tr ports.Transport, f ports.Frame) { | |||||
| 27 | _ = tr.Send(frameCommandResult(ports.CommandResult{Code: ports.ErrInternal, Text: "malformed command request"})) | 29 | _ = tr.Send(frameCommandResult(ports.CommandResult{Code: ports.ErrInternal, Text: "malformed command request"})) | ||
| 28 | return | 30 | return | ||
| 29 | } | 31 | } | ||
| 30 | - | _ = tr.Send(frameCommandResult(d.dispatchCommand(request))) | 32 | + | _ = tr.Send(frameCommandResult(d.dispatchCommand(d.serveCtx, request))) |
| 31 | } | 33 | } | ||
| 32 | 34 | ||||
| 33 | func frameCommandResult(result ports.CommandResult) ports.Frame { | 35 | func frameCommandResult(result ports.CommandResult) ports.Frame { | ||
| 34 | return ports.Frame{Type: ports.MsgCommandResult, Payload: ports.MarshalCommandResult(result)} | 36 | return ports.Frame{Type: ports.MsgCommandResult, Payload: ports.MarshalCommandResult(result)} | ||
| 35 | } | 37 | } | ||
| 36 | 38 | ||||
| 37 | - | func (d *Daemon) dispatchCommand(request ports.CommandRequest) ports.CommandResult { | 39 | + | func (d *Daemon) dispatchCommand(ctx context.Context, request ports.CommandRequest) ports.CommandResult { |
| 38 | cmd, ok := command.BySlug(request.Slug) | 40 | cmd, ok := command.BySlug(request.Slug) | ||
| 39 | if !ok { | 41 | if !ok { | ||
| 40 | return commandFailure(ports.ErrUnknownCommand, "unknown command: "+request.Slug) | 42 | return commandFailure(ports.ErrUnknownCommand, "unknown command: "+request.Slug) | ||
| @@ -43,7 +45,7 @@ func (d *Daemon) dispatchCommand(request ports.CommandRequest) ports.CommandResu | |||||
| 43 | return commandFailure(ports.ErrNotScriptable, request.Slug+" requires an attached client") | 45 | return commandFailure(ports.ErrNotScriptable, request.Slug+" requires an attached client") | ||
| 44 | } | 46 | } | ||
| 45 | if cmd.Target == command.TargetNone && !request.Self { | 47 | if cmd.Target == command.TargetNone && !request.Self { | ||
| 46 | - | return d.runControl(cmd, controlExec{d: d}, request) | 48 | + | return d.runControl(cmd, controlExec{ctx: ctx, d: d, recoveryName: request.TargetSession}, request) |
| 47 | } | 49 | } | ||
| 48 | 50 | ||||
| 49 | sess, code, text := d.resolveTargetSession(request) | 51 | sess, code, text := d.resolveTargetSession(request) | ||
| @@ -57,7 +59,7 @@ func (d *Daemon) dispatchCommand(request ports.CommandRequest) ports.CommandResu | |||||
| 57 | if code != 0 { | 59 | if code != 0 { | ||
| 58 | return commandFailure(code, text) | 60 | return commandFailure(code, text) | ||
| 59 | } | 61 | } | ||
| 60 | - | return d.runControl(cmd, controlExec{d: d, sess: sess, tab: tb, target: daemonActionTarget{session: sess, tab: tb, pane: pane}}, request) | 62 | + | return d.runControl(cmd, controlExec{ctx: ctx, d: d, sess: sess, tab: tb, target: daemonActionTarget{session: sess, tab: tb, pane: pane}}, request) |
| 61 | } | 63 | } | ||
| 62 | 64 | ||||
| 63 | func (d *Daemon) runControl(cmd command.Command, exec controlExec, request ports.CommandRequest) ports.CommandResult { | 65 | func (d *Daemon) runControl(cmd command.Command, exec controlExec, request ports.CommandRequest) ports.CommandResult { | ||
| @@ -306,14 +308,7 @@ func (d *Daemon) hasDaemonActionPaneTarget(target daemonActionTarget) bool { | |||||
| 306 | } | 308 | } | ||
| 307 | target.session.mu.Lock() | 309 | target.session.mu.Lock() | ||
| 308 | defer target.session.mu.Unlock() | 310 | defer target.session.mu.Unlock() | ||
| 309 | - | foundTab := false | 311 | + | if !slices.Contains(target.session.tabs, target.tab) { |
| 310 | - | for _, tb := range target.session.tabs { | |||
| 311 | - | if tb == target.tab { | |||
| 312 | - | foundTab = true | |||
| 313 | - | break | |||
| 314 | - | } | |||
| 315 | - | } | |||
| 316 | - | if !foundTab { | |||
| 317 | return false | 312 | return false | ||
| 318 | } | 313 | } | ||
| 319 | target.tab.mu.Lock() | 314 | target.tab.mu.Lock() | ||
| @@ -332,11 +327,13 @@ func (a daemonActions) switchRelative(sess *session, delta int) error { | |||||
| 332 | // controlExec implements command.ControlContext against resolved daemon-owned | 327 | // controlExec implements command.ControlContext against resolved daemon-owned | ||
| 333 | // targets. It deliberately contains no attached-client state. | 328 | // targets. It deliberately contains no attached-client state. | ||
| 334 | type controlExec struct { | 329 | type controlExec struct { | ||
| 335 | - | d *Daemon | 330 | + | ctx context.Context |
| 336 | - | sess *session | 331 | + | d *Daemon |
| 337 | - | tab *tab | 332 | + | sess *session |
| 338 | - | target daemonActionTarget | 333 | + | tab *tab |
| 339 | - | actions daemonActionRunner | 334 | + | recoveryName string |
| 335 | + | target daemonActionTarget | |||
| 336 | + | actions daemonActionRunner | |||
| 340 | } | 337 | } | ||
| 341 | 338 | ||||
| 342 | func (e controlExec) runAction(request daemonActionRequest) error { | 339 | func (e controlExec) runAction(request daemonActionRequest) error { | ||
| @@ -457,6 +454,30 @@ func (e controlExec) Toast(severity, message string) error { | |||||
| 457 | return nil | 454 | return nil | ||
| 458 | } | 455 | } | ||
| 459 | 456 | ||||
| 457 | + | func (e controlExec) SessionRecovery(action string) (string, error) { | |||
| 458 | + | if e.d == nil || e.d.recovery == nil || e.recoveryName == "" || action != "discard" { | |||
| 459 | + | return "", command.ErrInvalidArguments | |||
| 460 | + | } | |||
| 461 | + | ctx := e.ctx | |||
| 462 | + | if ctx == nil { | |||
| 463 | + | ctx = e.d.serveCtx | |||
| 464 | + | } | |||
| 465 | + | if ctx == nil { | |||
| 466 | + | ctx = context.Background() | |||
| 467 | + | } | |||
| 468 | + | if err := e.d.recovery.Discard(ctx, e.recoveryName); err != nil { | |||
| 469 | + | return "", err | |||
| 470 | + | } | |||
| 471 | + | record, ok, err := e.d.catalogue.Record(e.recoveryName) | |||
| 472 | + | if err != nil { | |||
| 473 | + | return "", err | |||
| 474 | + | } | |||
| 475 | + | if ok { | |||
| 476 | + | e.d.setStoppedRecovery(record, ports.SessionStopped) | |||
| 477 | + | } | |||
| 478 | + | return "", nil | |||
| 479 | + | } | |||
| 480 | + | ||||
| 460 | func (e controlExec) ListSessions(asJSON bool) (string, error) { | 481 | func (e controlExec) ListSessions(asJSON bool) (string, error) { | ||
| 461 | type row struct { | 482 | type row struct { | ||
| 462 | Name string `json:"name"` | 483 | Name string `json:"name"` | ||
M internal/usecase/daemon/daemon.go +270 -86
| @@ -7,15 +7,16 @@ import ( | |||||
| 7 | "log/slog" | 7 | "log/slog" | ||
| 8 | "math" | 8 | "math" | ||
| 9 | "os" | 9 | "os" | ||
| 10 | + | "reflect" | |||
| 10 | "sort" | 11 | "sort" | ||
| 11 | "sync" | 12 | "sync" | ||
| 12 | "sync/atomic" | 13 | "sync/atomic" | ||
| 13 | "time" | 14 | "time" | ||
| 14 | 15 | ||||
| 15 | "github.com/bnema/vev/internal/domain" | 16 | "github.com/bnema/vev/internal/domain" | ||
| 16 | - | "github.com/bnema/vev/internal/persist" | |||
| 17 | "github.com/bnema/vev/internal/ports" | 17 | "github.com/bnema/vev/internal/ports" | ||
| 18 | "github.com/bnema/vev/internal/usecase/keys" | 18 | "github.com/bnema/vev/internal/usecase/keys" | ||
| 19 | + | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | |||
| 19 | ) | 20 | ) | ||
| 20 | 21 | ||||
| 21 | // Scheduler debounce bounds. Idle updates use the minimum for low latency; | 22 | // Scheduler debounce bounds. Idle updates use the minimum for low latency; | ||
| @@ -70,6 +71,8 @@ type Daemon struct { | |||||
| 70 | mu sync.Mutex | 71 | mu sync.Mutex | ||
| 71 | sessions map[domain.SessionID]*session | 72 | sessions map[domain.SessionID]*session | ||
| 72 | stopped map[string]stoppedSession | 73 | stopped map[string]stoppedSession | ||
| 74 | + | // creating reserves names while durable creation I/O runs without mu. | |||
| 75 | + | creating map[string]struct{} | |||
| 73 | nextID uint64 | 76 | nextID uint64 | ||
| 74 | // lastAllocatedCreatedAt is the named-session lifecycle timestamp high-water | 77 | // lastAllocatedCreatedAt is the named-session lifecycle timestamp high-water | ||
| 75 | // mark. It is guarded by mu and prevents a wall-clock regression from | 78 | // mark. It is guarded by mu and prevents a wall-clock regression from | ||
| @@ -117,19 +120,25 @@ type Daemon struct { | |||||
| 117 | shell string | 120 | shell string | ||
| 118 | shellArgs []string | 121 | shellArgs []string | ||
| 119 | shellOverride bool | 122 | shellOverride bool | ||
| 120 | - | persist *persist.Persister | |||
| 121 | persistEnabled bool | 123 | persistEnabled bool | ||
| 122 | - | // snapshotRepository is the sole checkpoint contract. legacySnapshots is | 124 | + | catalogue ports.Catalogue |
| 123 | - | // read-only migration input and is never used for new writes. | 125 | + | catalogueRecords []domain.CatalogueRecord |
| 124 | - | snapshotRepository ports.SnapshotRepository | 126 | + | catalogueRecordsProvided bool |
| 125 | - | legacySnapshots ports.LegacySnapshotSource | 127 | + | // snapshotRepository is the sole checkpoint storage contract. |
| 126 | - | snapsEnabled bool | 128 | + | snapshotRepository ports.SnapshotRepository |
| 127 | - | // legacyImportPending holds verified legacy blobs whose source deletion | 129 | + | recovery *recoveryusecase.Coordinator |
| 128 | - | // failed, so a later import retries only deletion rather than publication. | 130 | + | maintenanceWorkerCancel context.CancelFunc |
| 129 | - | legacyImportMu sync.Mutex | 131 | + | maintenanceWorkerDone chan struct{} |
| 130 | - | legacyImportPending map[string][]byte | 132 | + | // restoreWorkerDone is the restoration goroutine's ownership signal. Startup |
| 131 | - | noticeStore ports.NoticeStore | 133 | + | // restoration reconciles durable checkpoints, so it is a durable writer and |
| 132 | - | snapshotJobs chan *snapshotCapture | 134 | + | // is guarded by snapshotWorkerMu with the other two. |
| 135 | + | restoreWorkerDone chan struct{} | |||
| 136 | + | snapsEnabled bool | |||
| 137 | + | noticeStore ports.NoticeStore | |||
| 138 | + | snapshotJobs chan *snapshotCapture | |||
| 139 | + | // snapshotAdmitted contains every capture accepted by either worker queue, | |||
| 140 | + | // including captures buffered in snapshotJobs. Guarded by snapshotWorkerMu. | |||
| 141 | + | snapshotAdmitted map[*snapshotCapture]struct{} | |||
| 133 | // snapshotWake wakes the repository scheduler when a session becomes dirty | 142 | // snapshotWake wakes the repository scheduler when a session becomes dirty | ||
| 134 | // or an attempt completes. It is never closed and producers only send | 143 | // or an attempt completes. It is never closed and producers only send | ||
| 135 | // non-blockingly. | 144 | // non-blockingly. | ||
| @@ -153,6 +162,8 @@ type Daemon struct { | |||||
| 153 | // producer or a repository worker. | 162 | // producer or a repository worker. | ||
| 154 | snapshotNoticeMu sync.Mutex | 163 | snapshotNoticeMu sync.Mutex | ||
| 155 | snapshotActiveFailureSignature string | 164 | snapshotActiveFailureSignature string | ||
| 165 | + | shutdownNoticeMu sync.Mutex | |||
| 166 | + | shutdownNoticedSessions map[string]struct{} | |||
| 156 | restoreDone chan struct{} | 167 | restoreDone chan struct{} | ||
| 157 | restoreOnce sync.Once | 168 | restoreOnce sync.Once | ||
| 158 | procCwd func(int) (string, error) | 169 | procCwd func(int) (string, error) | ||
| @@ -212,9 +223,13 @@ type stoppedSession struct { | |||||
| 212 | name string | 223 | name string | ||
| 213 | cwd string | 224 | cwd string | ||
| 214 | createdAt int64 | 225 | createdAt int64 | ||
| 226 | + | incarnation domain.IncarnationID | |||
| 215 | lastUsedSeq uint64 | 227 | lastUsedSeq uint64 | ||
| 216 | tabNames []string | 228 | tabNames []string | ||
| 217 | purging bool | 229 | purging bool | ||
| 230 | + | record domain.CatalogueRecord | |||
| 231 | + | state ports.SessionState | |||
| 232 | + | restoreDone chan struct{} | |||
| 218 | } | 233 | } | ||
| 219 | 234 | ||||
| 220 | type Option func(*Daemon) | 235 | type Option func(*Daemon) | ||
| @@ -237,22 +252,70 @@ func WithShell(cmd string, args []string) Option { | |||||
| 237 | } | 252 | } | ||
| 238 | } | 253 | } | ||
| 239 | 254 | ||||
| 240 | - | // WithStore enables persisted named session metadata. A nil store keeps the | 255 | + | // WithCatalogue installs the singular catalogue and the strictly opened |
| 241 | - | // daemon in no-op persistence mode. | 256 | + | // records that define the daemon's expected-session registry at startup. |
| 242 | - | func WithStore(store ports.Store) Option { | 257 | + | func WithCatalogue(catalogue ports.Catalogue, records []domain.CatalogueRecord) Option { |
| 243 | return func(d *Daemon) { | 258 | return func(d *Daemon) { | ||
| 244 | - | d.persist = persist.New(store) | 259 | + | d.catalogue = catalogue |
| 245 | - | d.persistEnabled = store != nil | 260 | + | d.persistEnabled = catalogue != nil |
| 261 | + | d.catalogueRecords = append([]domain.CatalogueRecord(nil), records...) | |||
| 262 | + | d.catalogueRecordsProvided = true | |||
| 246 | } | 263 | } | ||
| 247 | } | 264 | } | ||
| 248 | 265 | ||||
| 249 | - | // WithSnapshotRepository enables content-addressed incremental snapshots. The | 266 | + | func (d *Daemon) catalogueRecord(name string) (domain.CatalogueRecord, bool, error) { |
| 250 | - | // legacy source is read-only migration input and is never used for new writes. | 267 | + | if d == nil || d.catalogue == nil { |
| 251 | - | func WithSnapshotRepository(repository ports.SnapshotRepository, legacy ports.LegacySnapshotSource) Option { | 268 | + | return domain.CatalogueRecord{}, false, nil |
| 269 | + | } | |||
| 270 | + | return d.catalogue.Record(name) | |||
| 271 | + | } | |||
| 272 | + | ||||
| 273 | + | // markCatalogueDirty buffers metadata. The periodic catalogue timer, the next | |||
| 274 | + | // identity write, or shutdown provides the durability barrier. | |||
| 275 | + | func (d *Daemon) markCatalogueDirty(update domain.CatalogueMetadataUpdate) { | |||
| 276 | + | if d == nil || d.catalogue == nil { | |||
| 277 | + | return | |||
| 278 | + | } | |||
| 279 | + | if err := d.catalogue.UpdateMetadata(update); err != nil { | |||
| 280 | + | d.log.Warn("buffering session metadata failed", "err", err, "session", update.Name) | |||
| 281 | + | } | |||
| 282 | + | } | |||
| 283 | + | ||||
| 284 | + | func (d *Daemon) flushCatalogue() error { | |||
| 285 | + | if d == nil || d.catalogue == nil { | |||
| 286 | + | return nil | |||
| 287 | + | } | |||
| 288 | + | return d.catalogue.Sync() | |||
| 289 | + | } | |||
| 290 | + | ||||
| 291 | + | // WithRecoveryCoordinator installs the durable recovery coordinator. | |||
| 292 | + | func WithRecoveryCoordinator(coordinator *recoveryusecase.Coordinator) Option { | |||
| 252 | return func(d *Daemon) { | 293 | return func(d *Daemon) { | ||
| 294 | + | d.recovery = coordinator | |||
| 295 | + | } | |||
| 296 | + | } | |||
| 297 | + | ||||
| 298 | + | // WithSnapshotRepository enables content-addressed incremental snapshots. | |||
| 299 | + | func WithSnapshotRepository(repository ports.SnapshotRepository) Option { | |||
| 300 | + | return func(d *Daemon) { | |||
| 301 | + | if isNilSnapshotRepository(repository) { | |||
| 302 | + | return | |||
| 303 | + | } | |||
| 253 | d.snapshotRepository = repository | 304 | d.snapshotRepository = repository | ||
| 254 | - | d.legacySnapshots = legacy | 305 | + | d.snapsEnabled = true |
| 255 | - | d.snapsEnabled = repository != nil | 306 | + | } |
| 307 | + | } | |||
| 308 | + | ||||
| 309 | + | func isNilSnapshotRepository(repository ports.SnapshotRepository) bool { | |||
| 310 | + | if repository == nil { | |||
| 311 | + | return true | |||
| 312 | + | } | |||
| 313 | + | value := reflect.ValueOf(repository) | |||
| 314 | + | switch value.Kind() { | |||
| 315 | + | case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: | |||
| 316 | + | return value.IsNil() | |||
| 317 | + | default: | |||
| 318 | + | return false | |||
| 256 | } | 319 | } | ||
| 257 | } | 320 | } | ||
| 258 | 321 | ||||
| @@ -262,6 +325,17 @@ func WithNoticeStore(store ports.NoticeStore) Option { | |||||
| 262 | return func(d *Daemon) { d.noticeStore = store } | 325 | return func(d *Daemon) { d.noticeStore = store } | ||
| 263 | } | 326 | } | ||
| 264 | 327 | ||||
| 328 | + | // WithDurableMaintenance retains the application wiring for the one | |||
| 329 | + | // pre-publication GC pass. Standalone users without an explicitly supplied | |||
| 330 | + | // coordinator receive the same canonical coordinator path. | |||
| 331 | + | func WithDurableMaintenance(catalogue ports.Catalogue, repository ports.SnapshotRepository) Option { | |||
| 332 | + | return func(d *Daemon) { | |||
| 333 | + | if d.recovery == nil { | |||
| 334 | + | d.recovery = recoveryusecase.NewCoordinator(catalogue, repository, nil) | |||
| 335 | + | } | |||
| 336 | + | } | |||
| 337 | + | } | |||
| 338 | + | ||||
| 265 | // WithCwdReader overrides the process cwd reader used for persistence tests. | 339 | // WithCwdReader overrides the process cwd reader used for persistence tests. | ||
| 266 | func WithCwdReader(fn func(int) (string, error)) Option { | 340 | func WithCwdReader(fn func(int) (string, error)) Option { | ||
| 267 | return func(d *Daemon) { | 341 | return func(d *Daemon) { | ||
| @@ -347,35 +421,47 @@ func (d *Daemon) nowUnixNano() int64 { | |||||
| 347 | return d.clock.Now().UnixNano() | 421 | return d.clock.Now().UnixNano() | ||
| 348 | } | 422 | } | ||
| 349 | 423 | ||||
| 424 | + | type systemClock struct{} | |||
| 425 | + | type systemTimer struct{ *time.Timer } | |||
| 426 | + | ||||
| 427 | + | func (systemClock) Now() time.Time { return time.Now() } | |||
| 428 | + | func (systemClock) NewTimer(delay time.Duration) ports.Timer { | |||
| 429 | + | return systemTimer{Timer: time.NewTimer(delay)} | |||
| 430 | + | } | |||
| 431 | + | func (t systemTimer) C() <-chan time.Time { return t.Timer.C } | |||
| 432 | + | ||||
| 350 | // New constructs a Daemon. ptys spawns PTY-backed children, clock drives the | 433 | // New constructs a Daemon. ptys spawns PTY-backed children, clock drives the | ||
| 351 | // render debounce, and log receives diagnostics (defaults to slog.Default). | 434 | // render debounce, and log receives diagnostics (defaults to slog.Default). | ||
| 352 | func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Option) *Daemon { | 435 | func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Option) *Daemon { | ||
| 353 | if log == nil { | 436 | if log == nil { | ||
| 354 | log = slog.Default() | 437 | log = slog.Default() | ||
| 355 | } | 438 | } | ||
| 439 | + | if clock == nil { | |||
| 440 | + | clock = systemClock{} | |||
| 441 | + | } | |||
| 356 | shell := os.Getenv("SHELL") | 442 | shell := os.Getenv("SHELL") | ||
| 357 | if shell == "" { | 443 | if shell == "" { | ||
| 358 | shell = "/bin/sh" | 444 | shell = "/bin/sh" | ||
| 359 | } | 445 | } | ||
| 360 | d := &Daemon{ | 446 | d := &Daemon{ | ||
| 361 | - | sessions: make(map[domain.SessionID]*session), | 447 | + | sessions: make(map[domain.SessionID]*session), |
| 362 | - | stopped: make(map[string]stoppedSession), | 448 | + | stopped: make(map[string]stoppedSession), |
| 363 | - | parked: make(map[uint64]*parkedAttachment), | 449 | + | creating: make(map[string]struct{}), |
| 364 | - | ptys: ptys, | 450 | + | parked: make(map[uint64]*parkedAttachment), |
| 365 | - | clock: clock, | 451 | + | ptys: ptys, |
| 366 | - | log: log, | 452 | + | clock: clock, |
| 367 | - | baseEnv: os.Environ(), | 453 | + | log: log, |
| 368 | - | shell: shell, | 454 | + | baseEnv: os.Environ(), |
| 369 | - | persist: persist.New(nil), | 455 | + | shell: shell, |
| 370 | - | dirOrHome: dirOrHome, | 456 | + | dirOrHome: dirOrHome, |
| 371 | - | done: make(chan struct{}), | 457 | + | done: make(chan struct{}), |
| 372 | - | restoreDone: make(chan struct{}), | 458 | + | restoreDone: make(chan struct{}), |
| 373 | - | animWake: make(chan struct{}, 1), | 459 | + | animWake: make(chan struct{}, 1), |
| 374 | - | legacyImportPending: make(map[string][]byte), | 460 | + | snapshotJobs: make(chan *snapshotCapture, snapshotQueueCapacity), |
| 375 | - | snapshotJobs: make(chan *snapshotCapture, snapshotQueueCapacity), | 461 | + | snapshotAdmitted: make(map[*snapshotCapture]struct{}), |
| 376 | - | snapshotWake: make(chan struct{}, 1), | 462 | + | snapshotWake: make(chan struct{}, 1), |
| 377 | - | notices: newNoticeCenter(), | 463 | + | notices: newNoticeCenter(), |
| 378 | - | resumeParkGrace: defaultResumeParkGrace, | 464 | + | resumeParkGrace: defaultResumeParkGrace, |
| 379 | barScripts: &barScriptState{ | 465 | barScripts: &barScriptState{ | ||
| 380 | cfg: barConfigFromDomain(domain.Defaults().Bar), | 466 | cfg: barConfigFromDomain(domain.Defaults().Bar), | ||
| 381 | outputs: make(map[domain.SessionID]barScriptOutputs), | 467 | outputs: make(map[domain.SessionID]barScriptOutputs), | ||
| @@ -395,9 +481,6 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt | |||||
| 395 | for _, o := range opts { | 481 | for _, o := range opts { | ||
| 396 | o(d) | 482 | o(d) | ||
| 397 | } | 483 | } | ||
| 398 | - | if d.persist == nil { | |||
| 399 | - | d.persist = persist.New(nil) | |||
| 400 | - | } | |||
| 401 | if d.dirOrHome == nil { | 484 | if d.dirOrHome == nil { | ||
| 402 | d.dirOrHome = dirOrHome | 485 | d.dirOrHome = dirOrHome | ||
| 403 | } | 486 | } | ||
| @@ -412,27 +495,29 @@ func New(ptys ports.PTYFactory, clock ports.Clock, log *slog.Logger, opts ...Opt | |||||
| 412 | allow := buildRestoreProcessAllowlist(domain.DefaultSnapshotRestoreProcesses()) | 495 | allow := buildRestoreProcessAllowlist(domain.DefaultSnapshotRestoreProcesses()) | ||
| 413 | d.restoreProcessAllowlist.Store(&allow) | 496 | d.restoreProcessAllowlist.Store(&allow) | ||
| 414 | } | 497 | } | ||
| 415 | - | if records, err := d.persist.LoadAll(); err != nil { | 498 | + | records := d.catalogueRecords |
| 416 | - | d.log.Warn("loading persisted sessions failed", "err", err) | 499 | + | var maxSeq uint64 |
| 417 | - | } else { | 500 | + | var maxCreatedAt int64 |
| 418 | - | var maxSeq uint64 | 501 | + | hasCreatedAt := false |
| 419 | - | var maxCreatedAt int64 | 502 | + | for _, r := range records { |
| 420 | - | hasCreatedAt := false | 503 | + | if d.catalogueRecordsProvided { |
| 421 | - | for _, r := range records { | 504 | + | state, done := initialSessionState(r) |
| 422 | - | d.stopped[r.Name] = stoppedSession{name: r.Name, cwd: r.Cwd, createdAt: r.CreatedAt, lastUsedSeq: r.LastUsedSeq, tabNames: r.TabNames} | 505 | + | d.stopped[r.Name] = stoppedSessionFromRecord(r, state, done) |
| 423 | - | if !hasCreatedAt || r.CreatedAt > maxCreatedAt { | 506 | + | } else { |
| 424 | - | maxCreatedAt = r.CreatedAt | 507 | + | d.stopped[r.Name] = stoppedSession{name: r.Name, cwd: r.Cwd, createdAt: r.CreatedAt, incarnation: r.IncarnationID, lastUsedSeq: r.LastUsedSeq, tabNames: append([]string(nil), r.TabNames...)} |
| 425 | - | hasCreatedAt = true | 508 | + | } |
| 426 | - | } | 509 | + | if !hasCreatedAt || r.CreatedAt > maxCreatedAt { |
| 427 | - | if r.LastUsedSeq > maxSeq { | 510 | + | maxCreatedAt = r.CreatedAt |
| 428 | - | maxSeq = r.LastUsedSeq | 511 | + | hasCreatedAt = true |
| 429 | - | } | |||
| 430 | } | 512 | } | ||
| 431 | - | if hasCreatedAt { | 513 | + | if r.LastUsedSeq > maxSeq { |
| 432 | - | d.lastAllocatedCreatedAt = maxCreatedAt | 514 | + | maxSeq = r.LastUsedSeq |
| 433 | } | 515 | } | ||
| 434 | - | d.mruSeq.Store(maxSeq) | |||
| 435 | } | 516 | } | ||
| 517 | + | if hasCreatedAt { | |||
| 518 | + | d.lastAllocatedCreatedAt = maxCreatedAt | |||
| 519 | + | } | |||
| 520 | + | d.mruSeq.Store(maxSeq) | |||
| 436 | return d | 521 | return d | ||
| 437 | } | 522 | } | ||
| 438 | 523 | ||||
| @@ -451,7 +536,7 @@ func (d *Daemon) Serve(ctx context.Context, l ports.Listener) error { | |||||
| 451 | d.sessWg.Go(func() { | 536 | d.sessWg.Go(func() { | ||
| 452 | d.barScriptPoller(d.serveCtx) | 537 | d.barScriptPoller(d.serveCtx) | ||
| 453 | }) | 538 | }) | ||
| 454 | - | if d.persistEnabled && d.procCwd != nil { | 539 | + | if d.persistEnabled { |
| 455 | d.sessWg.Go(func() { | 540 | d.sessWg.Go(func() { | ||
| 456 | d.cwdSampler(d.serveCtx) | 541 | d.cwdSampler(d.serveCtx) | ||
| 457 | }) | 542 | }) | ||
| @@ -461,13 +546,10 @@ func (d *Daemon) Serve(ctx context.Context, l ports.Listener) error { | |||||
| 461 | d.sessWg.Go(func() { | 546 | d.sessWg.Go(func() { | ||
| 462 | d.snapshotRepositorySaver(d.serveCtx) | 547 | d.snapshotRepositorySaver(d.serveCtx) | ||
| 463 | }) | 548 | }) | ||
| 464 | - | d.sessWg.Go(func() { | 549 | + | d.startSnapshotRestoration() |
| 465 | - | d.restoreSnapshots(d.serveCtx) | |||
| 466 | - | }) | |||
| 467 | } else { | 550 | } else { | ||
| 468 | d.closeRestoreDone() | 551 | d.closeRestoreDone() | ||
| 469 | } | 552 | } | ||
| 470 | - | ||||
| 471 | // Break the accept loop when either the parent context is cancelled or the | 553 | // Break the accept loop when either the parent context is cancelled or the | ||
| 472 | // registry drains to empty: both close the listener, which fails Accept. | 554 | // registry drains to empty: both close the listener, which fails Accept. | ||
| 473 | go func() { | 555 | go func() { | ||
| @@ -520,14 +602,26 @@ func (d *Daemon) Serve(ctx context.Context, l ports.Listener) error { | |||||
| 520 | d.connWg.Wait() | 602 | d.connWg.Wait() | ||
| 521 | d.attachmentCleanupWg.Wait() | 603 | d.attachmentCleanupWg.Wait() | ||
| 522 | // Forced terminal checkpoints run before shutdown stops producers or the | 604 | // Forced terminal checkpoints run before shutdown stops producers or the | ||
| 523 | - | // worker. The same deadline bounds the worker's final drain, including a | 605 | + | // worker. StopDurableWriters reports every session whose final capture still |
| 524 | - | // repository call that ignores cancellation. | 606 | + | // owns work when the shared checkpoint budget expires. Persist one notice per |
| 525 | - | d.stopSnapshotEncodeWorkerWithDeadline(snapshotDeadline) | 607 | + | // affected session, then retain ownership until every writer actually exits. |
| 608 | + | stopCtx, stopCancel := snapshotStopContext(snapshotDeadline) | |||
| 609 | + | timedOutSessions := d.StopDurableWriters(stopCtx) | |||
| 610 | + | stopCancel() | |||
| 611 | + | for _, name := range timedOutSessions { | |||
| 612 | + | d.persistShutdownSnapshotFailure(name, context.DeadlineExceeded) | |||
| 613 | + | } | |||
| 614 | + | d.WaitDurableWriters() | |||
| 526 | d.shutdownAllWithSnapshotDeadline(ports.ReasonServerShutdown, snapshotDeadline) | 615 | d.shutdownAllWithSnapshotDeadline(ports.ReasonServerShutdown, snapshotDeadline) | ||
| 527 | d.waitSessionWorkersWithSnapshotDeadline(snapshotDeadline) | 616 | d.waitSessionWorkersWithSnapshotDeadline(snapshotDeadline) | ||
| 528 | d.waitNotifies() | 617 | d.waitNotifies() | ||
| 529 | - | if err := d.persist.Close(); err != nil { | 618 | + | if err := d.flushCatalogue(); err != nil { |
| 530 | - | d.log.Warn("closing session persister failed", "err", err) | 619 | + | d.log.Warn("flushing session catalogue at shutdown failed", "err", err) |
| 620 | + | } | |||
| 621 | + | if d.catalogue != nil { | |||
| 622 | + | if err := d.catalogue.Close(); err != nil { | |||
| 623 | + | d.log.Warn("closing session catalogue failed", "err", err) | |||
| 624 | + | } | |||
| 531 | } | 625 | } | ||
| 532 | return nil | 626 | return nil | ||
| 533 | } | 627 | } | ||
| @@ -557,11 +651,23 @@ func (d *Daemon) shutdownAllWithSnapshotDeadline(reason uint8, deadline *snapsho | |||||
| 557 | for _, s := range snapshot { | 651 | for _, s := range snapshot { | ||
| 558 | // Cancellation and PTY closure must not wait behind a teardown owner that | 652 | // Cancellation and PTY closure must not wait behind a teardown owner that | ||
| 559 | // is blocked in snapshot publication or purge work. | 653 | // is blocked in snapshot publication or purge work. | ||
| 654 | + | s.mu.Lock() | |||
| 655 | + | name := s.name | |||
| 656 | + | ephemeral := s.ephemeral | |||
| 657 | + | s.mu.Unlock() | |||
| 560 | s.stopInMemoryLifecycle() | 658 | s.stopInMemoryLifecycle() | ||
| 561 | if err := d.killSessionWithSnapshotDeadline(s, reason, false, deadline); err != nil { | 659 | if err := d.killSessionWithSnapshotDeadline(s, reason, false, deadline); err != nil { | ||
| 562 | checkpointIncomplete = true | 660 | checkpointIncomplete = true | ||
| 563 | d.log.Error("closing session with unpersisted terminal state", "err", err) | 661 | d.log.Error("closing session with unpersisted terminal state", "err", err) | ||
| 564 | } | 662 | } | ||
| 663 | + | if !ephemeral && deadline != nil { | |||
| 664 | + | select { | |||
| 665 | + | case <-deadline.Done(): | |||
| 666 | + | checkpointIncomplete = true | |||
| 667 | + | d.persistShutdownSnapshotFailure(name, context.DeadlineExceeded) | |||
| 668 | + | default: | |||
| 669 | + | } | |||
| 670 | + | } | |||
| 565 | } | 671 | } | ||
| 566 | return checkpointIncomplete | 672 | return checkpointIncomplete | ||
| 567 | } | 673 | } | ||
| @@ -628,6 +734,32 @@ func (d *snapshotShutdownDeadline) Done() <-chan struct{} { | |||||
| 628 | return d.done | 734 | return d.done | ||
| 629 | } | 735 | } | ||
| 630 | 736 | ||||
| 737 | + | func snapshotStopContext(deadline *snapshotShutdownDeadline) (context.Context, context.CancelFunc) { | |||
| 738 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 739 | + | if deadline == nil { | |||
| 740 | + | return ctx, cancel | |||
| 741 | + | } | |||
| 742 | + | select { | |||
| 743 | + | case <-deadline.Done(): | |||
| 744 | + | cancel() | |||
| 745 | + | return ctx, cancel | |||
| 746 | + | default: | |||
| 747 | + | } | |||
| 748 | + | stop := make(chan struct{}) | |||
| 749 | + | go func() { | |||
| 750 | + | select { | |||
| 751 | + | case <-deadline.Done(): | |||
| 752 | + | cancel() | |||
| 753 | + | case <-stop: | |||
| 754 | + | } | |||
| 755 | + | }() | |||
| 756 | + | var stopOnce sync.Once | |||
| 757 | + | return ctx, func() { | |||
| 758 | + | stopOnce.Do(func() { close(stop) }) | |||
| 759 | + | cancel() | |||
| 760 | + | } | |||
| 761 | + | } | |||
| 762 | + | ||||
| 631 | func (d *Daemon) sessionsSnapshotLocked() []*session { | 763 | func (d *Daemon) sessionsSnapshotLocked() []*session { | ||
| 632 | snapshot := make([]*session, 0, len(d.sessions)) | 764 | snapshot := make([]*session, 0, len(d.sessions)) | ||
| 633 | for _, s := range d.sessions { | 765 | for _, s := range d.sessions { | ||
| @@ -678,6 +810,7 @@ func (d *Daemon) handleList(tr ports.Transport) { | |||||
| 678 | info := ports.SessionInfo{ | 810 | info := ports.SessionInfo{ | ||
| 679 | SessionID: string(s.id), | 811 | SessionID: string(s.id), | ||
| 680 | Name: s.name, | 812 | Name: s.name, | ||
| 813 | + | State: ports.SessionRunning, | |||
| 681 | Ephemeral: s.ephemeral, | 814 | Ephemeral: s.ephemeral, | ||
| 682 | Tabs: uint16(len(s.tabs)), | 815 | Tabs: uint16(len(s.tabs)), | ||
| 683 | Attached: s.client != nil, | 816 | Attached: s.client != nil, | ||
| @@ -687,13 +820,16 @@ func (d *Daemon) handleList(tr ports.Transport) { | |||||
| 687 | infos = append(infos, info) | 820 | infos = append(infos, info) | ||
| 688 | } | 821 | } | ||
| 689 | for name, stopped := range d.stopped { | 822 | for name, stopped := range d.stopped { | ||
| 690 | - | if stopped.purging { | |||
| 691 | - | continue | |||
| 692 | - | } | |||
| 693 | if _, live := liveNames[name]; live { | 823 | if _, live := liveNames[name]; live { | ||
| 694 | continue | 824 | continue | ||
| 695 | } | 825 | } | ||
| 696 | - | infos = append(infos, ports.SessionInfo{Name: name, Stopped: true}) | 826 | + | state := ports.SessionStopped |
| 827 | + | if stopped.purging || stopped.state == ports.SessionBroken { | |||
| 828 | + | // Purge is the dominant externally visible state: restoration must | |||
| 829 | + | // never make a deletion-reserved record appear attachable. | |||
| 830 | + | state = ports.SessionBroken | |||
| 831 | + | } | |||
| 832 | + | infos = append(infos, ports.SessionInfo{Name: name, State: state}) | |||
| 697 | } | 833 | } | ||
| 698 | d.mu.Unlock() | 834 | d.mu.Unlock() | ||
| 699 | 835 | ||||
| @@ -722,9 +858,8 @@ func (d *Daemon) handleKill(tr ports.Transport, f ports.Frame) { | |||||
| 722 | if target == nil { | 858 | if target == nil { | ||
| 723 | if _, ok := d.stopped[k.Name]; ok { | 859 | if _, ok := d.stopped[k.Name]; ok { | ||
| 724 | d.mu.Unlock() | 860 | d.mu.Unlock() | ||
| 725 | - | // Stopped sessions follow the same tombstone-backed transaction as | 861 | + | // Stopped sessions use the same catalogue-first, incarnation-second |
| 726 | - | // live and offline purges. In particular, a retained legacy source | 862 | + | // deletion order as live and offline purges. |
| 727 | - | // must never outlive metadata after an interrupted control kill. | |||
| 728 | if err := d.retryStoppedPurge(k.Name); err != nil { | 863 | if err := d.retryStoppedPurge(k.Name); err != nil { | ||
| 729 | d.log.Warn("deleting stopped session failed", "err", err, "session", k.Name) | 864 | d.log.Warn("deleting stopped session failed", "err", err, "session", k.Name) | ||
| 730 | _ = tr.Send(frameError(ports.ErrInternal, "deleting stopped session failed")) | 865 | _ = tr.Send(frameError(ports.ErrInternal, "deleting stopped session failed")) | ||
| @@ -830,6 +965,53 @@ func (d *Daemon) finishAttach(sess *session, tr ports.Transport, sz domain.Size, | |||||
| 830 | return ac | 965 | return ac | ||
| 831 | } | 966 | } | ||
| 832 | 967 | ||||
| 968 | + | func (d *Daemon) waitForTargetRestore(ctx context.Context, name string) error { | |||
| 969 | + | d.mu.Lock() | |||
| 970 | + | var ( | |||
| 971 | + | done chan struct{} | |||
| 972 | + | stopped stoppedSession | |||
| 973 | + | ok bool | |||
| 974 | + | ) | |||
| 975 | + | if sess := d.findByNameLocked(name); sess != nil { | |||
| 976 | + | sess.mu.Lock() | |||
| 977 | + | done = sess.restoreDone | |||
| 978 | + | sess.mu.Unlock() | |||
| 979 | + | } else { | |||
| 980 | + | stopped, ok = d.stopped[name] | |||
| 981 | + | if !ok || stopped.purging { | |||
| 982 | + | d.mu.Unlock() | |||
| 983 | + | return nil | |||
| 984 | + | } | |||
| 985 | + | done = stopped.restoreDone | |||
| 986 | + | } | |||
| 987 | + | d.mu.Unlock() | |||
| 988 | + | ||||
| 989 | + | if done != nil { | |||
| 990 | + | select { | |||
| 991 | + | case <-done: | |||
| 992 | + | case <-ctx.Done(): | |||
| 993 | + | return ctx.Err() | |||
| 994 | + | } | |||
| 995 | + | } | |||
| 996 | + | ||||
| 997 | + | d.mu.Lock() | |||
| 998 | + | defer d.mu.Unlock() | |||
| 999 | + | if d.findByNameLocked(name) != nil { | |||
| 1000 | + | return nil | |||
| 1001 | + | } | |||
| 1002 | + | stopped, ok = d.stopped[name] | |||
| 1003 | + | if !ok || stopped.record.Name == "" { | |||
| 1004 | + | return nil | |||
| 1005 | + | } | |||
| 1006 | + | if stopped.state == ports.SessionBroken { | |||
| 1007 | + | return &protoErr{ports.ErrInternal, "session durable state is broken: " + name} | |||
| 1008 | + | } | |||
| 1009 | + | if stopped.record.Committed == nil { | |||
| 1010 | + | return nil | |||
| 1011 | + | } | |||
| 1012 | + | return &protoErr{ports.ErrInternal, "session was not restored into this daemon: " + name} | |||
| 1013 | + | } | |||
| 1014 | + | ||||
| 833 | // route resolves a Hello to a session and a freshly attached client, creating | 1015 | // route resolves a Hello to a session and a freshly attached client, creating | ||
| 834 | // the session for ephemeral/new intents. | 1016 | // the session for ephemeral/new intents. | ||
| 835 | func (d *Daemon) route(h ports.Hello, tr ports.Transport) (*session, *attachedClient, error) { | 1017 | func (d *Daemon) route(h ports.Hello, tr ports.Transport) (*session, *attachedClient, error) { | ||
| @@ -837,13 +1019,6 @@ func (d *Daemon) route(h ports.Hello, tr ports.Transport) (*session, *attachedCl | |||||
| 837 | if !sz.Valid() { | 1019 | if !sz.Valid() { | ||
| 838 | sz = defaultSize | 1020 | sz = defaultSize | ||
| 839 | } | 1021 | } | ||
| 840 | - | if d.snapsEnabled && (h.Intent == ports.IntentResume || h.Intent == ports.IntentAttach || h.Intent == ports.IntentNew) { | |||
| 841 | - | select { | |||
| 842 | - | case <-d.restoreDone: | |||
| 843 | - | case <-d.serveCtx.Done(): | |||
| 844 | - | return nil, nil, &protoErr{ports.ErrServerShutdown, "daemon is shutting down"} | |||
| 845 | - | } | |||
| 846 | - | } | |||
| 847 | term := terminalEnv{TrueColor: h.TrueColor} | 1022 | term := terminalEnv{TrueColor: h.TrueColor} | ||
| 848 | 1023 | ||||
| 849 | if h.Intent == ports.IntentResume { | 1024 | if h.Intent == ports.IntentResume { | ||
| @@ -851,6 +1026,15 @@ func (d *Daemon) route(h ports.Hello, tr ports.Transport) (*session, *attachedCl | |||||
| 851 | return sess, ac, err | 1026 | return sess, ac, err | ||
| 852 | } | 1027 | } | ||
| 853 | } | 1028 | } | ||
| 1029 | + | if h.Intent == ports.IntentResume || h.Intent == ports.IntentAttach { | |||
| 1030 | + | ctx := d.serveCtx | |||
| 1031 | + | if ctx == nil { | |||
| 1032 | + | ctx = context.Background() | |||
| 1033 | + | } | |||
| 1034 | + | if err := d.waitForTargetRestore(ctx, h.Name); err != nil { | |||
| 1035 | + | return nil, nil, err | |||
| 1036 | + | } | |||
| 1037 | + | } | |||
| 854 | 1038 | ||||
| 855 | d.mu.Lock() | 1039 | d.mu.Lock() | ||
| 856 | // Shutdown/create interlock: once shutdown has begun (last session removed, | 1040 | // Shutdown/create interlock: once shutdown has begun (last session removed, | ||
M internal/usecase/daemon/daemon_test.go +6 -0
| @@ -68,6 +68,12 @@ func newTestDaemon(t testing.TB, ptys ports.PTYFactory, clk ports.Clock) *Daemon | |||||
| 68 | return newTestDaemonWithCleanup(t, ptys, clk, true) | 68 | return newTestDaemonWithCleanup(t, ptys, clk, true) | ||
| 69 | } | 69 | } | ||
| 70 | 70 | ||||
| 71 | + | func createSessionForTest(d *Daemon, name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) { | |||
| 72 | + | d.mu.Lock() | |||
| 73 | + | defer d.mu.Unlock() | |||
| 74 | + | return d.createSessionLocked(name, ephemeral, cwd, sz, term, env, restoredTabNames...) | |||
| 75 | + | } | |||
| 76 | + | ||||
| 71 | func newTestDaemonWithCleanup(t testing.TB, ptys ports.PTYFactory, clk ports.Clock, registerCleanup bool) *Daemon { | 77 | func newTestDaemonWithCleanup(t testing.TB, ptys ports.PTYFactory, clk ports.Clock, registerCleanup bool) *Daemon { | ||
| 72 | t.Helper() | 78 | t.Helper() | ||
| 73 | d := New(ptys, clk, slog.New(slog.NewTextHandler(io.Discard, nil))) | 79 | d := New(ptys, clk, slog.New(slog.NewTextHandler(io.Discard, nil))) | ||
A internal/usecase/daemon/durable_authority_test.go +53 -0
| @@ -0,0 +1,53 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "crypto/rand" | |||
| 6 | + | "testing" | |||
| 7 | + | ||||
| 8 | + | "github.com/stretchr/testify/require" | |||
| 9 | + | ||||
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 11 | + | "github.com/bnema/vev/internal/persist" | |||
| 12 | + | "github.com/bnema/vev/internal/ports" | |||
| 13 | + | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | |||
| 14 | + | ) | |||
| 15 | + | ||||
| 16 | + | type testLifecycleRepository struct{ ports.SnapshotRepository } | |||
| 17 | + | ||||
| 18 | + | func (testLifecycleRepository) DeleteIncarnation(context.Context, domain.IncarnationID) error { | |||
| 19 | + | return nil | |||
| 20 | + | } | |||
| 21 | + | func (testLifecycleRepository) CollectGarbage(context.Context, map[domain.IncarnationID]domain.CheckpointRef) error { | |||
| 22 | + | return nil | |||
| 23 | + | } | |||
| 24 | + | ||||
| 25 | + | // WithStore is package-test composition glue. Production injects the same | |||
| 26 | + | // recovery coordinator explicitly from internal/app. | |||
| 27 | + | func WithStore(t testing.TB, store ports.Store) Option { | |||
| 28 | + | t.Helper() | |||
| 29 | + | return func(d *Daemon) { | |||
| 30 | + | if store == nil { | |||
| 31 | + | return | |||
| 32 | + | } | |||
| 33 | + | catalogue := persist.New(store) | |||
| 34 | + | records, err := catalogue.Records() | |||
| 35 | + | require.NoError(t, err) | |||
| 36 | + | d.catalogue = catalogue | |||
| 37 | + | d.persistEnabled = true | |||
| 38 | + | d.catalogueRecords = records | |||
| 39 | + | d.catalogueRecordsProvided = true | |||
| 40 | + | repository := d.snapshotRepository | |||
| 41 | + | if repository == nil { | |||
| 42 | + | repository = testLifecycleRepository{} | |||
| 43 | + | } | |||
| 44 | + | d.recovery = recoveryusecase.NewCoordinator(catalogue, repository, rand.Reader) | |||
| 45 | + | } | |||
| 46 | + | } | |||
| 47 | + | ||||
| 48 | + | func testPersister(t testing.TB, d *Daemon) *persist.Persister { | |||
| 49 | + | t.Helper() | |||
| 50 | + | persister, ok := d.catalogue.(*persist.Persister) | |||
| 51 | + | require.True(t, ok, "catalogue type %T is not *persist.Persister", d.catalogue) | |||
| 52 | + | return persister | |||
| 53 | + | } | |||
A internal/usecase/daemon/durable_recovery_test.go +722 -0
| @@ -0,0 +1,722 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "bytes" | |||
| 5 | + | "context" | |||
| 6 | + | "errors" | |||
| 7 | + | "fmt" | |||
| 8 | + | "io" | |||
| 9 | + | "log/slog" | |||
| 10 | + | "os" | |||
| 11 | + | "sync" | |||
| 12 | + | "sync/atomic" | |||
| 13 | + | "syscall" | |||
| 14 | + | "testing" | |||
| 15 | + | "time" | |||
| 16 | + | ||||
| 17 | + | "github.com/stretchr/testify/mock" | |||
| 18 | + | "github.com/stretchr/testify/require" | |||
| 19 | + | ||||
| 20 | + | "github.com/bnema/vev/internal/domain" | |||
| 21 | + | "github.com/bnema/vev/internal/ports" | |||
| 22 | + | portsmocks "github.com/bnema/vev/internal/ports/mocks" | |||
| 23 | + | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | |||
| 24 | + | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |||
| 25 | + | ) | |||
| 26 | + | ||||
| 27 | + | type durableRecoveryCatalogue struct { | |||
| 28 | + | ports.Catalogue | |||
| 29 | + | mu sync.Mutex | |||
| 30 | + | records map[string]domain.CatalogueRecord | |||
| 31 | + | metadataUpdates []domain.CatalogueMetadataUpdate | |||
| 32 | + | } | |||
| 33 | + | ||||
| 34 | + | type recordErrorCatalogue struct { | |||
| 35 | + | *durableRecoveryCatalogue | |||
| 36 | + | err error | |||
| 37 | + | } | |||
| 38 | + | ||||
| 39 | + | func (c recordErrorCatalogue) Record(string) (domain.CatalogueRecord, bool, error) { | |||
| 40 | + | return domain.CatalogueRecord{}, false, c.err | |||
| 41 | + | } | |||
| 42 | + | ||||
| 43 | + | func newDurableRecoveryCatalogue(records []domain.CatalogueRecord) *durableRecoveryCatalogue { | |||
| 44 | + | catalogue := &durableRecoveryCatalogue{records: make(map[string]domain.CatalogueRecord, len(records))} | |||
| 45 | + | for _, record := range records { | |||
| 46 | + | catalogue.records[record.Name] = record | |||
| 47 | + | } | |||
| 48 | + | return catalogue | |||
| 49 | + | } | |||
| 50 | + | ||||
| 51 | + | func (c *durableRecoveryCatalogue) Records() ([]domain.CatalogueRecord, error) { | |||
| 52 | + | c.mu.Lock() | |||
| 53 | + | defer c.mu.Unlock() | |||
| 54 | + | records := make([]domain.CatalogueRecord, 0, len(c.records)) | |||
| 55 | + | for _, record := range c.records { | |||
| 56 | + | records = append(records, record) | |||
| 57 | + | } | |||
| 58 | + | return records, nil | |||
| 59 | + | } | |||
| 60 | + | ||||
| 61 | + | func (c *durableRecoveryCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) { | |||
| 62 | + | c.mu.Lock() | |||
| 63 | + | defer c.mu.Unlock() | |||
| 64 | + | record, ok := c.records[name] | |||
| 65 | + | return record, ok, nil | |||
| 66 | + | } | |||
| 67 | + | ||||
| 68 | + | func mustDurableRecords(t *testing.T, c *durableRecoveryCatalogue) []domain.CatalogueRecord { | |||
| 69 | + | t.Helper() | |||
| 70 | + | records, err := c.Records() | |||
| 71 | + | require.NoError(t, err) | |||
| 72 | + | return records | |||
| 73 | + | } | |||
| 74 | + | ||||
| 75 | + | func (c *durableRecoveryCatalogue) Replace(name string, record domain.CatalogueRecord) error { | |||
| 76 | + | c.mu.Lock() | |||
| 77 | + | defer c.mu.Unlock() | |||
| 78 | + | if name != record.Name { | |||
| 79 | + | return errors.New("catalogue key mismatch") | |||
| 80 | + | } | |||
| 81 | + | c.records[name] = record | |||
| 82 | + | return nil | |||
| 83 | + | } | |||
| 84 | + | ||||
| 85 | + | func (c *durableRecoveryCatalogue) UpdateMetadata(update domain.CatalogueMetadataUpdate) error { | |||
| 86 | + | c.mu.Lock() | |||
| 87 | + | defer c.mu.Unlock() | |||
| 88 | + | record, ok := c.records[update.Name] | |||
| 89 | + | if !ok { | |||
| 90 | + | return errors.New("catalogue record not found") | |||
| 91 | + | } | |||
| 92 | + | if update.IncarnationID == (domain.IncarnationID{}) || record.IncarnationID != update.IncarnationID { | |||
| 93 | + | return errors.New("catalogue incarnation changed") | |||
| 94 | + | } | |||
| 95 | + | if update.Cwd != nil { | |||
| 96 | + | record.Cwd = *update.Cwd | |||
| 97 | + | } | |||
| 98 | + | if update.UpdatedAt != nil { | |||
| 99 | + | record.UpdatedAt = *update.UpdatedAt | |||
| 100 | + | } | |||
| 101 | + | if update.LastUsedSeq != nil { | |||
| 102 | + | record.LastUsedSeq = *update.LastUsedSeq | |||
| 103 | + | } | |||
| 104 | + | if update.TabNames != nil { | |||
| 105 | + | record.TabNames = append([]string(nil), (*update.TabNames)...) | |||
| 106 | + | } | |||
| 107 | + | c.records[update.Name] = record | |||
| 108 | + | c.metadataUpdates = append(c.metadataUpdates, update) | |||
| 109 | + | return nil | |||
| 110 | + | } | |||
| 111 | + | ||||
| 112 | + | func (c *durableRecoveryCatalogue) MetadataUpdates() []domain.CatalogueMetadataUpdate { | |||
| 113 | + | c.mu.Lock() | |||
| 114 | + | defer c.mu.Unlock() | |||
| 115 | + | return append([]domain.CatalogueMetadataUpdate(nil), c.metadataUpdates...) | |||
| 116 | + | } | |||
| 117 | + | ||||
| 118 | + | func (c *durableRecoveryCatalogue) Sync() error { return nil } | |||
| 119 | + | func (c *durableRecoveryCatalogue) Close() error { return nil } | |||
| 120 | + | ||||
| 121 | + | type durableRecoveryRepository struct { | |||
| 122 | + | ports.SnapshotRepository | |||
| 123 | + | mu sync.Mutex | |||
| 124 | + | generations map[string]ports.SnapshotGeneration | |||
| 125 | + | errors map[string]error | |||
| 126 | + | loads map[string]int | |||
| 127 | + | repairs map[string]domain.CheckpointRef | |||
| 128 | + | repairErr error | |||
| 129 | + | repairCalls int | |||
| 130 | + | } | |||
| 131 | + | ||||
| 132 | + | func (r *durableRecoveryRepository) LoadCheckpoint(_ context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, error) { | |||
| 133 | + | r.mu.Lock() | |||
| 134 | + | defer r.mu.Unlock() | |||
| 135 | + | if r.loads == nil { | |||
| 136 | + | r.loads = make(map[string]int) | |||
| 137 | + | } | |||
| 138 | + | r.loads[name]++ | |||
| 139 | + | if err := r.errors[name]; err != nil { | |||
| 140 | + | return ports.SnapshotGeneration{}, err | |||
| 141 | + | } | |||
| 142 | + | generation, ok := r.generations[name] | |||
| 143 | + | if !ok || generation.Generation != ref.Generation { | |||
| 144 | + | return ports.SnapshotGeneration{}, errors.New("checkpoint unavailable") | |||
| 145 | + | } | |||
| 146 | + | return generation, nil | |||
| 147 | + | } | |||
| 148 | + | ||||
| 149 | + | func (r *durableRecoveryRepository) ReconcileCheckpoint(_ context.Context, id domain.IncarnationID, ref ports.CheckpointRef) error { | |||
| 150 | + | r.mu.Lock() | |||
| 151 | + | defer r.mu.Unlock() | |||
| 152 | + | r.repairCalls++ | |||
| 153 | + | if r.repairErr != nil { | |||
| 154 | + | return r.repairErr | |||
| 155 | + | } | |||
| 156 | + | if r.repairs == nil { | |||
| 157 | + | r.repairs = make(map[string]domain.CheckpointRef) | |||
| 158 | + | } | |||
| 159 | + | r.repairs[id.String()] = ref | |||
| 160 | + | return nil | |||
| 161 | + | } | |||
| 162 | + | ||||
| 163 | + | func durableRecoveryRecord(index int) domain.CatalogueRecord { | |||
| 164 | + | id := domain.IncarnationID{0: byte(index>>8 + 1), 1: byte(index + 1)} | |||
| 165 | + | ref := domain.CheckpointRef{Generation: uint64(index + 1), ManifestDigest: [32]byte{byte(index%255 + 1)}} | |||
| 166 | + | return domain.CatalogueRecord{ | |||
| 167 | + | Name: fmt.Sprintf("session-%04d", index), | |||
| 168 | + | IncarnationID: id, | |||
| 169 | + | Cwd: "/tmp", | |||
| 170 | + | CreatedAt: int64(index + 1), | |||
| 171 | + | UpdatedAt: int64(index + 1), | |||
| 172 | + | Committed: &ref, | |||
| 173 | + | } | |||
| 174 | + | } | |||
| 175 | + | ||||
| 176 | + | func validGeneration(t testing.TB, record domain.CatalogueRecord) ports.SnapshotGeneration { | |||
| 177 | + | t.Helper() | |||
| 178 | + | manifest, err := snapcodec.MarshalManifest(snapcodec.Manifest{ | |||
| 179 | + | Generation: record.Committed.Generation, | |||
| 180 | + | IncarnationID: record.IncarnationID, | |||
| 181 | + | Name: record.Name, | |||
| 182 | + | CreatedAt: uint64(record.CreatedAt), | |||
| 183 | + | }) | |||
| 184 | + | require.NoError(t, err) | |||
| 185 | + | return ports.SnapshotGeneration{ | |||
| 186 | + | IncarnationID: record.IncarnationID, | |||
| 187 | + | Name: record.Name, | |||
| 188 | + | Generation: record.Committed.Generation, | |||
| 189 | + | Manifest: manifest, | |||
| 190 | + | Objects: map[ports.SnapshotDigest][]byte{}, | |||
| 191 | + | } | |||
| 192 | + | } | |||
| 193 | + | ||||
| 194 | + | func newDurableRecoveryDaemon(t testing.TB, records []domain.CatalogueRecord, repository ports.SnapshotRepository) (*Daemon, *durableRecoveryCatalogue) { | |||
| 195 | + | t.Helper() | |||
| 196 | + | catalogue := newDurableRecoveryCatalogue(records) | |||
| 197 | + | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil) | |||
| 198 | + | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, records), WithSnapshotRepository(repository), WithRecoveryCoordinator(coordinator)) | |||
| 199 | + | d.serveCtx, d.serveCancel = context.WithCancel(context.Background()) | |||
| 200 | + | t.Cleanup(d.serveCancel) | |||
| 201 | + | return d, catalogue | |||
| 202 | + | } | |||
| 203 | + | ||||
| 204 | + | func TestFinalCheckpointFailurePreservesCatalogue(t *testing.T) { | |||
| 205 | + | priorRef := domain.CheckpointRef{Generation: 7, ManifestDigest: [32]byte{7}} | |||
| 206 | + | prior := domain.CatalogueRecord{ | |||
| 207 | + | Name: "work", | |||
| 208 | + | IncarnationID: domain.IncarnationID{1}, | |||
| 209 | + | Cwd: "/work", | |||
| 210 | + | CreatedAt: 42, | |||
| 211 | + | Committed: &priorRef, | |||
| 212 | + | } | |||
| 213 | + | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{prior}) | |||
| 214 | + | repository := &durableRecoveryRepository{} | |||
| 215 | + | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil) | |||
| 216 | + | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(catalogue, []domain.CatalogueRecord{prior}), WithSnapshotRepository(repository), WithRecoveryCoordinator(coordinator)) | |||
| 217 | + | startSnapshotEncodeWorker(t, d) | |||
| 218 | + | ||||
| 219 | + | sess := newSnapshotTestSession(t, prior.Name, false, prior.Cwd) | |||
| 220 | + | expired, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) | |||
| 221 | + | defer cancel() | |||
| 222 | + | sess.snapshotMu.Lock() | |||
| 223 | + | sess.snapshotPublishedGeneration = priorRef.Generation | |||
| 224 | + | sess.snapshotPublishedCheckpoint = &priorRef | |||
| 225 | + | sess.snapshotGeneration = priorRef.Generation + 1 | |||
| 226 | + | sess.snapshotPublicationContext = expired | |||
| 227 | + | sess.snapDirty.Store(true) | |||
| 228 | + | sess.snapshotMu.Unlock() | |||
| 229 | + | ||||
| 230 | + | require.True(t, d.scheduleFinalSnapshot(sess)) | |||
| 231 | + | awaitSnapshotIdle(t, sess) | |||
| 232 | + | got, ok, _ := catalogue.Record(prior.Name) | |||
| 233 | + | require.True(t, ok) | |||
| 234 | + | require.Equal(t, prior, got, "a timed-out final checkpoint must not change authoritative bytes") | |||
| 235 | + | sess.snapshotMu.Lock() | |||
| 236 | + | published := sess.snapshotPublishedCheckpoint | |||
| 237 | + | sess.snapshotMu.Unlock() | |||
| 238 | + | require.Equal(t, priorRef, *published) | |||
| 239 | + | } | |||
| 240 | + | ||||
| 241 | + | func TestCatalogueRestoreIndependent(t *testing.T) { | |||
| 242 | + | t.Run("empty catalogue restoration", func(t *testing.T) { | |||
| 243 | + | emptyDaemon, emptyCatalogue := newDurableRecoveryDaemon(t, nil, &durableRecoveryRepository{}) | |||
| 244 | + | emptyDone := make(chan struct{}) | |||
| 245 | + | go func() { | |||
| 246 | + | emptyDaemon.restoreCatalogue(context.Background(), mustDurableRecords(t, emptyCatalogue)) | |||
| 247 | + | close(emptyDone) | |||
| 248 | + | }() | |||
| 249 | + | select { | |||
| 250 | + | case <-emptyDone: | |||
| 251 | + | case <-time.After(time.Second): | |||
| 252 | + | t.Fatal("empty catalogue restoration blocked") | |||
| 253 | + | } | |||
| 254 | + | }) | |||
| 255 | + | ||||
| 256 | + | t.Run("healthy and broken restoration", func(t *testing.T) { | |||
| 257 | + | healthy := durableRecoveryRecord(0) | |||
| 258 | + | broken := durableRecoveryRecord(1) | |||
| 259 | + | repository := &durableRecoveryRepository{ | |||
| 260 | + | generations: map[string]ports.SnapshotGeneration{healthy.Name: validGeneration(t, healthy)}, | |||
| 261 | + | errors: map[string]error{broken.Name: errors.New("corrupt checkpoint")}, | |||
| 262 | + | loads: make(map[string]int), | |||
| 263 | + | repairs: make(map[string]domain.CheckpointRef), | |||
| 264 | + | } | |||
| 265 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{healthy, broken}, repository) | |||
| 266 | + | ||||
| 267 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |||
| 268 | + | ||||
| 269 | + | d.mu.Lock() | |||
| 270 | + | healthyEntry := d.stopped[healthy.Name] | |||
| 271 | + | brokenEntry := d.stopped[broken.Name] | |||
| 272 | + | d.mu.Unlock() | |||
| 273 | + | require.Equal(t, ports.SessionStopped, healthyEntry.state) | |||
| 274 | + | require.Equal(t, ports.SessionBroken, brokenEntry.state) | |||
| 275 | + | for name, done := range map[string]<-chan struct{}{healthy.Name: healthyEntry.restoreDone, broken.Name: brokenEntry.restoreDone} { | |||
| 276 | + | select { | |||
| 277 | + | case <-done: | |||
| 278 | + | default: | |||
| 279 | + | t.Fatalf("restore completion for %q was not closed", name) | |||
| 280 | + | } | |||
| 281 | + | } | |||
| 282 | + | brokenRecord, ok, _ := catalogue.Record(broken.Name) | |||
| 283 | + | require.True(t, ok) | |||
| 284 | + | require.Equal(t, "checkpoint validation failed", brokenRecord.DegradedReason) | |||
| 285 | + | require.Equal(t, 1, repository.loads[healthy.Name]) | |||
| 286 | + | require.Equal(t, 1, repository.loads[broken.Name]) | |||
| 287 | + | require.Empty(t, d.sessions, "a corrupt record must never open a replacement runtime") | |||
| 288 | + | }) | |||
| 289 | + | ||||
| 290 | + | } | |||
| 291 | + | ||||
| 292 | + | func TestCatalogueRestoreAggregateOver4096(t *testing.T) { | |||
| 293 | + | const count = 4200 | |||
| 294 | + | records := make([]domain.CatalogueRecord, count) | |||
| 295 | + | repository := &durableRecoveryRepository{generations: make(map[string]ports.SnapshotGeneration, count), errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | |||
| 296 | + | for i := range records { | |||
| 297 | + | records[i] = durableRecoveryRecord(i) | |||
| 298 | + | repository.generations[records[i].Name] = validGeneration(t, records[i]) | |||
| 299 | + | } | |||
| 300 | + | d, catalogue := newDurableRecoveryDaemon(t, records, repository) | |||
| 301 | + | ||||
| 302 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |||
| 303 | + | ||||
| 304 | + | require.Len(t, d.stopped, count) | |||
| 305 | + | for _, record := range records { | |||
| 306 | + | require.Equal(t, ports.SessionStopped, d.stopped[record.Name].state) | |||
| 307 | + | require.Equal(t, 1, repository.loads[record.Name]) | |||
| 308 | + | } | |||
| 309 | + | } | |||
| 310 | + | ||||
| 311 | + | func TestCatalogueRestoreSingleSessionOver4096(t *testing.T) { | |||
| 312 | + | record := durableRecoveryRecord(0) | |||
| 313 | + | repository := &durableRecoveryRepository{generations: map[string]ports.SnapshotGeneration{record.Name: validGeneration(t, record)}, errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | |||
| 314 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |||
| 315 | + | ||||
| 316 | + | // The repository may contain any number of stale entries; restoration has no | |||
| 317 | + | // discovery API and therefore performs exactly one direct catalogue lookup. | |||
| 318 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |||
| 319 | + | ||||
| 320 | + | require.Equal(t, ports.SessionStopped, d.stopped[record.Name].state) | |||
| 321 | + | require.Equal(t, 1, repository.loads[record.Name]) | |||
| 322 | + | } | |||
| 323 | + | ||||
| 324 | + | func TestCatalogueRestoreIncarnationMismatch(t *testing.T) { | |||
| 325 | + | record := durableRecoveryRecord(0) | |||
| 326 | + | generation := validGeneration(t, record) | |||
| 327 | + | generation.IncarnationID = domain.IncarnationID{99} | |||
| 328 | + | repository := &durableRecoveryRepository{generations: map[string]ports.SnapshotGeneration{record.Name: generation}, errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | |||
| 329 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |||
| 330 | + | ||||
| 331 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |||
| 332 | + | ||||
| 333 | + | require.Equal(t, ports.SessionBroken, d.stopped[record.Name].state) | |||
| 334 | + | got, _, _ := catalogue.Record(record.Name) | |||
| 335 | + | require.Equal(t, "checkpoint validation failed", got.DegradedReason) | |||
| 336 | + | } | |||
| 337 | + | ||||
| 338 | + | type cancellationRecoveryRepository struct { | |||
| 339 | + | ports.SnapshotRepository | |||
| 340 | + | started chan struct{} | |||
| 341 | + | once sync.Once | |||
| 342 | + | } | |||
| 343 | + | ||||
| 344 | + | func (r *cancellationRecoveryRepository) LoadCheckpoint(ctx context.Context, _ domain.IncarnationID, _ string, _ ports.CheckpointRef) (ports.SnapshotGeneration, error) { | |||
| 345 | + | r.once.Do(func() { close(r.started) }) | |||
| 346 | + | <-ctx.Done() | |||
| 347 | + | return ports.SnapshotGeneration{}, ctx.Err() | |||
| 348 | + | } | |||
| 349 | + | ||||
| 350 | + | type recoveryCountingPTYFactory struct { | |||
| 351 | + | calls atomic.Int32 | |||
| 352 | + | } | |||
| 353 | + | ||||
| 354 | + | func (f *recoveryCountingPTYFactory) Open(context.Context, string, []string, []string, string, domain.Size) (ports.PTY, error) { | |||
| 355 | + | f.calls.Add(1) | |||
| 356 | + | return nil, errors.New("unexpected PTY open") | |||
| 357 | + | } | |||
| 358 | + | ||||
| 359 | + | func TestSessionRecoveryUsesIncomingCommandContext(t *testing.T) { | |||
| 360 | + | type contextKey struct{} | |||
| 361 | + | key := contextKey{} | |||
| 362 | + | ctx := context.WithValue(t.Context(), key, "request") | |||
| 363 | + | wantErr := errors.New("stop after context capture") | |||
| 364 | + | record := durableRecoveryRecord(0) | |||
| 365 | + | record.DegradedReason = "checkpoint validation failed" | |||
| 366 | + | catalogue := portsmocks.NewMockCatalogue(t) | |||
| 367 | + | repository := portsmocks.NewMockSnapshotRepository(t) | |||
| 368 | + | catalogue.EXPECT().Record(record.Name).Return(record, true, nil).Once() | |||
| 369 | + | catalogue.EXPECT().Replace(record.Name, mock.Anything).Return(nil).Once() | |||
| 370 | + | seen := false | |||
| 371 | + | repository.EXPECT().DeleteIncarnation(mock.Anything, record.IncarnationID). | |||
| 372 | + | Run(func(got context.Context, _ domain.IncarnationID) { seen = got.Value(key) == "request" }). | |||
| 373 | + | Return(wantErr).Once() | |||
| 374 | + | ||||
| 375 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 376 | + | d.catalogue = catalogue | |||
| 377 | + | d.recovery = recoveryusecase.NewCoordinator(catalogue, repository, bytes.NewReader(bytes.Repeat([]byte{9}, 16))) | |||
| 378 | + | ||||
| 379 | + | _, err := (controlExec{ctx: ctx, d: d, recoveryName: record.Name}).SessionRecovery("discard") | |||
| 380 | + | require.ErrorIs(t, err, wantErr) | |||
| 381 | + | require.True(t, seen) | |||
| 382 | + | } | |||
| 383 | + | ||||
| 384 | + | type temporaryRestoreError struct{} | |||
| 385 | + | ||||
| 386 | + | func (temporaryRestoreError) Error() string { return "temporary restore failure" } | |||
| 387 | + | func (temporaryRestoreError) Temporary() bool { return true } | |||
| 388 | + | ||||
| 389 | + | func TestRetryableRestoreLoadErrorClassification(t *testing.T) { | |||
| 390 | + | tests := []struct { | |||
| 391 | + | name string | |||
| 392 | + | err error | |||
| 393 | + | retryable bool | |||
| 394 | + | }{ | |||
| 395 | + | {name: "missing path", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.ENOENT})}, | |||
| 396 | + | {name: "permission denied", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EACCES})}, | |||
| 397 | + | {name: "invalid path operation", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EINVAL})}, | |||
| 398 | + | {name: "temporary path failure", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.EAGAIN}), retryable: true}, | |||
| 399 | + | {name: "resource exhausted path", err: fmt.Errorf("wrapped: %w", &os.PathError{Op: "open", Path: "/snapshot", Err: syscall.ENOSPC}), retryable: true}, | |||
| 400 | + | {name: "temporary interface", err: fmt.Errorf("wrapped: %w", temporaryRestoreError{}), retryable: true}, | |||
| 401 | + | {name: "corrupt data", err: errors.New("corrupt checkpoint")}, | |||
| 402 | + | {name: "context canceled", err: context.Canceled}, | |||
| 403 | + | {name: "context deadline exceeded", err: context.DeadlineExceeded}, | |||
| 404 | + | } | |||
| 405 | + | for _, tt := range tests { | |||
| 406 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 407 | + | require.Equal(t, tt.retryable, isRetryableRestoreLoadError(tt.err)) | |||
| 408 | + | }) | |||
| 409 | + | } | |||
| 410 | + | } | |||
| 411 | + | ||||
| 412 | + | func TestFinishRecordRestoreDoesNotDegradeStaleCatalogueAuthority(t *testing.T) { | |||
| 413 | + | for _, tt := range []struct { | |||
| 414 | + | name string | |||
| 415 | + | setup func(*Daemon, *durableRecoveryCatalogue, domain.CatalogueRecord) | |||
| 416 | + | }{ | |||
| 417 | + | { | |||
| 418 | + | name: "missing record", | |||
| 419 | + | setup: func(_ *Daemon, c *durableRecoveryCatalogue, record domain.CatalogueRecord) { | |||
| 420 | + | c.mu.Lock() | |||
| 421 | + | delete(c.records, record.Name) | |||
| 422 | + | c.mu.Unlock() | |||
| 423 | + | }, | |||
| 424 | + | }, | |||
| 425 | + | { | |||
| 426 | + | name: "replacement incarnation", | |||
| 427 | + | setup: func(_ *Daemon, c *durableRecoveryCatalogue, record domain.CatalogueRecord) { | |||
| 428 | + | record.IncarnationID = domain.IncarnationID{99} | |||
| 429 | + | c.mu.Lock() | |||
| 430 | + | c.records[record.Name] = record | |||
| 431 | + | c.mu.Unlock() | |||
| 432 | + | }, | |||
| 433 | + | }, | |||
| 434 | + | { | |||
| 435 | + | name: "catalogue read error", | |||
| 436 | + | setup: func(d *Daemon, c *durableRecoveryCatalogue, _ domain.CatalogueRecord) { | |||
| 437 | + | d.catalogue = recordErrorCatalogue{durableRecoveryCatalogue: c, err: errors.New("read failed")} | |||
| 438 | + | d.recovery = recoveryusecase.NewCoordinator(d.catalogue, nil, nil) | |||
| 439 | + | }, | |||
| 440 | + | }, | |||
| 441 | + | } { | |||
| 442 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 443 | + | record := durableRecoveryRecord(0) | |||
| 444 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | |||
| 445 | + | tt.setup(d, catalogue, record) | |||
| 446 | + | d.mu.Lock() | |||
| 447 | + | before := d.stopped[record.Name] | |||
| 448 | + | d.mu.Unlock() | |||
| 449 | + | ||||
| 450 | + | d.finishRecordRestore(record, errors.New("invalid checkpoint"), before.restoreDone) | |||
| 451 | + | ||||
| 452 | + | d.mu.Lock() | |||
| 453 | + | after := d.stopped[record.Name] | |||
| 454 | + | d.mu.Unlock() | |||
| 455 | + | require.Equal(t, before.record, after.record) | |||
| 456 | + | require.Equal(t, before.state, after.state) | |||
| 457 | + | }) | |||
| 458 | + | } | |||
| 459 | + | } | |||
| 460 | + | ||||
| 461 | + | func TestPersistAndRegisterRestoredSessionRejectsReplacementIncarnation(t *testing.T) { | |||
| 462 | + | record := durableRecoveryRecord(0) | |||
| 463 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | |||
| 464 | + | replacement := record | |||
| 465 | + | replacement.IncarnationID = domain.IncarnationID{99} | |||
| 466 | + | require.NoError(t, catalogue.Replace(record.Name, replacement)) | |||
| 467 | + | sess := newSnapshotTestSession(t, record.Name, false, record.Cwd) | |||
| 468 | + | sess.incarnation = record.IncarnationID | |||
| 469 | + | ||||
| 470 | + | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess) | |||
| 471 | + | require.ErrorContains(t, err, "incarnation") | |||
| 472 | + | require.False(t, registered) | |||
| 473 | + | require.Empty(t, d.sessions) | |||
| 474 | + | } | |||
| 475 | + | ||||
| 476 | + | func TestSessionRecoveryCommand(t *testing.T) { | |||
| 477 | + | degraded := durableRecoveryRecord(2) | |||
| 478 | + | degraded.DegradedReason = "checkpoint validation failed" | |||
| 479 | + | fresh := degraded | |||
| 480 | + | fresh.IncarnationID = domain.IncarnationID{9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9} | |||
| 481 | + | fresh.Committed = nil | |||
| 482 | + | fresh.DegradedReason = "" | |||
| 483 | + | catalogue := portsmocks.NewMockCatalogue(t) | |||
| 484 | + | repository := portsmocks.NewMockSnapshotRepository(t) | |||
| 485 | + | catalogue.EXPECT().Record(degraded.Name).Return(degraded, true, nil).Once() | |||
| 486 | + | catalogue.EXPECT().Replace(degraded.Name, fresh).Return(nil).Once() | |||
| 487 | + | repository.EXPECT().DeleteIncarnation(mock.Anything, degraded.IncarnationID).Return(nil).Once() | |||
| 488 | + | catalogue.EXPECT().Record(degraded.Name).Return(fresh, true, nil).Once() | |||
| 489 | + | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, bytes.NewReader(bytes.Repeat([]byte{9}, 16))) | |||
| 490 | + | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), | |||
| 491 | + | WithCatalogue(catalogue, []domain.CatalogueRecord{degraded}), | |||
| 492 | + | WithSnapshotRepository(repository), WithRecoveryCoordinator(coordinator)) | |||
| 493 | + | ||||
| 494 | + | _, err := (controlExec{d: d, recoveryName: degraded.Name}).SessionRecovery("discard") | |||
| 495 | + | require.NoError(t, err) | |||
| 496 | + | require.Equal(t, ports.SessionStopped, d.stopped[degraded.Name].state) | |||
| 497 | + | require.Equal(t, fresh, d.stopped[degraded.Name].record) | |||
| 498 | + | } | |||
| 499 | + | ||||
| 500 | + | func TestListShowsDegraded(t *testing.T) { | |||
| 501 | + | fresh := durableRecoveryRecord(0) | |||
| 502 | + | fresh.Committed = nil | |||
| 503 | + | restoring := durableRecoveryRecord(1) | |||
| 504 | + | degraded := durableRecoveryRecord(2) | |||
| 505 | + | degraded.DegradedReason = "checkpoint validation failed" | |||
| 506 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{fresh, restoring, degraded}, &durableRecoveryRepository{}) | |||
| 507 | + | ||||
| 508 | + | listed := listSessions(t, d) | |||
| 509 | + | require.Equal(t, []ports.SessionInfo{ | |||
| 510 | + | {Name: fresh.Name, State: ports.SessionStopped}, | |||
| 511 | + | {Name: restoring.Name, State: ports.SessionStopped}, | |||
| 512 | + | {Name: degraded.Name, State: ports.SessionBroken}, | |||
| 513 | + | }, listed.Sessions) | |||
| 514 | + | } | |||
| 515 | + | ||||
| 516 | + | func TestListPurgingDominatesRestoringState(t *testing.T) { | |||
| 517 | + | record := durableRecoveryRecord(0) | |||
| 518 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | |||
| 519 | + | d.mu.Lock() | |||
| 520 | + | entry := d.stopped[record.Name] | |||
| 521 | + | entry.purging = true | |||
| 522 | + | entry.state = ports.SessionStopped | |||
| 523 | + | d.stopped[record.Name] = entry | |||
| 524 | + | d.mu.Unlock() | |||
| 525 | + | ||||
| 526 | + | listed := listSessions(t, d) | |||
| 527 | + | require.Equal(t, []ports.SessionInfo{{Name: record.Name, State: ports.SessionBroken}}, listed.Sessions) | |||
| 528 | + | } | |||
| 529 | + | ||||
| 530 | + | func TestAttachWaitsForRestore(t *testing.T) { | |||
| 531 | + | record := durableRecoveryRecord(0) | |||
| 532 | + | factory := &recoveryCountingPTYFactory{} | |||
| 533 | + | d := New(factory, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(newDurableRecoveryCatalogue([]domain.CatalogueRecord{record}), []domain.CatalogueRecord{record})) | |||
| 534 | + | d.serveCtx, d.serveCancel = context.WithCancel(context.Background()) | |||
| 535 | + | defer d.serveCancel() | |||
| 536 | + | ||||
| 537 | + | result := make(chan error, 1) | |||
| 538 | + | go func() { | |||
| 539 | + | _, _, err := d.route(ports.Hello{Version: ports.ProtocolVersion, Intent: ports.IntentAttach, Name: record.Name}, nil) | |||
| 540 | + | result <- err | |||
| 541 | + | }() | |||
| 542 | + | ||||
| 543 | + | select { | |||
| 544 | + | case err := <-result: | |||
| 545 | + | t.Fatalf("attach returned before target restoration completed: %v", err) | |||
| 546 | + | case <-time.After(25 * time.Millisecond): | |||
| 547 | + | } | |||
| 548 | + | require.Zero(t, factory.calls.Load()) | |||
| 549 | + | ||||
| 550 | + | d.setStoppedRecovery(record, ports.SessionBroken) | |||
| 551 | + | d.mu.Lock() | |||
| 552 | + | closeRuntimeRestoreDoneLocked(d.stopped[record.Name].restoreDone) | |||
| 553 | + | d.mu.Unlock() | |||
| 554 | + | var protocolError *protoErr | |||
| 555 | + | require.ErrorAs(t, <-result, &protocolError) | |||
| 556 | + | require.Equal(t, ports.ErrInternal, protocolError.code) | |||
| 557 | + | require.Zero(t, factory.calls.Load()) | |||
| 558 | + | } | |||
| 559 | + | ||||
| 560 | + | func TestAttachBarrierSurvivesLiveRegistryPublication(t *testing.T) { | |||
| 561 | + | record := durableRecoveryRecord(0) | |||
| 562 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | |||
| 563 | + | d.mu.Lock() | |||
| 564 | + | done := d.stopped[record.Name].restoreDone | |||
| 565 | + | d.mu.Unlock() | |||
| 566 | + | sess := &session{name: record.Name, incarnation: record.IncarnationID} | |||
| 567 | + | ||||
| 568 | + | registered, err := d.persistAndRegisterRestoredSession(t.Context(), sess) | |||
| 569 | + | require.NoError(t, err) | |||
| 570 | + | require.True(t, registered) | |||
| 571 | + | ||||
| 572 | + | cancelled, cancel := context.WithCancel(context.Background()) | |||
| 573 | + | cancel() | |||
| 574 | + | require.ErrorIs(t, d.waitForTargetRestore(cancelled, record.Name), context.Canceled, | |||
| 575 | + | "an attach racing after live publication must still observe the restore barrier") | |||
| 576 | + | ||||
| 577 | + | d.finishRecordRestore(record, nil, done) | |||
| 578 | + | require.NoError(t, d.waitForTargetRestore(context.Background(), record.Name), | |||
| 579 | + | "attach must proceed after successful restoration completes") | |||
| 580 | + | } | |||
| 581 | + | ||||
| 582 | + | func TestAttachRestoreWaitHonorsContextCancellation(t *testing.T) { | |||
| 583 | + | record := durableRecoveryRecord(0) | |||
| 584 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | |||
| 585 | + | ctx, cancel := context.WithCancel(context.Background()) | |||
| 586 | + | result := make(chan error, 1) | |||
| 587 | + | go func() { result <- d.waitForTargetRestore(ctx, record.Name) }() | |||
| 588 | + | ||||
| 589 | + | select { | |||
| 590 | + | case err := <-result: | |||
| 591 | + | t.Fatalf("restore wait returned before cancellation: %v", err) | |||
| 592 | + | case <-time.After(25 * time.Millisecond): | |||
| 593 | + | } | |||
| 594 | + | cancel() | |||
| 595 | + | require.ErrorIs(t, <-result, context.Canceled) | |||
| 596 | + | } | |||
| 597 | + | ||||
| 598 | + | func TestRestoreCancellationTransitionsBeforeAttachCompletion(t *testing.T) { | |||
| 599 | + | record := durableRecoveryRecord(0) | |||
| 600 | + | repository := &cancellationRecoveryRepository{started: make(chan struct{})} | |||
| 601 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |||
| 602 | + | restoreCtx, cancelRestore := context.WithCancel(context.Background()) | |||
| 603 | + | restored := make(chan struct{}) | |||
| 604 | + | go func() { | |||
| 605 | + | d.restoreCatalogue(restoreCtx, mustDurableRecords(t, catalogue)) | |||
| 606 | + | close(restored) | |||
| 607 | + | }() | |||
| 608 | + | <-repository.started | |||
| 609 | + | ||||
| 610 | + | attached := make(chan error, 1) | |||
| 611 | + | go func() { | |||
| 612 | + | _, _, err := d.route(ports.Hello{Version: ports.ProtocolVersion, Intent: ports.IntentAttach, Name: record.Name}, nil) | |||
| 613 | + | attached <- err | |||
| 614 | + | }() | |||
| 615 | + | cancelRestore() | |||
| 616 | + | ||||
| 617 | + | var protocolError *protoErr | |||
| 618 | + | require.ErrorAs(t, <-attached, &protocolError) | |||
| 619 | + | require.Equal(t, ports.ErrInternal, protocolError.code) | |||
| 620 | + | <-restored | |||
| 621 | + | entry := d.stopped[record.Name] | |||
| 622 | + | require.Equal(t, ports.SessionBroken, entry.state) | |||
| 623 | + | require.Equal(t, "restore interrupted", entry.record.DegradedReason) | |||
| 624 | + | select { | |||
| 625 | + | case <-entry.restoreDone: | |||
| 626 | + | default: | |||
| 627 | + | t.Fatal("restore completion was not signaled") | |||
| 628 | + | } | |||
| 629 | + | committed, _, _ := catalogue.Record(record.Name) | |||
| 630 | + | require.Equal(t, entry.record, committed) | |||
| 631 | + | } | |||
| 632 | + | ||||
| 633 | + | func TestWaitForTargetRestoreRejectsClosedRestoringChannel(t *testing.T) { | |||
| 634 | + | record := durableRecoveryRecord(0) | |||
| 635 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | |||
| 636 | + | d.mu.Lock() | |||
| 637 | + | closeRuntimeRestoreDoneLocked(d.stopped[record.Name].restoreDone) | |||
| 638 | + | d.mu.Unlock() | |||
| 639 | + | ||||
| 640 | + | result := make(chan error, 1) | |||
| 641 | + | go func() { result <- d.waitForTargetRestore(context.Background(), record.Name) }() | |||
| 642 | + | ||||
| 643 | + | select { | |||
| 644 | + | case err := <-result: | |||
| 645 | + | var protocolError *protoErr | |||
| 646 | + | require.ErrorAs(t, err, &protocolError) | |||
| 647 | + | require.Equal(t, ports.ErrInternal, protocolError.code) | |||
| 648 | + | case <-time.After(time.Second): | |||
| 649 | + | t.Fatal("waiter spun on a closed restore channel") | |||
| 650 | + | } | |||
| 651 | + | } | |||
| 652 | + | ||||
| 653 | + | // A healthy catalogue record whose runtime registration was skipped is not | |||
| 654 | + | // degraded durable state, and must not send the operator to durable recovery. | |||
| 655 | + | func TestWaitForTargetRestoreDistinguishesHealthyRuntime(t *testing.T) { | |||
| 656 | + | record := durableRecoveryRecord(0) | |||
| 657 | + | tests := []struct { | |||
| 658 | + | name string | |||
| 659 | + | state ports.SessionState | |||
| 660 | + | message string | |||
| 661 | + | }{ | |||
| 662 | + | {name: "healthy record without runtime", state: ports.SessionStopped, message: "session was not restored into this daemon: " + record.Name}, | |||
| 663 | + | {name: "broken durable state", state: ports.SessionBroken, message: "session durable state is broken: " + record.Name}, | |||
| 664 | + | } | |||
| 665 | + | for _, tt := range tests { | |||
| 666 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 667 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, &durableRecoveryRepository{}) | |||
| 668 | + | d.setStoppedRecovery(record, tt.state) | |||
| 669 | + | d.mu.Lock() | |||
| 670 | + | closeRuntimeRestoreDoneLocked(d.stopped[record.Name].restoreDone) | |||
| 671 | + | d.mu.Unlock() | |||
| 672 | + | ||||
| 673 | + | var protocolError *protoErr | |||
| 674 | + | require.ErrorAs(t, d.waitForTargetRestore(context.Background(), record.Name), &protocolError) | |||
| 675 | + | require.Equal(t, ports.ErrInternal, protocolError.code) | |||
| 676 | + | require.Equal(t, tt.message, protocolError.Error()) | |||
| 677 | + | }) | |||
| 678 | + | } | |||
| 679 | + | } | |||
| 680 | + | ||||
| 681 | + | func TestAttachRejectsDegraded(t *testing.T) { | |||
| 682 | + | record := durableRecoveryRecord(0) | |||
| 683 | + | record.DegradedReason = "checkpoint validation failed" | |||
| 684 | + | factory := &recoveryCountingPTYFactory{} | |||
| 685 | + | d := New(factory, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(newDurableRecoveryCatalogue([]domain.CatalogueRecord{record}), []domain.CatalogueRecord{record})) | |||
| 686 | + | d.serveCtx, d.serveCancel = context.WithCancel(context.Background()) | |||
| 687 | + | defer d.serveCancel() | |||
| 688 | + | ||||
| 689 | + | _, _, err := d.route(ports.Hello{Version: ports.ProtocolVersion, Intent: ports.IntentAttach, Name: record.Name}, nil) | |||
| 690 | + | var protocolError *protoErr | |||
| 691 | + | require.ErrorAs(t, err, &protocolError) | |||
| 692 | + | require.Equal(t, ports.ErrInternal, protocolError.code) | |||
| 693 | + | require.Zero(t, factory.calls.Load()) | |||
| 694 | + | } | |||
| 695 | + | ||||
| 696 | + | func TestRestoreUnavailableReleasesPerRecordWaiters(t *testing.T) { | |||
| 697 | + | record := durableRecoveryRecord(0) | |||
| 698 | + | d, _ := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, nil) | |||
| 699 | + | d.closeRestoreDone() | |||
| 700 | + | ||||
| 701 | + | d.mu.Lock() | |||
| 702 | + | entry := d.stopped[record.Name] | |||
| 703 | + | d.mu.Unlock() | |||
| 704 | + | require.Equal(t, ports.SessionBroken, entry.state) | |||
| 705 | + | select { | |||
| 706 | + | case <-entry.restoreDone: | |||
| 707 | + | default: | |||
| 708 | + | t.Fatal("per-record restore waiter remained blocked when repository was unavailable") | |||
| 709 | + | } | |||
| 710 | + | } | |||
| 711 | + | ||||
| 712 | + | func TestCatalogueRestoreDegradedVisible(t *testing.T) { | |||
| 713 | + | record := durableRecoveryRecord(0) | |||
| 714 | + | record.DegradedReason = "checkpoint validation failed" | |||
| 715 | + | repository := &durableRecoveryRepository{generations: make(map[string]ports.SnapshotGeneration), errors: make(map[string]error), loads: make(map[string]int), repairs: make(map[string]domain.CheckpointRef)} | |||
| 716 | + | d, catalogue := newDurableRecoveryDaemon(t, []domain.CatalogueRecord{record}, repository) | |||
| 717 | + | ||||
| 718 | + | d.restoreCatalogue(context.Background(), mustDurableRecords(t, catalogue)) | |||
| 719 | + | ||||
| 720 | + | require.Equal(t, ports.SessionBroken, d.stopped[record.Name].state) | |||
| 721 | + | require.Zero(t, repository.loads[record.Name]) | |||
| 722 | + | } | |||
A internal/usecase/daemon/maintenance.go +20 -0
| @@ -0,0 +1,20 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "fmt" | |||
| 6 | + | ) | |||
| 7 | + | ||||
| 8 | + | // CollectStartupGarbage runs the one coordinator-owned GC pass before the | |||
| 9 | + | // daemon socket is published. | |||
| 10 | + | func (d *Daemon) CollectStartupGarbage(ctx context.Context) error { | |||
| 11 | + | if d == nil || d.recovery == nil { | |||
| 12 | + | return nil | |||
| 13 | + | } | |||
| 14 | + | incarnations, err := d.recovery.CollectGarbage(ctx) | |||
| 15 | + | if err != nil { | |||
| 16 | + | return fmt.Errorf("collect startup snapshots: %w", err) | |||
| 17 | + | } | |||
| 18 | + | d.log.Info("snapshot_garbage_collection_complete", "incarnations", incarnations) | |||
| 19 | + | return nil | |||
| 20 | + | } | |||
A internal/usecase/daemon/maintenance_test.go +41 -0
| @@ -0,0 +1,41 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "errors" | |||
| 6 | + | "testing" | |||
| 7 | + | ||||
| 8 | + | "github.com/bnema/vev/internal/domain" | |||
| 9 | + | portsmocks "github.com/bnema/vev/internal/ports/mocks" | |||
| 10 | + | "github.com/stretchr/testify/mock" | |||
| 11 | + | "github.com/stretchr/testify/require" | |||
| 12 | + | ) | |||
| 13 | + | ||||
| 14 | + | func TestStartupGarbageCollectionUsesValidatedCatalogue(t *testing.T) { | |||
| 15 | + | committed := domain.CheckpointRef{Generation: 3, ManifestDigest: [32]byte{3}} | |||
| 16 | + | records := []domain.CatalogueRecord{ | |||
| 17 | + | {Name: "work", IncarnationID: domain.IncarnationID{1}, Committed: &committed}, | |||
| 18 | + | {Name: "fresh", IncarnationID: domain.IncarnationID{2}}, | |||
| 19 | + | } | |||
| 20 | + | catalogue := portsmocks.NewMockCatalogue(t) | |||
| 21 | + | catalogue.EXPECT().Records().Return(records, nil).Once() | |||
| 22 | + | repository := portsmocks.NewMockSnapshotRepository(t) | |||
| 23 | + | repository.EXPECT().CollectGarbage(mock.Anything, mock.MatchedBy(func(keep map[domain.IncarnationID]domain.CheckpointRef) bool { | |||
| 24 | + | return len(keep) == 2 && keep[records[0].IncarnationID] == committed && keep[records[1].IncarnationID] == (domain.CheckpointRef{}) | |||
| 25 | + | })).Return(nil).Once() | |||
| 26 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 27 | + | WithDurableMaintenance(catalogue, repository)(d) | |||
| 28 | + | ||||
| 29 | + | require.NoError(t, d.CollectStartupGarbage(context.Background())) | |||
| 30 | + | } | |||
| 31 | + | ||||
| 32 | + | func TestStartupGarbageCollectionSkipsUnknownCatalogueState(t *testing.T) { | |||
| 33 | + | catalogue := portsmocks.NewMockCatalogue(t) | |||
| 34 | + | catalogue.EXPECT().Records().Return(nil, errors.New("catalogue read failed")).Once() | |||
| 35 | + | repository := portsmocks.NewMockSnapshotRepository(t) | |||
| 36 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 37 | + | WithDurableMaintenance(catalogue, repository)(d) | |||
| 38 | + | ||||
| 39 | + | require.Error(t, d.CollectStartupGarbage(context.Background())) | |||
| 40 | + | repository.AssertNotCalled(t, "CollectGarbage", mock.Anything, mock.Anything) | |||
| 41 | + | } | |||
A internal/usecase/daemon/metadata_order_test.go +95 -0
| @@ -0,0 +1,95 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "bytes" | |||
| 5 | + | "sync" | |||
| 6 | + | "testing" | |||
| 7 | + | ||||
| 8 | + | "github.com/stretchr/testify/require" | |||
| 9 | + | ||||
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 11 | + | "github.com/bnema/vev/internal/ports" | |||
| 12 | + | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | |||
| 13 | + | ) | |||
| 14 | + | ||||
| 15 | + | type syncCountingCatalogue struct { | |||
| 16 | + | *durableRecoveryCatalogue | |||
| 17 | + | ||||
| 18 | + | muSync sync.Mutex | |||
| 19 | + | syncs int | |||
| 20 | + | } | |||
| 21 | + | ||||
| 22 | + | func (c *syncCountingCatalogue) Sync() error { | |||
| 23 | + | c.muSync.Lock() | |||
| 24 | + | defer c.muSync.Unlock() | |||
| 25 | + | c.syncs++ | |||
| 26 | + | return nil | |||
| 27 | + | } | |||
| 28 | + | ||||
| 29 | + | func (c *syncCountingCatalogue) syncCount() int { | |||
| 30 | + | c.muSync.Lock() | |||
| 31 | + | defer c.muSync.Unlock() | |||
| 32 | + | return c.syncs | |||
| 33 | + | } | |||
| 34 | + | ||||
| 35 | + | func (c *syncCountingCatalogue) Create(record domain.CatalogueRecord) error { | |||
| 36 | + | c.mu.Lock() | |||
| 37 | + | if _, exists := c.records[record.Name]; exists { | |||
| 38 | + | c.mu.Unlock() | |||
| 39 | + | return errSessionNameInUse | |||
| 40 | + | } | |||
| 41 | + | c.records[record.Name] = record | |||
| 42 | + | c.mu.Unlock() | |||
| 43 | + | return c.Sync() | |||
| 44 | + | } | |||
| 45 | + | ||||
| 46 | + | func TestMetadataWritesAreDeferred(t *testing.T) { | |||
| 47 | + | t.Parallel() | |||
| 48 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |||
| 49 | + | sess.mu.Lock() | |||
| 50 | + | record := sess.persistRecordLocked(sess.createdAt) | |||
| 51 | + | sess.mu.Unlock() | |||
| 52 | + | cat := &syncCountingCatalogue{durableRecoveryCatalogue: newDurableRecoveryCatalogue([]domain.CatalogueRecord{record})} | |||
| 53 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 54 | + | d.catalogue = cat | |||
| 55 | + | d.persistEnabled = true | |||
| 56 | + | d.sessions[sess.id] = sess | |||
| 57 | + | ||||
| 58 | + | syncsBefore := cat.syncCount() | |||
| 59 | + | require.NoError(t, d.renameTab(sess, sess.tabs[0], "build")) | |||
| 60 | + | d.touchMRU(sess) | |||
| 61 | + | require.Equal(t, syncsBefore, cat.syncCount(), "metadata updates must not force a durable write") | |||
| 62 | + | ||||
| 63 | + | require.NoError(t, d.flushCatalogue()) | |||
| 64 | + | require.Greater(t, cat.syncCount(), syncsBefore) | |||
| 65 | + | ||||
| 66 | + | got, ok, err := cat.Record("work") | |||
| 67 | + | require.NoError(t, err) | |||
| 68 | + | require.True(t, ok) | |||
| 69 | + | require.Equal(t, []string{"build"}, got.TabNames) | |||
| 70 | + | } | |||
| 71 | + | ||||
| 72 | + | func TestIdentityWritesAreSynchronous(t *testing.T) { | |||
| 73 | + | t.Parallel() | |||
| 74 | + | sess := newSnapshotTestSession(t, "0", true, "/work") | |||
| 75 | + | cat := &syncCountingCatalogue{durableRecoveryCatalogue: newDurableRecoveryCatalogue(nil)} | |||
| 76 | + | d := newTestDaemon(t, nil, stubClock{}) | |||
| 77 | + | d.catalogue = cat | |||
| 78 | + | d.persistEnabled = true | |||
| 79 | + | d.recovery = recoveryusecase.NewCoordinator(cat, nil, bytes.NewReader(bytes.Repeat([]byte{1}, 16))) | |||
| 80 | + | d.sessions[sess.id] = sess | |||
| 81 | + | ||||
| 82 | + | syncsBefore := cat.syncCount() | |||
| 83 | + | require.NoError(t, d.renameSession(sess, "work")) | |||
| 84 | + | require.Greater(t, cat.syncCount(), syncsBefore, "creating a named session must be durable before it is exposed") | |||
| 85 | + | ||||
| 86 | + | d.mu.Lock() | |||
| 87 | + | published := d.findByNameLocked("work") | |||
| 88 | + | d.mu.Unlock() | |||
| 89 | + | require.Same(t, sess, published) | |||
| 90 | + | sess.mu.Lock() | |||
| 91 | + | require.False(t, sess.ephemeral) | |||
| 92 | + | sess.mu.Unlock() | |||
| 93 | + | } | |||
| 94 | + | ||||
| 95 | + | var _ ports.Catalogue = (*syncCountingCatalogue)(nil) | |||
M internal/usecase/daemon/notify.go +14 -1
| @@ -198,6 +198,16 @@ func (d *Daemon) persistShutdownSnapshotFailure(name string, cause error) { | |||||
| 198 | if d.noticeStore == nil { | 198 | if d.noticeStore == nil { | ||
| 199 | return | 199 | return | ||
| 200 | } | 200 | } | ||
| 201 | + | d.shutdownNoticeMu.Lock() | |||
| 202 | + | if d.shutdownNoticedSessions == nil { | |||
| 203 | + | d.shutdownNoticedSessions = make(map[string]struct{}) | |||
| 204 | + | } | |||
| 205 | + | if _, exists := d.shutdownNoticedSessions[name]; exists { | |||
| 206 | + | d.shutdownNoticeMu.Unlock() | |||
| 207 | + | return | |||
| 208 | + | } | |||
| 209 | + | d.shutdownNoticedSessions[name] = struct{}{} | |||
| 210 | + | d.shutdownNoticeMu.Unlock() | |||
| 201 | if err := d.noticeStore.Append(domain.Notification{ | 211 | if err := d.noticeStore.Append(domain.Notification{ | ||
| 202 | Code: domain.NoticeSnapshotWrite, | 212 | Code: domain.NoticeSnapshotWrite, | ||
| 203 | Severity: domain.NoticeError, | 213 | Severity: domain.NoticeError, | ||
| @@ -205,7 +215,10 @@ func (d *Daemon) persistShutdownSnapshotFailure(name string, cause error) { | |||||
| 205 | Details: noticeDetails(cause), | 215 | Details: noticeDetails(cause), | ||
| 206 | Time: d.clock.Now(), | 216 | Time: d.clock.Now(), | ||
| 207 | }); err != nil { | 217 | }); err != nil { | ||
| 208 | - | d.log.Warn("persisting shutdown notice failed", "err", err, "session", name) | 218 | + | d.shutdownNoticeMu.Lock() |
| 219 | + | delete(d.shutdownNoticedSessions, name) | |||
| 220 | + | d.shutdownNoticeMu.Unlock() | |||
| 221 | + | d.log.Warn("persisting shutdown notice failed", "err", err, "session", name, "action", "final-checkpoint-timeout") | |||
| 209 | } | 222 | } | ||
| 210 | } | 223 | } | ||
| 211 | 224 | ||||
M internal/usecase/daemon/performance_benchmark_test.go +19 -12
| @@ -627,12 +627,18 @@ func publishBenchmarkSnapshot(b testing.TB, fixture *performanceFixture, generat | |||||
| 627 | if err := fixture.snapshots.Publish(context.Background(), publication); err != nil { | 627 | if err := fixture.snapshots.Publish(context.Background(), publication); err != nil { | ||
| 628 | b.Fatal(err) | 628 | b.Fatal(err) | ||
| 629 | } | 629 | } | ||
| 630 | + | checkpoint := domain.CheckpointRef{Generation: generation, ManifestDigest: snapcodec.ManifestDigest(publication.Manifest)} | |||
| 631 | + | capture.checkpoint = checkpoint | |||
| 632 | + | fixture.sess.snapshotMu.Lock() | |||
| 633 | + | fixture.sess.snapshotPublishedGeneration = generation | |||
| 634 | + | fixture.sess.snapshotPublishedCheckpoint = &checkpoint | |||
| 635 | + | fixture.sess.snapshotMu.Unlock() | |||
| 630 | markSnapshotCaptureObjectsPublished(capture) | 636 | markSnapshotCaptureObjectsPublished(capture) | ||
| 631 | } | 637 | } | ||
| 632 | 638 | ||||
| 633 | func mutateBenchmarkVisible(fixture *performanceFixture, operation int) { | 639 | func mutateBenchmarkVisible(fixture *performanceFixture, operation int) { | ||
| 634 | fixture.activePane.mu.Lock() | 640 | fixture.activePane.mu.Lock() | ||
| 635 | - | fixture.activePane.screen.Write([]byte(fmt.Sprintf("\x1b[1;1Hvisible-%08d", operation))) | 641 | + | fixture.activePane.screen.Write(fmt.Appendf(nil, "\x1b[1;1Hvisible-%08d", operation)) |
| 636 | fixture.activePane.mu.Unlock() | 642 | fixture.activePane.mu.Unlock() | ||
| 637 | } | 643 | } | ||
| 638 | 644 | ||||
| @@ -647,7 +653,7 @@ func mutateBenchmarkTail(fixture *performanceFixture, operation int) { | |||||
| 647 | } | 653 | } | ||
| 648 | 654 | ||||
| 649 | func mutateBenchmarkSealedChunk(fixture *performanceFixture, operation int) { | 655 | func mutateBenchmarkSealedChunk(fixture *performanceFixture, operation int) { | ||
| 650 | - | for row := 0; row < 256; row++ { | 656 | + | for row := range 256 { |
| 651 | mutateBenchmarkTail(fixture, operation*256+row) | 657 | mutateBenchmarkTail(fixture, operation*256+row) | ||
| 652 | } | 658 | } | ||
| 653 | } | 659 | } | ||
| @@ -719,7 +725,14 @@ func snapshotGeneration(publication ports.SnapshotPublication) ports.SnapshotGen | |||||
| 719 | for _, object := range publication.Objects { | 725 | for _, object := range publication.Objects { | ||
| 720 | objects[object.Digest] = object.Data | 726 | objects[object.Digest] = object.Data | ||
| 721 | } | 727 | } | ||
| 722 | - | return ports.SnapshotGeneration{Name: publication.Name, Generation: publication.Generation, Manifest: publication.Manifest, Objects: objects} | 728 | + | return ports.SnapshotGeneration{ |
| 729 | + | IncarnationID: publication.IncarnationID, | |||
| 730 | + | Name: publication.Name, | |||
| 731 | + | Generation: publication.Generation, | |||
| 732 | + | ParentCheckpoint: publication.ParentCheckpoint, | |||
| 733 | + | Manifest: publication.Manifest, | |||
| 734 | + | Objects: objects, | |||
| 735 | + | } | |||
| 723 | } | 736 | } | ||
| 724 | 737 | ||||
| 725 | func snapshotObjectBytes(publication ports.SnapshotPublication) uint64 { | 738 | func snapshotObjectBytes(publication ports.SnapshotPublication) uint64 { | ||
| @@ -1036,6 +1049,7 @@ func newPerformanceFixtureWithCleanup(t testing.TB, config performanceConfig, re | |||||
| 1036 | ac.size = config.size | 1049 | ac.size = config.size | ||
| 1037 | sess.name = "performance" | 1050 | sess.name = "performance" | ||
| 1038 | sess.ephemeral = false | 1051 | sess.ephemeral = false | ||
| 1052 | + | sess.incarnation = domain.IncarnationID{1} | |||
| 1039 | sess.snapEligible.Store(true) | 1053 | sess.snapEligible.Store(true) | ||
| 1040 | 1054 | ||||
| 1041 | fixture := &performanceFixture{ | 1055 | fixture := &performanceFixture{ | ||
| @@ -1051,7 +1065,7 @@ func newPerformanceFixtureWithCleanup(t testing.TB, config performanceConfig, re | |||||
| 1051 | liveWrites: [][]byte{[]byte("\x1b[1;1HA\x1b[2;2HA"), []byte("\x1b[1;1HB\x1b[2;2HB")}, | 1065 | liveWrites: [][]byte{[]byte("\x1b[1;1HA\x1b[2;2HA"), []byte("\x1b[1;1HB\x1b[2;2HB")}, | ||
| 1052 | resizeSizes: [2]domain.Size{{Cols: 100, Rows: 30}, config.size}, | 1066 | resizeSizes: [2]domain.Size{{Cols: 100, Rows: 30}, config.size}, | ||
| 1053 | } | 1067 | } | ||
| 1054 | - | WithSnapshotRepository(fixture.snapshots, nil)(d) | 1068 | + | WithSnapshotRepository(fixture.snapshots)(d) |
| 1055 | d.startSnapshotEncodeWorker() | 1069 | d.startSnapshotEncodeWorker() | ||
| 1056 | if registerCleanup { | 1070 | if registerCleanup { | ||
| 1057 | t.Cleanup(d.stopSnapshotEncodeWorker) | 1071 | t.Cleanup(d.stopSnapshotEncodeWorker) | ||
| @@ -1376,6 +1390,7 @@ type countingSnapshotMetrics struct { | |||||
| 1376 | writes, objectBytes, historyBlobBytes, suppliedObjectBytes, suppliedHistoryBytes, manifestBytes, headBytes uint64 | 1390 | writes, objectBytes, historyBlobBytes, suppliedObjectBytes, suppliedHistoryBytes, manifestBytes, headBytes uint64 | ||
| 1377 | } | 1391 | } | ||
| 1378 | type countingSnapshotRepository struct { | 1392 | type countingSnapshotRepository struct { | ||
| 1393 | + | noOpSnapshotRepository | |||
| 1379 | mu sync.Mutex | 1394 | mu sync.Mutex | ||
| 1380 | countingSnapshotMetrics | 1395 | countingSnapshotMetrics | ||
| 1381 | objects map[string]map[ports.SnapshotDigest]struct{} | 1396 | objects map[string]map[ports.SnapshotDigest]struct{} | ||
| @@ -1418,14 +1433,6 @@ func (s *countingSnapshotRepository) Publish(_ context.Context, publication port | |||||
| 1418 | s.last = publication.Manifest | 1433 | s.last = publication.Manifest | ||
| 1419 | return nil | 1434 | return nil | ||
| 1420 | } | 1435 | } | ||
| 1421 | - | func (*countingSnapshotRepository) List(context.Context) ([]string, error) { return nil, nil } | |||
| 1422 | - | func (*countingSnapshotRepository) Load(context.Context, string) (ports.SnapshotGeneration, error) { | |||
| 1423 | - | return ports.SnapshotGeneration{}, nil | |||
| 1424 | - | } | |||
| 1425 | - | func (*countingSnapshotRepository) Delete(context.Context, string) error { return nil } | |||
| 1426 | - | func (*countingSnapshotRepository) Tombstone(context.Context, string) error { return nil } | |||
| 1427 | - | func (*countingSnapshotRepository) DeleteTombstone(context.Context, string) error { return nil } | |||
| 1428 | - | func (*countingSnapshotRepository) Maintain(context.Context) error { return nil } | |||
| 1429 | func (s *countingSnapshotRepository) reset() { | 1436 | func (s *countingSnapshotRepository) reset() { | ||
| 1430 | s.mu.Lock() | 1437 | s.mu.Lock() | ||
| 1431 | s.countingSnapshotMetrics = countingSnapshotMetrics{} | 1438 | s.countingSnapshotMetrics = countingSnapshotMetrics{} | ||
M internal/usecase/daemon/picker.go +16 -0
| @@ -2,6 +2,7 @@ | |||||
| 2 | package daemon | 2 | package daemon | ||
| 3 | 3 | ||||
| 4 | import ( | 4 | import ( | ||
| 5 | + | "context" | |||
| 5 | "sort" | 6 | "sort" | ||
| 6 | 7 | ||||
| 7 | "github.com/bnema/vev/internal/domain" | 8 | "github.com/bnema/vev/internal/domain" | ||
| @@ -358,6 +359,17 @@ func (d *Daemon) switchToTarget(from *session, ac *attachedClient, target picker | |||||
| 358 | // switchToTargetGuarded is the navigation-only variant whose guard is checked | 359 | // switchToTargetGuarded is the navigation-only variant whose guard is checked | ||
| 359 | // as part of the source ownership transfer. | 360 | // as part of the source ownership transfer. | ||
| 360 | func (d *Daemon) switchToTargetGuarded(from *session, ac *attachedClient, target picker.Target, guard sessionHandoffGuard) error { | 361 | func (d *Daemon) switchToTargetGuarded(from *session, ac *attachedClient, target picker.Target, guard sessionHandoffGuard) error { | ||
| 362 | + | if target.Name != "" { | |||
| 363 | + | ctx := d.serveCtx | |||
| 364 | + | if ctx == nil { | |||
| 365 | + | ctx = context.Background() | |||
| 366 | + | } | |||
| 367 | + | if err := d.waitForTargetRestore(ctx, target.Name); err != nil { | |||
| 368 | + | d.invalidateRender(from, ac, true, "picker.go") | |||
| 369 | + | return domain.UserErr(domain.NoticeSessionUnavailable, "couldn't switch to that session", err) | |||
| 370 | + | } | |||
| 371 | + | } | |||
| 372 | + | ||||
| 361 | d.mu.Lock() | 373 | d.mu.Lock() | ||
| 362 | var ( | 374 | var ( | ||
| 363 | targetSess *session | 375 | targetSess *session | ||
| @@ -508,6 +520,10 @@ func (d *Daemon) switchToActiveTargetLocked(from *session, ac *attachedClient, t | |||||
| 508 | // the handoff while d.mu is held. Creation failure leaves the source client | 520 | // the handoff while d.mu is held. Creation failure leaves the source client | ||
| 509 | // and stopped record untouched. | 521 | // and stopped record untouched. | ||
| 510 | func (d *Daemon) resumeStoppedAndSwitchLocked(from *session, ac *attachedClient, target picker.Target, stopped stoppedSession) (*session, []renderLifecycleCleanup, bool, error) { | 522 | func (d *Daemon) resumeStoppedAndSwitchLocked(from *session, ac *attachedClient, target picker.Target, stopped stoppedSession) (*session, []renderLifecycleCleanup, bool, error) { | ||
| 523 | + | if stopped.record.Name != "" && stopped.state == ports.SessionBroken { | |||
| 524 | + | return nil, nil, false, &protoErr{ports.ErrInternal, "session durable state is broken: " + target.Name} | |||
| 525 | + | } | |||
| 526 | + | ||||
| 511 | // The caller already holds d.mu. Keep handoff ownership atomic with global | 527 | // The caller already holds d.mu. Keep handoff ownership atomic with global | ||
| 512 | // routing so it cannot select an attachment midway between sessions. | 528 | // routing so it cannot select an attachment midway between sessions. | ||
| 513 | d.notices.routingMu.Lock() | 529 | d.notices.routingMu.Lock() | ||
M internal/usecase/daemon/picker_test.go +174 -34
| @@ -11,11 +11,13 @@ import ( | |||||
| 11 | "github.com/stretchr/testify/require" | 11 | "github.com/stretchr/testify/require" | ||
| 12 | 12 | ||||
| 13 | "github.com/bnema/vev/internal/domain" | 13 | "github.com/bnema/vev/internal/domain" | ||
| 14 | + | "github.com/bnema/vev/internal/persist" | |||
| 14 | "github.com/bnema/vev/internal/ports" | 15 | "github.com/bnema/vev/internal/ports" | ||
| 15 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | 16 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | ||
| 16 | "github.com/bnema/vev/internal/usecase/keys" | 17 | "github.com/bnema/vev/internal/usecase/keys" | ||
| 17 | "github.com/bnema/vev/internal/usecase/layout" | 18 | "github.com/bnema/vev/internal/usecase/layout" | ||
| 18 | "github.com/bnema/vev/internal/usecase/picker" | 19 | "github.com/bnema/vev/internal/usecase/picker" | ||
| 20 | + | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | |||
| 19 | themeui "github.com/bnema/vev/internal/usecase/theme" | 21 | themeui "github.com/bnema/vev/internal/usecase/theme" | ||
| 20 | "github.com/bnema/vev/pkg/renderer" | 22 | "github.com/bnema/vev/pkg/renderer" | ||
| 21 | "github.com/bnema/vev/pkg/vt" | 23 | "github.com/bnema/vev/pkg/vt" | ||
| @@ -23,33 +25,17 @@ import ( | |||||
| 23 | 25 | ||||
| 24 | // --- test doubles ----------------------------------------------------------- | 26 | // --- test doubles ----------------------------------------------------------- | ||
| 25 | 27 | ||||
| 26 | - | // failingDeleteStore is a ports.Store whose Delete always fails, used to | 28 | + | type refusingSnapshotDeleteRepository struct { |
| 27 | - | // exercise the picker's kill-target error path without disturbing the | 29 | + | noOpSnapshotRepository |
| 28 | - | // generated mock's default Delete expectation. | 30 | + | err error |
| 29 | - | type failingDeleteStore struct{ err error } | 31 | + | } |
| 30 | - | ||||
| 31 | - | func (failingDeleteStore) Get([]byte) ([]byte, bool) { return nil, false } | |||
| 32 | - | func (failingDeleteStore) Set([]byte, []byte) error { return nil } | |||
| 33 | - | func (s failingDeleteStore) Delete([]byte) error { return s.err } | |||
| 34 | - | func (failingDeleteStore) Range(func(k, v []byte) bool) {} | |||
| 35 | - | func (failingDeleteStore) Sync() error { return nil } | |||
| 36 | - | func (failingDeleteStore) Close() error { return nil } | |||
| 37 | - | ||||
| 38 | - | type refusingSnapshotDeleteRepository struct{ err error } | |||
| 39 | 32 | ||||
| 40 | func (refusingSnapshotDeleteRepository) Publish(context.Context, ports.SnapshotPublication) error { | 33 | func (refusingSnapshotDeleteRepository) Publish(context.Context, ports.SnapshotPublication) error { | ||
| 41 | return nil | 34 | return nil | ||
| 42 | } | 35 | } | ||
| 43 | - | func (refusingSnapshotDeleteRepository) List(context.Context) ([]string, error) { return nil, nil } | 36 | + | func (s refusingSnapshotDeleteRepository) DeleteIncarnation(context.Context, domain.IncarnationID) error { |
| 44 | - | func (refusingSnapshotDeleteRepository) Load(context.Context, string) (ports.SnapshotGeneration, error) { | 37 | + | return s.err |
| 45 | - | return ports.SnapshotGeneration{}, nil | |||
| 46 | } | 38 | } | ||
| 47 | - | func (s refusingSnapshotDeleteRepository) Delete(context.Context, string) error { return s.err } | |||
| 48 | - | func (refusingSnapshotDeleteRepository) Tombstone(context.Context, string) error { return nil } | |||
| 49 | - | func (refusingSnapshotDeleteRepository) DeleteTombstone(context.Context, string) error { | |||
| 50 | - | return nil | |||
| 51 | - | } | |||
| 52 | - | func (refusingSnapshotDeleteRepository) Maintain(context.Context) error { return nil } | |||
| 53 | 39 | ||||
| 54 | func newTestTabWithContext(p ports.PTY, ctx context.Context, cancel context.CancelFunc) *tab { | 40 | func newTestTabWithContext(p ports.PTY, ctx context.Context, cancel context.CancelFunc) *tab { | ||
| 55 | tb := newTab(p, domain.Size{Cols: 80, Rows: 23}) | 41 | tb := newTab(p, domain.Size{Cols: 80, Rows: 23}) | ||
| @@ -234,6 +220,148 @@ func TestPickerViewsOmitsTerminalTitleWhenTabsConfigDisabled(t *testing.T) { | |||||
| 234 | require.Equal(t, []picker.TabEntry{{Name: "1", Detail: " (vim)"}, {Name: "logs", Detail: " (sh)"}}, views[0].Tabs) | 220 | require.Equal(t, []picker.TabEntry{{Name: "1", Detail: " (vim)"}, {Name: "logs", Detail: " (sh)"}}, views[0].Tabs) | ||
| 235 | } | 221 | } | ||
| 236 | 222 | ||||
| 223 | + | func TestPickerWaitsForRestoringTargetBeforeSwitching(t *testing.T) { | |||
| 224 | + | p, release := newBlockingPTY(t) | |||
| 225 | + | defer release() | |||
| 226 | + | d, from, ac, _ := newManualSessionWithPTYs(t, p) | |||
| 227 | + | record := durableRecoveryRecord(0) | |||
| 228 | + | record.Name = "restoring" | |||
| 229 | + | state, done := initialSessionState(record) | |||
| 230 | + | d.stopped[record.Name] = stoppedSession{ | |||
| 231 | + | name: record.Name, | |||
| 232 | + | createdAt: record.CreatedAt, | |||
| 233 | + | incarnation: record.IncarnationID, | |||
| 234 | + | record: record, | |||
| 235 | + | state: state, | |||
| 236 | + | restoreDone: done, | |||
| 237 | + | } | |||
| 238 | + | factory := &recoveryCountingPTYFactory{} | |||
| 239 | + | d.ptys = factory | |||
| 240 | + | ||||
| 241 | + | result := make(chan error, 1) | |||
| 242 | + | go func() { | |||
| 243 | + | result <- d.switchToTarget(from, ac, picker.Target{Name: record.Name, Stopped: true}) | |||
| 244 | + | }() | |||
| 245 | + | ||||
| 246 | + | select { | |||
| 247 | + | case err := <-result: | |||
| 248 | + | t.Fatalf("picker switch returned before target restoration completed: %v", err) | |||
| 249 | + | case <-time.After(25 * time.Millisecond): | |||
| 250 | + | } | |||
| 251 | + | require.Zero(t, factory.calls.Load()) | |||
| 252 | + | require.Same(t, from, ac.currentSession()) | |||
| 253 | + | ||||
| 254 | + | ctx, cancel := context.WithCancel(d.serveCtx) | |||
| 255 | + | defer cancel() | |||
| 256 | + | target := &session{id: "restored", name: record.Name, createdAt: record.CreatedAt, ctx: ctx, cancel: cancel, tabs: []*tab{newTab(nil, domain.Size{Cols: 80, Rows: 23})}} | |||
| 257 | + | d.mu.Lock() | |||
| 258 | + | delete(d.stopped, record.Name) | |||
| 259 | + | d.sessions[target.id] = target | |||
| 260 | + | closeRuntimeRestoreDoneLocked(done) | |||
| 261 | + | d.mu.Unlock() | |||
| 262 | + | ||||
| 263 | + | require.NoError(t, <-result) | |||
| 264 | + | require.Same(t, target, ac.currentSession()) | |||
| 265 | + | } | |||
| 266 | + | ||||
| 267 | + | func TestRestoreCancellationTransitionsBeforePickerCompletion(t *testing.T) { | |||
| 268 | + | p, release := newBlockingPTY(t) | |||
| 269 | + | defer release() | |||
| 270 | + | d, from, ac, _ := newManualSessionWithPTYs(t, p) | |||
| 271 | + | record := durableRecoveryRecord(0) | |||
| 272 | + | record.Name = "restoring" | |||
| 273 | + | repository := &cancellationRecoveryRepository{started: make(chan struct{})} | |||
| 274 | + | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record}) | |||
| 275 | + | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil) | |||
| 276 | + | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | |||
| 277 | + | WithSnapshotRepository(repository)(d) | |||
| 278 | + | WithRecoveryCoordinator(coordinator)(d) | |||
| 279 | + | restoreCtx, cancelRestore := context.WithCancel(context.Background()) | |||
| 280 | + | restored := make(chan struct{}) | |||
| 281 | + | go func() { | |||
| 282 | + | d.restoreCatalogue(restoreCtx, mustDurableRecords(t, catalogue)) | |||
| 283 | + | close(restored) | |||
| 284 | + | }() | |||
| 285 | + | <-repository.started | |||
| 286 | + | ||||
| 287 | + | result := make(chan error, 1) | |||
| 288 | + | go func() { | |||
| 289 | + | result <- d.switchToTarget(from, ac, picker.Target{Name: record.Name, Stopped: true}) | |||
| 290 | + | }() | |||
| 291 | + | cancelRestore() | |||
| 292 | + | ||||
| 293 | + | require.Error(t, <-result) | |||
| 294 | + | <-restored | |||
| 295 | + | require.Same(t, from, ac.currentSession()) | |||
| 296 | + | d.mu.Lock() | |||
| 297 | + | entry := d.stopped[record.Name] | |||
| 298 | + | d.mu.Unlock() | |||
| 299 | + | require.Equal(t, ports.SessionBroken, entry.state) | |||
| 300 | + | require.Equal(t, "restore interrupted", entry.record.DegradedReason) | |||
| 301 | + | select { | |||
| 302 | + | case <-entry.restoreDone: | |||
| 303 | + | default: | |||
| 304 | + | t.Fatal("restore completion was not signaled") | |||
| 305 | + | } | |||
| 306 | + | } | |||
| 307 | + | ||||
| 308 | + | func TestPickerRejectsCatalogueTargetsWithoutFreshRuntime(t *testing.T) { | |||
| 309 | + | tests := []struct { | |||
| 310 | + | name string | |||
| 311 | + | broken bool | |||
| 312 | + | runtimeState ports.SessionState | |||
| 313 | + | }{ | |||
| 314 | + | {name: "broken", broken: true, runtimeState: ports.SessionBroken}, | |||
| 315 | + | {name: "healthy without restored runtime", runtimeState: ports.SessionStopped}, | |||
| 316 | + | } | |||
| 317 | + | ||||
| 318 | + | for _, tt := range tests { | |||
| 319 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 320 | + | p, release := newBlockingPTY(t) | |||
| 321 | + | defer release() | |||
| 322 | + | d, from, ac, _ := newManualSessionWithPTYs(t, p) | |||
| 323 | + | store, storeState := newMockStore(t) | |||
| 324 | + | record := durableRecoveryRecord(0) | |||
| 325 | + | record.Name = "unsafe" | |||
| 326 | + | if tt.broken { | |||
| 327 | + | record.DegradedReason = "checkpoint validation failed" | |||
| 328 | + | } | |||
| 329 | + | catalogue := persist.New(store) | |||
| 330 | + | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | |||
| 331 | + | d.stopped[record.Name] = stoppedSession{ | |||
| 332 | + | name: record.Name, | |||
| 333 | + | createdAt: record.CreatedAt, | |||
| 334 | + | incarnation: record.IncarnationID, | |||
| 335 | + | record: record, | |||
| 336 | + | state: tt.runtimeState, | |||
| 337 | + | } | |||
| 338 | + | factory := &recoveryCountingPTYFactory{} | |||
| 339 | + | d.ptys = factory | |||
| 340 | + | ||||
| 341 | + | err := d.switchToTarget(from, ac, picker.Target{Name: record.Name, Stopped: true}) | |||
| 342 | + | ||||
| 343 | + | var userError *domain.UserError | |||
| 344 | + | require.ErrorAs(t, err, &userError) | |||
| 345 | + | require.Equal(t, domain.NoticeSessionUnavailable, userError.Code) | |||
| 346 | + | var protocolError *protoErr | |||
| 347 | + | require.ErrorAs(t, err, &protocolError) | |||
| 348 | + | require.Equal(t, ports.ErrInternal, protocolError.code) | |||
| 349 | + | require.Zero(t, factory.calls.Load(), "unsafe target must not open a PTY") | |||
| 350 | + | require.Same(t, from, ac.currentSession()) | |||
| 351 | + | ||||
| 352 | + | storeState.mu.Lock() | |||
| 353 | + | sets := storeState.sets | |||
| 354 | + | storeState.mu.Unlock() | |||
| 355 | + | require.Zero(t, sets, "unsafe target must not mutate the catalogue") | |||
| 356 | + | d.mu.Lock() | |||
| 357 | + | entry := d.stopped[record.Name] | |||
| 358 | + | d.mu.Unlock() | |||
| 359 | + | require.Equal(t, record, entry.record) | |||
| 360 | + | require.Equal(t, tt.runtimeState, entry.state) | |||
| 361 | + | }) | |||
| 362 | + | } | |||
| 363 | + | } | |||
| 364 | + | ||||
| 237 | func TestPickerResumesStoppedSessionWithPersistedTabNames(t *testing.T) { | 365 | func TestPickerResumesStoppedSessionWithPersistedTabNames(t *testing.T) { | ||
| 238 | p1, release1 := newBlockingPTY(t) | 366 | p1, release1 := newBlockingPTY(t) | ||
| 239 | p2, release2 := newBlockingPTY(t) | 367 | p2, release2 := newBlockingPTY(t) | ||
| @@ -243,7 +371,7 @@ func TestPickerResumesStoppedSessionWithPersistedTabNames(t *testing.T) { | |||||
| 243 | defer release3() | 371 | defer release3() | ||
| 244 | d, from, ac, sends := newManualSessionWithPTYs(t, p1) | 372 | d, from, ac, sends := newManualSessionWithPTYs(t, p1) | ||
| 245 | d.ptys = newFactorySeq(t, p2, p3) | 373 | d.ptys = newFactorySeq(t, p2, p3) | ||
| 246 | - | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}} | 374 | + | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}, record: domain.CatalogueRecord{Name: "work"}, state: ports.SessionStopped} |
| 247 | 375 | ||||
| 248 | d.resumeStoppedAndSwitch(from, ac, picker.Target{Name: "work", Stopped: true}) | 376 | d.resumeStoppedAndSwitch(from, ac, picker.Target{Name: "work", Stopped: true}) | ||
| 249 | awaitFrame(t, sends, ports.MsgOutput) | 377 | awaitFrame(t, sends, ports.MsgOutput) | ||
| @@ -1007,8 +1135,17 @@ func TestPickerKillActiveSessionSnapshotDeleteRefusalReportsOnceAndKeepsPicker(t | |||||
| 1007 | d, from, ac, sends, releases := newRecentNavigationTestSessions(t) | 1135 | d, from, ac, sends, releases := newRecentNavigationTestSessions(t) | ||
| 1008 | defer releaseAll(releases) | 1136 | defer releaseAll(releases) | ||
| 1009 | cause := errors.New("snapshot delete refused") | 1137 | cause := errors.New("snapshot delete refused") | ||
| 1010 | - | WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause}, nil)(d) | 1138 | + | WithSnapshotRepository(refusingSnapshotDeleteRepository{err: cause})(d) |
| 1011 | target := d.sessions[domain.SessionID("recent")] | 1139 | target := d.sessions[domain.SessionID("recent")] | ||
| 1140 | + | store, _ := newMockStore(t) | |||
| 1141 | + | WithStore(t, store)(d) | |||
| 1142 | + | target.mu.Lock() | |||
| 1143 | + | if target.incarnation == (domain.IncarnationID{}) { | |||
| 1144 | + | target.incarnation = domain.IncarnationID{1} | |||
| 1145 | + | } | |||
| 1146 | + | record := target.persistRecordLocked(1) | |||
| 1147 | + | target.mu.Unlock() | |||
| 1148 | + | require.NoError(t, d.catalogue.Create(record)) | |||
| 1012 | 1149 | ||||
| 1013 | d.enterPicker(from, ac) | 1150 | d.enterPicker(from, ac) | ||
| 1014 | awaitFrame(t, sends, ports.MsgOutput) | 1151 | awaitFrame(t, sends, ports.MsgOutput) | ||
| @@ -1041,8 +1178,14 @@ func TestPickerKillStoppedSessionPersistDeleteFailureSurfacesNoticeAndKeepsEntry | |||||
| 1041 | defer release() | 1178 | defer release() | ||
| 1042 | d, from, ac, sends := newManualSessionWithPTYs(t, p) | 1179 | d, from, ac, sends := newManualSessionWithPTYs(t, p) | ||
| 1043 | cause := errors.New("delete failed") | 1180 | cause := errors.New("delete failed") | ||
| 1044 | - | WithStore(failingDeleteStore{err: cause})(d) | 1181 | + | store, state := newMockStore(t) |
| 1045 | - | d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp/stopped", createdAt: 7} | 1182 | + | WithStore(t, store)(d) |
| 1183 | + | record := domain.CatalogueRecord{Name: "stopped", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/stopped", CreatedAt: 7} | |||
| 1184 | + | require.NoError(t, d.catalogue.Create(record)) | |||
| 1185 | + | state.mu.Lock() | |||
| 1186 | + | state.deleteErr = func(string) error { return cause } | |||
| 1187 | + | state.mu.Unlock() | |||
| 1188 | + | d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp/stopped", createdAt: 7, incarnation: record.IncarnationID, record: record} | |||
| 1046 | 1189 | ||||
| 1047 | d.enterPicker(from, ac) | 1190 | d.enterPicker(from, ac) | ||
| 1048 | awaitFrame(t, sends, ports.MsgOutput) | 1191 | awaitFrame(t, sends, ports.MsgOutput) | ||
| @@ -1059,12 +1202,9 @@ func TestPickerKillStoppedSessionPersistDeleteFailureSurfacesNoticeAndKeepsEntry | |||||
| 1059 | toasts := awaitToastCount(t, ac, 1) | 1202 | toasts := awaitToastCount(t, ac, 1) | ||
| 1060 | require.Equal(t, domain.NoticePersistDelete, toasts[0].Code) | 1203 | require.Equal(t, domain.NoticePersistDelete, toasts[0].Code) | ||
| 1061 | 1204 | ||||
| 1062 | - | views, _ := d.pickerViews(from) | 1205 | + | d.mu.Lock() |
| 1063 | - | var stillListed bool | 1206 | + | stopped, retained := d.stopped["stopped"] |
| 1064 | - | for _, v := range views { | 1207 | + | d.mu.Unlock() |
| 1065 | - | if v.Stopped && v.Name == "stopped" { | 1208 | + | require.True(t, retained, "failed deletion must retain the reserved name") |
| 1066 | - | stillListed = true | 1209 | + | require.True(t, stopped.purging, "an uncertain deletion stays fenced from restore") |
| 1067 | - | } | |||
| 1068 | - | } | |||
| 1069 | - | require.True(t, stillListed, "entry must remain listed after refreshPicker when the delete failed") | |||
| 1070 | } | 1210 | } | ||
A internal/usecase/daemon/recovery_observability.go +61 -0
| @@ -0,0 +1,61 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import "github.com/bnema/vev/internal/domain" | |||
| 4 | + | ||||
| 5 | + | func (d *Daemon) logSessionRestoreComplete(record domain.CatalogueRecord, generation uint64, fallback bool) { | |||
| 6 | + | d.log.Info("session_restore_complete", | |||
| 7 | + | "session", record.Name, | |||
| 8 | + | "incarnation", record.IncarnationID.String(), | |||
| 9 | + | "generation", generation, | |||
| 10 | + | "fallback", fallback, | |||
| 11 | + | ) | |||
| 12 | + | } | |||
| 13 | + | ||||
| 14 | + | func (d *Daemon) logSessionDegraded(record domain.CatalogueRecord, reasonCode string) { | |||
| 15 | + | d.log.Warn("session_degraded", | |||
| 16 | + | "session", record.Name, | |||
| 17 | + | "incarnation", record.IncarnationID.String(), | |||
| 18 | + | "reason_code", reasonCode, | |||
| 19 | + | ) | |||
| 20 | + | } | |||
| 21 | + | ||||
| 22 | + | func (d *Daemon) logStartupRecoveryCounts(restoring int) { | |||
| 23 | + | var records []domain.CatalogueRecord | |||
| 24 | + | if d.persistEnabled && d.catalogue != nil { | |||
| 25 | + | var err error | |||
| 26 | + | records, err = d.catalogue.Records() | |||
| 27 | + | if err != nil { | |||
| 28 | + | d.log.Error("daemon_startup_catalogue_read_failed", "err", err) | |||
| 29 | + | records = d.stoppedRecords() | |||
| 30 | + | } | |||
| 31 | + | } else { | |||
| 32 | + | records = d.stoppedRecords() | |||
| 33 | + | } | |||
| 34 | + | healthy, fresh, broken := 0, 0, 0 | |||
| 35 | + | for _, record := range records { | |||
| 36 | + | switch { | |||
| 37 | + | case record.DegradedReason != "": | |||
| 38 | + | broken++ | |||
| 39 | + | case record.Committed == nil: | |||
| 40 | + | fresh++ | |||
| 41 | + | default: | |||
| 42 | + | healthy++ | |||
| 43 | + | } | |||
| 44 | + | } | |||
| 45 | + | d.log.Info("daemon_startup_complete", | |||
| 46 | + | "healthy", healthy, | |||
| 47 | + | "fresh", fresh, | |||
| 48 | + | "restoring", restoring, | |||
| 49 | + | "broken", broken, | |||
| 50 | + | ) | |||
| 51 | + | } | |||
| 52 | + | ||||
| 53 | + | func (d *Daemon) stoppedRecords() []domain.CatalogueRecord { | |||
| 54 | + | d.mu.Lock() | |||
| 55 | + | defer d.mu.Unlock() | |||
| 56 | + | records := make([]domain.CatalogueRecord, 0, len(d.stopped)) | |||
| 57 | + | for _, entry := range d.stopped { | |||
| 58 | + | records = append(records, entry.record) | |||
| 59 | + | } | |||
| 60 | + | return records | |||
| 61 | + | } | |||
A internal/usecase/daemon/recovery_observability_test.go +92 -0
| @@ -0,0 +1,92 @@ | |||||
| 1 | + | package daemon | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "bufio" | |||
| 5 | + | "bytes" | |||
| 6 | + | "encoding/json" | |||
| 7 | + | "errors" | |||
| 8 | + | "log/slog" | |||
| 9 | + | "testing" | |||
| 10 | + | ||||
| 11 | + | "github.com/bnema/vev/internal/domain" | |||
| 12 | + | "github.com/stretchr/testify/require" | |||
| 13 | + | ) | |||
| 14 | + | ||||
| 15 | + | func TestStartupRecoveryCounts(t *testing.T) { | |||
| 16 | + | var buffer bytes.Buffer | |||
| 17 | + | log := slog.New(slog.NewJSONHandler(&buffer, nil)) | |||
| 18 | + | ref := &domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}} | |||
| 19 | + | records := []domain.CatalogueRecord{ | |||
| 20 | + | {Name: "one", IncarnationID: domain.IncarnationID{1}, Committed: ref}, | |||
| 21 | + | {Name: "two", IncarnationID: domain.IncarnationID{2}, Committed: ref}, | |||
| 22 | + | {Name: "new", IncarnationID: domain.IncarnationID{3}}, | |||
| 23 | + | {Name: "broken", IncarnationID: domain.IncarnationID{4}, Committed: ref, DegradedReason: "terminal contents must not appear"}, | |||
| 24 | + | } | |||
| 25 | + | d := New(nil, stubClock{}, log, WithCatalogue(newDurableRecoveryCatalogue(records), records)) | |||
| 26 | + | d.logSessionRestoreComplete(records[0], 7, false) | |||
| 27 | + | d.logSessionDegraded(records[3], "checkpoint-invalid") | |||
| 28 | + | d.logStartupRecoveryCounts(0) | |||
| 29 | + | ||||
| 30 | + | entries := daemonJSONLogs(t, buffer.Bytes()) | |||
| 31 | + | restore := daemonRequireEvent(t, entries, "session_restore_complete") | |||
| 32 | + | require.Equal(t, "one", restore["session"]) | |||
| 33 | + | require.Equal(t, records[0].IncarnationID.String(), restore["incarnation"]) | |||
| 34 | + | require.EqualValues(t, 7, restore["generation"]) | |||
| 35 | + | degraded := daemonRequireEvent(t, entries, "session_degraded") | |||
| 36 | + | require.Equal(t, "checkpoint-invalid", degraded["reason_code"]) | |||
| 37 | + | require.NotContains(t, degraded, "reason") | |||
| 38 | + | startup := daemonRequireEvent(t, entries, "daemon_startup_complete") | |||
| 39 | + | require.EqualValues(t, 2, startup["healthy"]) | |||
| 40 | + | require.EqualValues(t, 1, startup["fresh"]) | |||
| 41 | + | require.EqualValues(t, 0, startup["restoring"]) | |||
| 42 | + | require.EqualValues(t, 1, startup["broken"]) | |||
| 43 | + | } | |||
| 44 | + | ||||
| 45 | + | type failingRecordsCatalogue struct { | |||
| 46 | + | *durableRecoveryCatalogue | |||
| 47 | + | err error | |||
| 48 | + | } | |||
| 49 | + | ||||
| 50 | + | func (c failingRecordsCatalogue) Records() ([]domain.CatalogueRecord, error) { | |||
| 51 | + | return nil, c.err | |||
| 52 | + | } | |||
| 53 | + | ||||
| 54 | + | func TestStartupRecoveryCountsFallsBackWhenCatalogueReadFails(t *testing.T) { | |||
| 55 | + | var buffer bytes.Buffer | |||
| 56 | + | record := domain.CatalogueRecord{Name: "broken", IncarnationID: domain.IncarnationID{4}, Committed: &domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}}, DegradedReason: "invalid"} | |||
| 57 | + | d := New(nil, stubClock{}, slog.New(slog.NewJSONHandler(&buffer, nil))) | |||
| 58 | + | d.persistEnabled = true | |||
| 59 | + | d.catalogue = failingRecordsCatalogue{durableRecoveryCatalogue: newDurableRecoveryCatalogue(nil), err: errors.New("read failed")} | |||
| 60 | + | d.stopped[record.Name] = stoppedSession{record: record} | |||
| 61 | + | ||||
| 62 | + | d.logStartupRecoveryCounts(0) | |||
| 63 | + | ||||
| 64 | + | entries := daemonJSONLogs(t, buffer.Bytes()) | |||
| 65 | + | daemonRequireEvent(t, entries, "daemon_startup_catalogue_read_failed") | |||
| 66 | + | startup := daemonRequireEvent(t, entries, "daemon_startup_complete") | |||
| 67 | + | require.EqualValues(t, 1, startup["broken"]) | |||
| 68 | + | } | |||
| 69 | + | ||||
| 70 | + | func daemonJSONLogs(t *testing.T, data []byte) []map[string]any { | |||
| 71 | + | t.Helper() | |||
| 72 | + | var entries []map[string]any | |||
| 73 | + | scanner := bufio.NewScanner(bytes.NewReader(data)) | |||
| 74 | + | for scanner.Scan() { | |||
| 75 | + | var entry map[string]any | |||
| 76 | + | require.NoError(t, json.Unmarshal(scanner.Bytes(), &entry)) | |||
| 77 | + | entries = append(entries, entry) | |||
| 78 | + | } | |||
| 79 | + | require.NoError(t, scanner.Err()) | |||
| 80 | + | return entries | |||
| 81 | + | } | |||
| 82 | + | ||||
| 83 | + | func daemonRequireEvent(t *testing.T, entries []map[string]any, name string) map[string]any { | |||
| 84 | + | t.Helper() | |||
| 85 | + | for _, entry := range entries { | |||
| 86 | + | if entry["msg"] == name { | |||
| 87 | + | return entry | |||
| 88 | + | } | |||
| 89 | + | } | |||
| 90 | + | t.Fatalf("event %q not found", name) | |||
| 91 | + | return nil | |||
| 92 | + | } | |||
M internal/usecase/daemon/render_capture.go +37 -12
| @@ -17,15 +17,21 @@ import ( | |||||
| 17 | // capture/compose/emit transaction holds that attachment's sendMu, so these | 17 | // capture/compose/emit transaction holds that attachment's sendMu, so these | ||
| 18 | // buffers cannot be observed or reused before emission completes. | 18 | // buffers cannot be observed or reused before emission completes. | ||
| 19 | type renderCaptureScratch struct { | 19 | type renderCaptureScratch struct { | ||
| 20 | - | state capturedRenderState | 20 | + | state capturedRenderState |
| 21 | - | panes []capturedPaneRenderState | 21 | + | panes []capturedPaneRenderState |
| 22 | - | placements []layout.Placement | 22 | + | placements []layout.Placement |
| 23 | - | dividers []layout.Divider | 23 | + | dividers []layout.Divider |
| 24 | - | statusTabs []statusTab | 24 | + | layoutTab *tab |
| 25 | - | mru []recentSession | 25 | + | layoutGeneration uint64 |
| 26 | - | ranked []rankedRecent | 26 | + | layoutArea domain.Rect |
| 27 | - | titleIDs []layout.PaneID | 27 | + | layoutFocus layout.PaneID |
| 28 | - | receipts []damageReceipt | 28 | + | layoutFingerprint string |
| 29 | + | layoutValid bool | |||
| 30 | + | statusTabs []statusTab | |||
| 31 | + | mru []recentSession | |||
| 32 | + | ranked []rankedRecent | |||
| 33 | + | titleIDs []layout.PaneID | |||
| 34 | + | receipts []damageReceipt | |||
| 29 | } | 35 | } | ||
| 30 | 36 | ||||
| 31 | type damageReceipt struct { | 37 | type damageReceipt struct { | ||
| @@ -225,9 +231,28 @@ func capturePrimaryRenderState( | |||||
| 225 | 231 | ||||
| 226 | tb.mu.Lock() | 232 | tb.mu.Lock() | ||
| 227 | defer tb.mu.Unlock() | 233 | defer tb.mu.Unlock() | ||
| 228 | - | layoutSnap := solveTabLayoutLocked(tb) | 234 | + | area := domain.Rect{Width: tb.size.Cols, Height: tb.size.Rows} |
| 229 | - | scratch.placements = append(scratch.placements[:0], layoutSnap.placements...) | 235 | + | var focus layout.PaneID |
| 230 | - | scratch.dividers = append(scratch.dividers[:0], layoutSnap.dividers...) | 236 | + | if tb.tree != nil { |
| 237 | + | focus = tb.tree.Focus | |||
| 238 | + | } | |||
| 239 | + | layoutSnap := tabLayoutSnapshot{ | |||
| 240 | + | area: area, focus: focus, placements: scratch.placements, | |||
| 241 | + | dividers: scratch.dividers, fingerprint: scratch.layoutFingerprint, ok: scratch.layoutValid, | |||
| 242 | + | } | |||
| 243 | + | if scratch.layoutTab != tb || scratch.layoutGeneration != tb.layoutGeneration || scratch.layoutArea != area || scratch.layoutFocus != focus { | |||
| 244 | + | layoutSnap = solveTabLayoutLocked(tb) | |||
| 245 | + | scratch.placements = append(scratch.placements[:0], layoutSnap.placements...) | |||
| 246 | + | scratch.dividers = append(scratch.dividers[:0], layoutSnap.dividers...) | |||
| 247 | + | scratch.layoutTab = tb | |||
| 248 | + | scratch.layoutGeneration = tb.layoutGeneration | |||
| 249 | + | scratch.layoutArea = area | |||
| 250 | + | scratch.layoutFocus = focus | |||
| 251 | + | scratch.layoutFingerprint = layoutSnap.fingerprint | |||
| 252 | + | scratch.layoutValid = layoutSnap.ok | |||
| 253 | + | layoutSnap.placements = scratch.placements | |||
| 254 | + | layoutSnap.dividers = scratch.dividers | |||
| 255 | + | } | |||
| 231 | state := &scratch.state | 256 | state := &scratch.state | ||
| 232 | *state = capturedRenderState{ | 257 | *state = capturedRenderState{ | ||
| 233 | attachment: ac, lease: lease, reset: reset, bars: bars, theme: bars.theme, | 258 | attachment: ac, lease: lease, reset: reset, bars: bars, theme: bars.theme, | ||
M internal/usecase/daemon/session.go +175 -189
| @@ -3,6 +3,7 @@ package daemon | |||||
| 3 | 3 | ||||
| 4 | import ( | 4 | import ( | ||
| 5 | "context" | 5 | "context" | ||
| 6 | + | "crypto/rand" | |||
| 6 | "errors" | 7 | "errors" | ||
| 7 | "fmt" | 8 | "fmt" | ||
| 8 | "os" | 9 | "os" | ||
| @@ -14,7 +15,6 @@ import ( | |||||
| 14 | "time" | 15 | "time" | ||
| 15 | 16 | ||||
| 16 | "github.com/bnema/vev/internal/domain" | 17 | "github.com/bnema/vev/internal/domain" | ||
| 17 | - | "github.com/bnema/vev/internal/persist" | |||
| 18 | "github.com/bnema/vev/internal/ports" | 18 | "github.com/bnema/vev/internal/ports" | ||
| 19 | "github.com/bnema/vev/internal/usecase/layout" | 19 | "github.com/bnema/vev/internal/usecase/layout" | ||
| 20 | ) | 20 | ) | ||
| @@ -46,26 +46,29 @@ type session struct { | |||||
| 46 | // tab, or pane locks. The owner performs teardown without holding teardownMu; | 46 | // tab, or pane locks. The owner performs teardown without holding teardownMu; | ||
| 47 | // duplicate daemon-shutdown callers can therefore bound their ownership wait | 47 | // duplicate daemon-shutdown callers can therefore bound their ownership wait | ||
| 48 | // with Serve's shared snapshot deadline. | 48 | // with Serve's shared snapshot deadline. | ||
| 49 | - | teardownMu sync.Mutex | 49 | + | teardownMu sync.Mutex |
| 50 | - | teardownActive bool | 50 | + | teardownActive bool |
| 51 | - | teardownDone chan struct{} | 51 | + | teardownDone chan struct{} |
| 52 | - | teardownWaiters uint | 52 | + | teardownWaiters uint |
| 53 | - | teardownChanged chan struct{} | 53 | + | teardownChanged chan struct{} |
| 54 | - | lifecycleStopOnce sync.Once | 54 | + | lifecycleStopOnce sync.Once |
| 55 | - | mu sync.Mutex // guards tabs, active, client, clipFiles, and clipboard queue state | 55 | + | mu sync.Mutex // guards tabs, active, client, restoreDone, clipFiles, and clipboard queue state |
| 56 | + | // restoreDone remains attached to a restored session after it is published in | |||
| 57 | + | // the live registry, so racing attaches cannot bypass restoration completion. | |||
| 58 | + | restoreDone chan struct{} | |||
| 56 | themeMu sync.Mutex | 59 | themeMu sync.Mutex | ||
| 57 | tabs []*tab | 60 | tabs []*tab | ||
| 58 | active int | 61 | active int | ||
| 59 | client *attachedClient | 62 | client *attachedClient | ||
| 60 | clipboardQueue []clipboardForward | 63 | clipboardQueue []clipboardForward | ||
| 61 | clipboardWorkerRunning bool | 64 | clipboardWorkerRunning bool | ||
| 62 | - | renameInProgress bool | |||
| 63 | cwd string | 65 | cwd string | ||
| 64 | terminal terminalEnv | 66 | terminal terminalEnv | ||
| 65 | // env is the authoritative immutable environment snapshot for future PTY children. | 67 | // env is the authoritative immutable environment snapshot for future PTY children. | ||
| 66 | // It is guarded by mu and is always copied on ingress and egress. | 68 | // It is guarded by mu and is always copied on ingress and egress. | ||
| 67 | env []string | 69 | env []string | ||
| 68 | createdAt int64 | 70 | createdAt int64 | ||
| 71 | + | incarnation domain.IncarnationID | |||
| 69 | mruAt atomic.Uint64 | 72 | mruAt atomic.Uint64 | ||
| 70 | snapDirty atomic.Bool | 73 | snapDirty atomic.Bool | ||
| 71 | snapEligible atomic.Bool | 74 | snapEligible atomic.Bool | ||
| @@ -75,6 +78,7 @@ type session struct { | |||||
| 75 | snapshotMu sync.Mutex | 78 | snapshotMu sync.Mutex | ||
| 76 | snapshotGeneration uint64 // newest mutation revision | 79 | snapshotGeneration uint64 // newest mutation revision | ||
| 77 | snapshotPublishedGeneration uint64 // newest repository generation | 80 | snapshotPublishedGeneration uint64 // newest repository generation | ||
| 81 | + | snapshotPublishedCheckpoint *domain.CheckpointRef | |||
| 78 | snapshotPublishedMutationRevision uint64 | 82 | snapshotPublishedMutationRevision uint64 | ||
| 79 | snapshotNextEligibleAt time.Time | 83 | snapshotNextEligibleAt time.Time | ||
| 80 | // The coordinator state below is guarded by snapshotMu. A capture can be | 84 | // The coordinator state below is guarded by snapshotMu. A capture can be | ||
| @@ -93,7 +97,6 @@ type session struct { | |||||
| 93 | snapshotPublicationContext context.Context | 97 | snapshotPublicationContext context.Context | ||
| 94 | snapshotPublicationCancel context.CancelFunc | 98 | snapshotPublicationCancel context.CancelFunc | ||
| 95 | snapshotQuarantined bool | 99 | snapshotQuarantined bool | ||
| 96 | - | snapshotQuarantineEpoch uint64 | |||
| 97 | // snapshotChunkCache retains encoded sealed history only for this named | 100 | // snapshotChunkCache retains encoded sealed history only for this named | ||
| 98 | // session. snapshotMu owns it so cache touches never contend with pane | 101 | // session. snapshotMu owns it so cache touches never contend with pane | ||
| 99 | // state or mutate bytes retained by queued/in-flight publications. | 102 | // state or mutate bytes retained by queued/in-flight publications. | ||
| @@ -164,40 +167,57 @@ func (d *Daemon) touchMRU(sess *session) { | |||||
| 164 | return | 167 | return | ||
| 165 | } | 168 | } | ||
| 166 | seq := d.mruSeq.Add(1) | 169 | seq := d.mruSeq.Add(1) | ||
| 167 | - | updated := false | |||
| 168 | for { | 170 | for { | ||
| 169 | old := sess.mruAt.Load() | 171 | old := sess.mruAt.Load() | ||
| 170 | if old >= seq { | 172 | if old >= seq { | ||
| 171 | return | 173 | return | ||
| 172 | } | 174 | } | ||
| 173 | if sess.mruAt.CompareAndSwap(old, seq) { | 175 | if sess.mruAt.CompareAndSwap(old, seq) { | ||
| 174 | - | updated = true | |||
| 175 | break | 176 | break | ||
| 176 | } | 177 | } | ||
| 177 | } | 178 | } | ||
| 178 | - | if !updated { | |||
| 179 | - | return | |||
| 180 | - | } | |||
| 181 | sess.mu.Lock() | 179 | sess.mu.Lock() | ||
| 182 | - | name := sess.name | 180 | + | if !sess.ephemeral { |
| 183 | - | ephemeral := sess.ephemeral | 181 | + | d.markCatalogueDirty(sess.persistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))).MetadataUpdate()) |
| 184 | - | sess.mu.Unlock() | |||
| 185 | - | if !ephemeral && d.persist != nil { | |||
| 186 | - | if err := d.persist.TouchMRU(name, seq); err != nil { | |||
| 187 | - | d.log.Warn("touching persisted session recency failed", "err", err, "session", name) | |||
| 188 | - | } | |||
| 189 | } | 182 | } | ||
| 183 | + | sess.mu.Unlock() | |||
| 190 | } | 184 | } | ||
| 191 | 185 | ||||
| 192 | func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) { | 186 | func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz domain.Size, term terminalEnv, env []string, restoredTabNames ...[]string) (*session, error) { | ||
| 193 | env = copyEnvironment(env) | 187 | env = copyEnvironment(env) | ||
| 188 | + | if _, reserved := d.creating[name]; reserved { | |||
| 189 | + | return nil, errSessionNameInUse | |||
| 190 | + | } | |||
| 191 | + | d.creating[name] = struct{}{} | |||
| 192 | + | defer delete(d.creating, name) | |||
| 194 | stopped, resuming := d.stopped[name] | 193 | stopped, resuming := d.stopped[name] | ||
| 194 | + | var authoritative domain.CatalogueRecord | |||
| 195 | + | var authoritativeExists bool | |||
| 196 | + | if !ephemeral && resuming && d.persistEnabled { | |||
| 197 | + | d.mu.Unlock() | |||
| 198 | + | var err error | |||
| 199 | + | authoritative, authoritativeExists, err = d.catalogueRecord(name) | |||
| 200 | + | d.mu.Lock() | |||
| 201 | + | if err != nil { | |||
| 202 | + | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't read session catalogue", err) | |||
| 203 | + | } | |||
| 204 | + | if d.closing { | |||
| 205 | + | return nil, &protoErr{ports.ErrServerShutdown, "daemon is shutting down"} | |||
| 206 | + | } | |||
| 207 | + | stopped, resuming = d.stopped[name] | |||
| 208 | + | } | |||
| 195 | var createdAt int64 | 209 | var createdAt int64 | ||
| 210 | + | var incarnation domain.IncarnationID | |||
| 196 | if !ephemeral { | 211 | if !ephemeral { | ||
| 197 | if resuming { | 212 | if resuming { | ||
| 198 | // A stopped session is the same lifecycle when resumed; it must retain | 213 | // A stopped session is the same lifecycle when resumed; it must retain | ||
| 199 | // the persisted identity rather than receive a fresh timestamp. | 214 | // the persisted identity rather than receive a fresh timestamp. | ||
| 200 | createdAt = stopped.createdAt | 215 | createdAt = stopped.createdAt | ||
| 216 | + | incarnation = stopped.incarnation | |||
| 217 | + | if authoritativeExists { | |||
| 218 | + | createdAt = authoritative.CreatedAt | |||
| 219 | + | incarnation = authoritative.IncarnationID | |||
| 220 | + | } | |||
| 201 | if createdAt > d.lastAllocatedCreatedAt { | 221 | if createdAt > d.lastAllocatedCreatedAt { | ||
| 202 | d.lastAllocatedCreatedAt = createdAt | 222 | d.lastAllocatedCreatedAt = createdAt | ||
| 203 | } | 223 | } | ||
| @@ -208,6 +228,13 @@ func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz | |||||
| 208 | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", err) | 228 | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", err) | ||
| 209 | } | 229 | } | ||
| 210 | } | 230 | } | ||
| 231 | + | if incarnation == (domain.IncarnationID{}) { | |||
| 232 | + | var err error | |||
| 233 | + | incarnation, err = domain.NewIncarnationID(rand.Reader) | |||
| 234 | + | if err != nil { | |||
| 235 | + | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", fmt.Errorf("generate durable identity: %w", err)) | |||
| 236 | + | } | |||
| 237 | + | } | |||
| 211 | } | 238 | } | ||
| 212 | 239 | ||||
| 213 | tbSize := tabSize(sz) | 240 | tbSize := tabSize(sz) | ||
| @@ -247,6 +274,9 @@ func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz | |||||
| 247 | if !ephemeral && resuming { | 274 | if !ephemeral && resuming { | ||
| 248 | lastUsedSeq = stopped.lastUsedSeq | 275 | lastUsedSeq = stopped.lastUsedSeq | ||
| 249 | } | 276 | } | ||
| 277 | + | if lastUsedSeq == 0 { | |||
| 278 | + | lastUsedSeq = d.mruSeq.Add(1) | |||
| 279 | + | } | |||
| 250 | sess := &session{ | 280 | sess := &session{ | ||
| 251 | id: id, | 281 | id: id, | ||
| 252 | name: name, | 282 | name: name, | ||
| @@ -258,27 +288,41 @@ func (d *Daemon) createSessionLocked(name string, ephemeral bool, cwd string, sz | |||||
| 258 | terminal: term, | 288 | terminal: term, | ||
| 259 | env: env, | 289 | env: env, | ||
| 260 | createdAt: createdAt, | 290 | createdAt: createdAt, | ||
| 291 | + | incarnation: incarnation, | |||
| 261 | snapshotWake: d.snapshotWake, | 292 | snapshotWake: d.snapshotWake, | ||
| 262 | } | 293 | } | ||
| 263 | if !ephemeral && name != "" { | 294 | if !ephemeral && name != "" { | ||
| 264 | sess.snapshotChunkCache = newSnapshotChunkCache(snapshotChunkCacheLimit) | 295 | sess.snapshotChunkCache = newSnapshotChunkCache(snapshotChunkCacheLimit) | ||
| 265 | } | 296 | } | ||
| 266 | - | if lastUsedSeq > 0 { | 297 | + | sess.mruAt.Store(lastUsedSeq) |
| 267 | - | sess.mruAt.Store(lastUsedSeq) | |||
| 268 | - | } | |||
| 269 | sess.snapEligible.Store(!ephemeral && name != "") | 298 | sess.snapEligible.Store(!ephemeral && name != "") | ||
| 270 | if !ephemeral { | 299 | if !ephemeral { | ||
| 271 | - | if err := d.persist.Save(persist.Record{Name: name, Cwd: cwd, CreatedAt: createdAt, UpdatedAt: createdAt, LastUsedSeq: lastUsedSeq, TabNames: names}); err != nil { | 300 | + | if d.persistEnabled { |
| 272 | - | closeTabs(tabs) | 301 | + | record := domain.CatalogueRecord{Name: name, IncarnationID: incarnation, Cwd: cwd, CreatedAt: createdAt, UpdatedAt: createdAt, LastUsedSeq: lastUsedSeq, TabNames: names} |
| 273 | - | cancel() | 302 | + | if d.recovery == nil { |
| 274 | - | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", err) | 303 | + | closeTabs(tabs) |
| 304 | + | cancel() | |||
| 305 | + | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", errors.New("durable session authority is not configured")) | |||
| 306 | + | } | |||
| 307 | + | var err error | |||
| 308 | + | if resuming && authoritativeExists { | |||
| 309 | + | err = d.catalogue.UpdateMetadata(record.MetadataUpdate()) | |||
| 310 | + | if err == nil { | |||
| 311 | + | err = d.catalogue.Sync() | |||
| 312 | + | } | |||
| 313 | + | } else { | |||
| 314 | + | record, err = d.recovery.Create(d.serveCtx, record) | |||
| 315 | + | } | |||
| 316 | + | if err != nil { | |||
| 317 | + | closeTabs(tabs) | |||
| 318 | + | cancel() | |||
| 319 | + | return nil, domain.UserErr(domain.NoticeSessionSpawn, "couldn't create session", err) | |||
| 320 | + | } | |||
| 321 | + | sess.incarnation = record.IncarnationID | |||
| 275 | } | 322 | } | ||
| 276 | delete(d.stopped, name) | 323 | delete(d.stopped, name) | ||
| 277 | } | 324 | } | ||
| 278 | d.sessions[id] = sess | 325 | d.sessions[id] = sess | ||
| 279 | - | if lastUsedSeq == 0 { | |||
| 280 | - | d.touchMRU(sess) | |||
| 281 | - | } | |||
| 282 | d.log.Info("session created", "session", name, "id", id, "ephemeral", ephemeral) | 326 | d.log.Info("session created", "session", name, "id", id, "ephemeral", ephemeral) | ||
| 283 | for i, tb := range tabs { | 327 | for i, tb := range tabs { | ||
| 284 | d.log.Info("tab created", "session", name, "tab", i) | 328 | d.log.Info("tab created", "session", name, "tab", i) | ||
| @@ -406,29 +450,16 @@ func (d *Daemon) createTab(sess *session, sz domain.Size) error { | |||||
| 406 | p.ctx, p.cancel = context.WithCancel(tb.ctx) | 450 | p.ctx, p.cancel = context.WithCancel(tb.ctx) | ||
| 407 | } | 451 | } | ||
| 408 | sess.mu.Lock() | 452 | sess.mu.Lock() | ||
| 409 | - | oldActive := sess.active | |||
| 410 | sess.tabs = append(sess.tabs, tb) | 453 | sess.tabs = append(sess.tabs, tb) | ||
| 411 | sess.active = len(sess.tabs) - 1 | 454 | sess.active = len(sess.tabs) - 1 | ||
| 412 | tabIndex := sess.active | 455 | tabIndex := sess.active | ||
| 413 | - | record := sess.persistRecordLocked(time.Now().UnixNano()) | 456 | + | if !sess.ephemeral { |
| 414 | - | ephemeral := sess.ephemeral | 457 | + | d.markCatalogueDirty(sess.persistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))).MetadataUpdate()) |
| 415 | - | if !ephemeral { | |||
| 416 | - | if err := d.persist.Save(record); err != nil { | |||
| 417 | - | sess.tabs = sess.tabs[:len(sess.tabs)-1] | |||
| 418 | - | sess.active = oldActive | |||
| 419 | - | if tb.cancel != nil { | |||
| 420 | - | tb.cancel() | |||
| 421 | - | } | |||
| 422 | - | sess.mu.Unlock() | |||
| 423 | - | d.mu.Unlock() | |||
| 424 | - | _ = pty.Close() | |||
| 425 | - | return err | |||
| 426 | - | } | |||
| 427 | } | 458 | } | ||
| 428 | sess.mu.Unlock() | 459 | sess.mu.Unlock() | ||
| 460 | + | d.mu.Unlock() | |||
| 429 | d.log.Info("tab created", "session", name, "tab", tabIndex) | 461 | d.log.Info("tab created", "session", name, "tab", tabIndex) | ||
| 430 | d.startTabGoroutines(sess, tb) | 462 | d.startTabGoroutines(sess, tb) | ||
| 431 | - | d.mu.Unlock() | |||
| 432 | d.activateTab(sess, tb) | 463 | d.activateTab(sess, tb) | ||
| 433 | markSnapshotDirty(sess) | 464 | markSnapshotDirty(sess) | ||
| 434 | return nil | 465 | return nil | ||
| @@ -612,21 +643,14 @@ func (d *Daemon) renameSession(sess *session, name string) error { | |||||
| 612 | return err | 643 | return err | ||
| 613 | } | 644 | } | ||
| 614 | 645 | ||||
| 615 | - | // Reserve the rename while state is inspected, then perform persistence and | 646 | + | // Hold daemon then session state across the durable identity write. The |
| 616 | - | // repository I/O without daemon/session locks. The old identity is first | 647 | + | // incarnation-keyed snapshot namespace is never relocated. |
| 617 | - | // quarantined and durably purged; committing the new record before that | |||
| 618 | - | // transaction completes could leave a restorable old legacy source. | |||
| 619 | d.mu.Lock() | 648 | d.mu.Lock() | ||
| 620 | if taken := d.findByNameLocked(name); taken != nil && taken != sess { | 649 | if taken := d.findByNameLocked(name); taken != nil && taken != sess { | ||
| 621 | d.mu.Unlock() | 650 | d.mu.Unlock() | ||
| 622 | return errSessionNameInUse | 651 | return errSessionNameInUse | ||
| 623 | } | 652 | } | ||
| 624 | sess.mu.Lock() | 653 | sess.mu.Lock() | ||
| 625 | - | if sess.renameInProgress { | |||
| 626 | - | sess.mu.Unlock() | |||
| 627 | - | d.mu.Unlock() | |||
| 628 | - | return errors.New("session rename already in progress") | |||
| 629 | - | } | |||
| 630 | if stopped, ok := d.stopped[name]; ok && stopped.name != sess.name { | 654 | if stopped, ok := d.stopped[name]; ok && stopped.name != sess.name { | ||
| 631 | sess.mu.Unlock() | 655 | sess.mu.Unlock() | ||
| 632 | d.mu.Unlock() | 656 | d.mu.Unlock() | ||
| @@ -635,6 +659,7 @@ func (d *Daemon) renameSession(sess *session, name string) error { | |||||
| 635 | oldName := sess.name | 659 | oldName := sess.name | ||
| 636 | wasEphemeral := sess.ephemeral | 660 | wasEphemeral := sess.ephemeral | ||
| 637 | createdAt := sess.createdAt | 661 | createdAt := sess.createdAt | ||
| 662 | + | priorIncarnation := sess.incarnation | |||
| 638 | if wasEphemeral { | 663 | if wasEphemeral { | ||
| 639 | var err error | 664 | var err error | ||
| 640 | createdAt, err = d.allocateLifecycleCreatedAtLocked() | 665 | createdAt, err = d.allocateLifecycleCreatedAtLocked() | ||
| @@ -643,71 +668,54 @@ func (d *Daemon) renameSession(sess *session, name string) error { | |||||
| 643 | d.mu.Unlock() | 668 | d.mu.Unlock() | ||
| 644 | return err | 669 | return err | ||
| 645 | } | 670 | } | ||
| 671 | + | sess.incarnation, err = domain.NewIncarnationID(rand.Reader) | |||
| 672 | + | if err != nil { | |||
| 673 | + | sess.mu.Unlock() | |||
| 674 | + | d.mu.Unlock() | |||
| 675 | + | return fmt.Errorf("generate durable identity: %w", err) | |||
| 676 | + | } | |||
| 646 | } | 677 | } | ||
| 647 | lastUsedSeq := sess.mruAt.Load() | 678 | lastUsedSeq := sess.mruAt.Load() | ||
| 648 | - | record := sess.persistRecordLocked(d.nowUnixNano()) | 679 | + | record := sess.persistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))) |
| 649 | record.Name = name | 680 | record.Name = name | ||
| 650 | record.CreatedAt = createdAt | 681 | record.CreatedAt = createdAt | ||
| 651 | record.LastUsedSeq = lastUsedSeq | 682 | record.LastUsedSeq = lastUsedSeq | ||
| 652 | - | sess.renameInProgress = true | |||
| 653 | - | sess.mu.Unlock() | |||
| 654 | - | d.mu.Unlock() | |||
| 655 | - | ||||
| 656 | rollback := func(err error) error { | 683 | rollback := func(err error) error { | ||
| 657 | - | sess.mu.Lock() | 684 | + | sess.incarnation = priorIncarnation |
| 658 | - | sess.renameInProgress = false | |||
| 659 | sess.mu.Unlock() | 685 | sess.mu.Unlock() | ||
| 686 | + | d.mu.Unlock() | |||
| 660 | return err | 687 | return err | ||
| 661 | } | 688 | } | ||
| 662 | - | var quarantine snapshotCoordinatorQuarantine | 689 | + | // Durable named renames are one atomic catalogue batch. The incarnation-keyed |
| 663 | - | if !wasEphemeral && oldName != name { | 690 | + | // snapshot namespace is intentionally untouched. |
| 664 | - | quarantine = quarantineSnapshotCoordinatorWithEpoch(sess) | 691 | + | if (wasEphemeral || oldName != name) && d.persistEnabled { |
| 665 | - | <-quarantine.done | 692 | + | if d.recovery == nil { |
| 666 | - | if err := d.beginSnapshotPurge(oldName); err != nil { | 693 | + | return rollback(errors.New("durable session authority is not configured")) |
| 667 | - | return rollback(err) | |||
| 668 | } | 694 | } | ||
| 669 | - | // finishSnapshotPurge preserves the tombstone through both source | 695 | + | { |
| 670 | - | // deletions and the old metadata delete, including a retained legacy | 696 | + | var committed domain.CatalogueRecord |
| 671 | - | // import whose verified source delete is still pending. | 697 | + | var err error |
| 672 | - | if err := d.finishSnapshotPurge(oldName); err != nil { | 698 | + | if wasEphemeral { |
| 673 | - | return rollback(err) | 699 | + | committed, err = d.recovery.Create(d.serveCtx, record) |
| 674 | - | } | 700 | + | } else { |
| 675 | - | } | 701 | + | committed, err = d.recovery.Rename(d.serveCtx, oldName, name) |
| 676 | - | // For a named rename this is the durable commit point, deliberately after | 702 | + | } |
| 677 | - | // the old tombstone transaction. Ephemeral promotion has no old durable | 703 | + | if err != nil { |
| 678 | - | // identity and can create its first record directly. | 704 | + | return rollback(err) |
| 679 | - | if wasEphemeral || oldName != name { | 705 | + | } |
| 680 | - | if err := d.persist.Save(record); err != nil { | 706 | + | record = committed |
| 681 | - | return rollback(err) | |||
| 682 | } | 707 | } | ||
| 683 | } | 708 | } | ||
| 684 | 709 | ||||
| 685 | - | d.mu.Lock() | |||
| 686 | - | sess.mu.Lock() | |||
| 687 | - | // The reservation prevents another rename. A closing session is allowed to | |||
| 688 | - | // finish this short commit; killSession subsequently owns its teardown. | |||
| 689 | delete(d.stopped, oldName) | 710 | delete(d.stopped, oldName) | ||
| 690 | delete(d.stopped, name) | 711 | delete(d.stopped, name) | ||
| 691 | sess.name = name | 712 | sess.name = name | ||
| 692 | sess.createdAt = createdAt | 713 | sess.createdAt = createdAt | ||
| 714 | + | sess.incarnation = record.IncarnationID | |||
| 693 | sess.ephemeral = false | 715 | sess.ephemeral = false | ||
| 694 | - | sess.renameInProgress = false | 716 | + | sess.snapEligible.Store(name != "") |
| 695 | - | // Keep the coordinator in quarantine until the new in-memory identity is | |||
| 696 | - | // fully committed. The fresh coordinator below makes the new repository | |||
| 697 | - | // start at generation one. | |||
| 698 | - | if !wasEphemeral && oldName != name { | |||
| 699 | - | sess.snapEligible.Store(false) | |||
| 700 | - | } else { | |||
| 701 | - | sess.snapEligible.Store(name != "") | |||
| 702 | - | } | |||
| 703 | sess.mu.Unlock() | 717 | sess.mu.Unlock() | ||
| 704 | d.mu.Unlock() | 718 | d.mu.Unlock() | ||
| 705 | - | if !wasEphemeral && oldName != name { | |||
| 706 | - | // A concurrent teardown may have quarantined after this rename did. In | |||
| 707 | - | // that case its newer epoch owns the stopped coordinator. | |||
| 708 | - | resumeSnapshotCoordinatorForNewIdentity(sess, quarantine) | |||
| 709 | - | return nil | |||
| 710 | - | } | |||
| 711 | if wasEphemeral { | 719 | if wasEphemeral { | ||
| 712 | sess.snapshotMu.Lock() | 720 | sess.snapshotMu.Lock() | ||
| 713 | sess.snapshotChunkCache = newSnapshotChunkCache(snapshotChunkCacheLimit) | 721 | sess.snapshotChunkCache = newSnapshotChunkCache(snapshotChunkCacheLimit) | ||
| @@ -732,24 +740,16 @@ func (d *Daemon) renameTab(sess *session, tb *tab, name string) error { | |||||
| 732 | d.mu.Unlock() | 740 | d.mu.Unlock() | ||
| 733 | return errors.New("tab not found") | 741 | return errors.New("tab not found") | ||
| 734 | } | 742 | } | ||
| 735 | - | oldName := tb.name | |||
| 736 | tb.name = name | 743 | tb.name = name | ||
| 737 | - | record := sess.persistRecordLocked(time.Now().UnixNano()) | 744 | + | if !sess.ephemeral { |
| 738 | - | ephemeral := sess.ephemeral | 745 | + | d.markCatalogueDirty(sess.persistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))).MetadataUpdate()) |
| 739 | - | if !ephemeral { | |||
| 740 | - | if err := d.persist.Save(record); err != nil { | |||
| 741 | - | tb.name = oldName | |||
| 742 | - | sess.mu.Unlock() | |||
| 743 | - | d.mu.Unlock() | |||
| 744 | - | return err | |||
| 745 | - | } | |||
| 746 | } | 746 | } | ||
| 747 | sess.mu.Unlock() | 747 | sess.mu.Unlock() | ||
| 748 | d.mu.Unlock() | 748 | d.mu.Unlock() | ||
| 749 | return nil | 749 | return nil | ||
| 750 | } | 750 | } | ||
| 751 | 751 | ||||
| 752 | - | func (s *session) persistRecordLocked(updatedAt int64) persist.Record { | 752 | + | func (s *session) persistRecordLocked(updatedAt int64) domain.CatalogueRecord { |
| 753 | createdAt := s.createdAt | 753 | createdAt := s.createdAt | ||
| 754 | tabNames := make([]string, len(s.tabs)) | 754 | tabNames := make([]string, len(s.tabs)) | ||
| 755 | lastCustom := -1 | 755 | lastCustom := -1 | ||
| @@ -764,7 +764,7 @@ func (s *session) persistRecordLocked(updatedAt int64) persist.Record { | |||||
| 764 | } else { | 764 | } else { | ||
| 765 | tabNames = tabNames[:lastCustom+1] | 765 | tabNames = tabNames[:lastCustom+1] | ||
| 766 | } | 766 | } | ||
| 767 | - | return persist.Record{Name: s.name, Cwd: s.cwd, CreatedAt: createdAt, UpdatedAt: updatedAt, LastUsedSeq: s.mruAt.Load(), TabNames: tabNames} | 767 | + | return domain.CatalogueRecord{Name: s.name, IncarnationID: s.incarnation, Cwd: s.cwd, CreatedAt: createdAt, UpdatedAt: updatedAt, LastUsedSeq: s.mruAt.Load(), TabNames: tabNames} |
| 768 | } | 768 | } | ||
| 769 | 769 | ||||
| 770 | func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | 770 | func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | ||
| @@ -813,12 +813,8 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | |||||
| 813 | destination := sess.tabs[sess.active] | 813 | destination := sess.tabs[sess.active] | ||
| 814 | ac := sess.client | 814 | ac := sess.client | ||
| 815 | name := sess.name | 815 | name := sess.name | ||
| 816 | - | record := sess.persistRecordLocked(time.Now().UnixNano()) | 816 | + | if !sess.ephemeral { |
| 817 | - | ephemeral := sess.ephemeral | 817 | + | d.markCatalogueDirty(sess.persistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))).MetadataUpdate()) |
| 818 | - | if !ephemeral { | |||
| 819 | - | if err := d.persist.Save(record); err != nil { | |||
| 820 | - | d.log.Warn("persisting closed tab failed", "err", err, "session", name) | |||
| 821 | - | } | |||
| 822 | } | 818 | } | ||
| 823 | sess.mu.Unlock() | 819 | sess.mu.Unlock() | ||
| 824 | d.mu.Unlock() | 820 | d.mu.Unlock() | ||
| @@ -860,68 +856,58 @@ func (d *Daemon) closeTab(sess *session, tb *tab, repaint bool) error { | |||||
| 860 | // beginSnapshotPurge commits a logical named-session kill before its live | 856 | // beginSnapshotPurge commits a logical named-session kill before its live | ||
| 861 | // identity or persisted metadata is removed. It never runs under daemon or | 857 | // identity or persisted metadata is removed. It never runs under daemon or | ||
| 862 | // session locks. | 858 | // session locks. | ||
| 863 | - | func (d *Daemon) beginSnapshotPurge(name string) error { | 859 | + | func (d *Daemon) beginSnapshotPurge(_ string, _ domain.IncarnationID) error { |
| 864 | - | if d.snapshotRepository == nil || name == "" { | 860 | + | if d.persistEnabled && d.recovery == nil { |
| 865 | - | return nil | 861 | + | return errors.New("durable session authority is not configured") |
| 866 | } | 862 | } | ||
| 867 | - | return d.snapshotRepository.Tombstone(context.Background(), name) | 863 | + | // The lifecycle coordinator performs the complete ordered protocol after |
| 864 | + | // snapshot writers have joined. | |||
| 865 | + | return nil | |||
| 868 | } | 866 | } | ||
| 869 | 867 | ||||
| 870 | - | // finishSnapshotPurge deletes both independently durable snapshot sources, | 868 | + | // finishSnapshotPurge removes catalogue metadata first, then deletes the |
| 871 | - | // then metadata, and only then clears the tombstone. Any failure deliberately | 869 | + | // incarnation directory. Startup garbage collection removes the directory if |
| 872 | - | // leaves the marker in place so startup cannot restore or import the name and | 870 | + | // the second step is interrupted. |
| 873 | - | // a later live/offline kill can retry the idempotent source deletes. | 871 | + | func (d *Daemon) finishSnapshotPurge(ctx context.Context, name string, _ domain.IncarnationID) error { |
| 874 | - | func (d *Daemon) finishSnapshotPurge(name string) error { | 872 | + | if !d.persistEnabled { |
| 875 | - | if d.snapshotRepository == nil { | 873 | + | return nil |
| 876 | - | return d.persist.Delete(name) | |||
| 877 | - | } | |||
| 878 | - | incrementalErr := d.snapshotRepository.Delete(context.Background(), name) | |||
| 879 | - | var legacyErr error | |||
| 880 | - | if d.legacySnapshots != nil { | |||
| 881 | - | legacyErr = d.legacySnapshots.DeleteLegacy(context.Background(), name) | |||
| 882 | } | 874 | } | ||
| 883 | - | if incrementalErr != nil || legacyErr != nil { | 875 | + | if d.recovery == nil { |
| 884 | - | return errors.Join(incrementalErr, legacyErr) | 876 | + | return errors.New("durable session authority is not configured") |
| 885 | } | 877 | } | ||
| 886 | - | if err := d.persist.Delete(name); err != nil { | 878 | + | if ctx == nil { |
| 887 | - | return err | 879 | + | ctx = d.serveCtx |
| 888 | } | 880 | } | ||
| 889 | - | return d.snapshotRepository.DeleteTombstone(context.Background(), name) | 881 | + | purgeCtx, cancel := context.WithTimeout(ctx, snapshotFinalFlushTimeout) |
| 882 | + | defer cancel() | |||
| 883 | + | return d.recovery.Delete(purgeCtx, name) | |||
| 890 | } | 884 | } | ||
| 891 | 885 | ||||
| 892 | // retryStoppedPurge completes a previously closed session's durable purge. | 886 | // retryStoppedPurge completes a previously closed session's durable purge. | ||
| 893 | // It keeps the stopped record hidden until all sources and metadata cross their | 887 | // It keeps the stopped record hidden until all sources and metadata cross their | ||
| 894 | // deletion boundaries. | 888 | // deletion boundaries. | ||
| 895 | func (d *Daemon) retryStoppedPurge(name string) error { | 889 | func (d *Daemon) retryStoppedPurge(name string) error { | ||
| 890 | + | return d.retryStoppedPurgeContext(d.serveCtx, name) | |||
| 891 | + | } | |||
| 892 | + | ||||
| 893 | + | func (d *Daemon) retryStoppedPurgeContext(ctx context.Context, name string) error { | |||
| 896 | d.mu.Lock() | 894 | d.mu.Lock() | ||
| 897 | stopped, ok := d.stopped[name] | 895 | stopped, ok := d.stopped[name] | ||
| 898 | if !ok { | 896 | if !ok { | ||
| 899 | d.mu.Unlock() | 897 | d.mu.Unlock() | ||
| 900 | return nil | 898 | return nil | ||
| 901 | } | 899 | } | ||
| 902 | - | // Without a repository there is no restore/import source to fence, so the | |||
| 903 | - | // stopped record retains the metadata-only deletion behavior. | |||
| 904 | - | if d.snapshotRepository == nil { | |||
| 905 | - | d.mu.Unlock() | |||
| 906 | - | if err := d.persist.Delete(name); err != nil { | |||
| 907 | - | return err | |||
| 908 | - | } | |||
| 909 | - | d.mu.Lock() | |||
| 910 | - | delete(d.stopped, name) | |||
| 911 | - | d.mu.Unlock() | |||
| 912 | - | return nil | |||
| 913 | - | } | |||
| 914 | stopped.purging = true | 900 | stopped.purging = true | ||
| 915 | d.stopped[name] = stopped | 901 | d.stopped[name] = stopped | ||
| 916 | - | d.mu.Unlock() | |||
| 917 | 902 | ||||
| 918 | - | if err := d.beginSnapshotPurge(name); err != nil { | 903 | + | if err := d.beginSnapshotPurge(name, stopped.incarnation); err != nil { |
| 904 | + | d.mu.Unlock() | |||
| 919 | return err | 905 | return err | ||
| 920 | } | 906 | } | ||
| 921 | - | if err := d.finishSnapshotPurge(name); err != nil { | 907 | + | if err := d.finishSnapshotPurge(ctx, name, stopped.incarnation); err != nil { |
| 908 | + | d.mu.Unlock() | |||
| 922 | return err | 909 | return err | ||
| 923 | } | 910 | } | ||
| 924 | - | d.mu.Lock() | |||
| 925 | if current, ok := d.stopped[name]; ok && current.purging { | 911 | if current, ok := d.stopped[name]; ok && current.purging { | ||
| 926 | delete(d.stopped, name) | 912 | delete(d.stopped, name) | ||
| 927 | } | 913 | } | ||
| @@ -1063,13 +1049,12 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu | |||||
| 1063 | sess.mu.Lock() | 1049 | sess.mu.Lock() | ||
| 1064 | isEphemeral := sess.ephemeral | 1050 | isEphemeral := sess.ephemeral | ||
| 1065 | name := sess.name | 1051 | name := sess.name | ||
| 1052 | + | incarnation := sess.incarnation | |||
| 1066 | sess.mu.Unlock() | 1053 | sess.mu.Unlock() | ||
| 1067 | - | // Commit the tombstone before changing live identity or metadata. A | 1054 | + | // Join snapshot publication before changing live identity or deleting the |
| 1068 | - | // repository delete must never race an older publication that could recreate | 1055 | + | // incarnation directory, so an older publication cannot recreate it. |
| 1069 | - | // the just-quarantined name. Do not hold daemon or session locks while | |||
| 1070 | - | // tombstoning or joining: a repository implementation may block on I/O. | |||
| 1071 | if purge && !isEphemeral { | 1056 | if purge && !isEphemeral { | ||
| 1072 | - | if err := d.beginSnapshotPurge(name); err != nil { | 1057 | + | if err := d.beginSnapshotPurge(name, incarnation); err != nil { |
| 1073 | return err | 1058 | return err | ||
| 1074 | } | 1059 | } | ||
| 1075 | if d.snapshotRepository != nil { | 1060 | if d.snapshotRepository != nil { | ||
| @@ -1145,7 +1130,7 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu | |||||
| 1145 | ephemeral := sess.ephemeral | 1130 | ephemeral := sess.ephemeral | ||
| 1146 | sess.mu.Unlock() | 1131 | sess.mu.Unlock() | ||
| 1147 | if !ephemeral { | 1132 | if !ephemeral { | ||
| 1148 | - | stopped := stoppedSession{name: stoppedName, cwd: stoppedCwd, createdAt: createdAt, lastUsedSeq: sess.mruAt.Load(), tabNames: tabNames, purging: purge} | 1133 | + | stopped := stoppedSession{name: stoppedName, cwd: stoppedCwd, createdAt: createdAt, incarnation: incarnation, lastUsedSeq: sess.mruAt.Load(), tabNames: tabNames, purging: purge} |
| 1149 | d.stopped[stoppedName] = stopped | 1134 | d.stopped[stoppedName] = stopped | ||
| 1150 | } | 1135 | } | ||
| 1151 | empty := len(d.sessions) == 0 | 1136 | empty := len(d.sessions) == 0 | ||
| @@ -1201,20 +1186,20 @@ func (d *Daemon) killSessionWithSnapshotDeadline(sess *session, reason uint8, pu | |||||
| 1201 | } | 1186 | } | ||
| 1202 | } | 1187 | } | ||
| 1203 | // The coordinator and all panes are now stopped, so no producer can publish | 1188 | // The coordinator and all panes are now stopped, so no producer can publish | ||
| 1204 | - | // another generation after this destructive source deletion. Keep the | 1189 | + | // another generation after this destructive source deletion. Keep the hidden |
| 1205 | - | // tombstone and hidden stopped record when any source or metadata operation | 1190 | + | // stopped record when deletion fails so a repeated live kill can retry. |
| 1206 | - | // fails; retryStoppedPurge (including a repeated live kill) resumes safely. | |||
| 1207 | if !ephemeral && purge { | 1191 | if !ephemeral && purge { | ||
| 1208 | - | if err := d.finishSnapshotPurge(stoppedName); err != nil { | 1192 | + | d.mu.Lock() |
| 1193 | + | sess.mu.Lock() | |||
| 1194 | + | err := d.finishSnapshotPurge(d.serveCtx, stoppedName, incarnation) | |||
| 1195 | + | sess.mu.Unlock() | |||
| 1196 | + | if err != nil { | |||
| 1209 | purgeErr = errors.Join(purgeErr, err) | 1197 | purgeErr = errors.Join(purgeErr, err) | ||
| 1210 | d.log.Warn("finishing session snapshot purge failed", "err", err, "session", stoppedName) | 1198 | d.log.Warn("finishing session snapshot purge failed", "err", err, "session", stoppedName) | ||
| 1211 | - | } else { | 1199 | + | } else if stopped, ok := d.stopped[stoppedName]; ok && stopped.purging { |
| 1212 | - | d.mu.Lock() | 1200 | + | delete(d.stopped, stoppedName) |
| 1213 | - | if stopped, ok := d.stopped[stoppedName]; ok && stopped.purging { | |||
| 1214 | - | delete(d.stopped, stoppedName) | |||
| 1215 | - | } | |||
| 1216 | - | d.mu.Unlock() | |||
| 1217 | } | 1201 | } | ||
| 1202 | + | d.mu.Unlock() | |||
| 1218 | } | 1203 | } | ||
| 1219 | if empty { | 1204 | if empty { | ||
| 1220 | d.doneOnce.Do(func() { close(d.done) }) | 1205 | d.doneOnce.Do(func() { close(d.done) }) | ||
| @@ -1259,8 +1244,11 @@ func (d *Daemon) nameLiveOrStoppedLocked(name string) bool { | |||||
| 1259 | if d.findByNameLocked(name) != nil { | 1244 | if d.findByNameLocked(name) != nil { | ||
| 1260 | return true | 1245 | return true | ||
| 1261 | } | 1246 | } | ||
| 1262 | - | _, ok := d.stopped[name] | 1247 | + | if _, ok := d.stopped[name]; ok { |
| 1263 | - | return ok | 1248 | + | return true |
| 1249 | + | } | |||
| 1250 | + | _, reserved := d.creating[name] | |||
| 1251 | + | return reserved | |||
| 1264 | } | 1252 | } | ||
| 1265 | 1253 | ||||
| 1266 | func (d *Daemon) cwdSampler(ctx context.Context) { | 1254 | func (d *Daemon) cwdSampler(ctx context.Context) { | ||
| @@ -1272,7 +1260,12 @@ func (d *Daemon) cwdSampler(ctx context.Context) { | |||||
| 1272 | return | 1260 | return | ||
| 1273 | case <-t.C(): | 1261 | case <-t.C(): | ||
| 1274 | } | 1262 | } | ||
| 1275 | - | d.refreshNamedSessionCwds() | 1263 | + | if d.procCwd != nil { |
| 1264 | + | d.refreshNamedSessionCwds() | |||
| 1265 | + | } | |||
| 1266 | + | if err := d.flushCatalogue(); err != nil { | |||
| 1267 | + | d.log.Warn("flushing session catalogue failed", "err", err) | |||
| 1268 | + | } | |||
| 1276 | t.Reset(5 * time.Second) | 1269 | t.Reset(5 * time.Second) | ||
| 1277 | } | 1270 | } | ||
| 1278 | } | 1271 | } | ||
| @@ -1321,16 +1314,12 @@ func (d *Daemon) refreshSessionCwd(sess *session) { | |||||
| 1321 | return | 1314 | return | ||
| 1322 | } | 1315 | } | ||
| 1323 | sess.cwd = cwd | 1316 | sess.cwd = cwd | ||
| 1324 | - | name := sess.name | 1317 | + | if !sess.ephemeral { |
| 1325 | - | ephemeral := sess.ephemeral | 1318 | + | d.markCatalogueDirty(sess.persistRecordLocked(max(d.nowUnixNano(), sess.createdAt, int64(1))).MetadataUpdate()) |
| 1326 | - | sess.mu.Unlock() | |||
| 1327 | - | if !ephemeral { | |||
| 1328 | - | if err := d.persist.Touch(name, cwd, time.Now().UnixNano()); err != nil { | |||
| 1329 | - | d.log.Warn("touching persisted session cwd failed", "err", err, "session", name) | |||
| 1330 | - | } | |||
| 1331 | - | markSnapshotDirty(sess) | |||
| 1332 | } | 1319 | } | ||
| 1320 | + | sess.mu.Unlock() | |||
| 1333 | d.mu.Unlock() | 1321 | d.mu.Unlock() | ||
| 1322 | + | markSnapshotDirty(sess) | |||
| 1334 | } | 1323 | } | ||
| 1335 | 1324 | ||||
| 1336 | // childEnv retains the daemon-environment helper for daemon-local legacy callers. | 1325 | // childEnv retains the daemon-environment helper for daemon-local legacy callers. | ||
| @@ -1344,11 +1333,8 @@ func copyEnvironment(env []string) []string { | |||||
| 1344 | } | 1333 | } | ||
| 1345 | 1334 | ||||
| 1346 | func environmentEntry(entry string) (name, value string, ok bool) { | 1335 | func environmentEntry(entry string) (name, value string, ok bool) { | ||
| 1347 | - | i := strings.IndexByte(entry, '=') | 1336 | + | name, value, ok = strings.Cut(entry, "=") |
| 1348 | - | if i < 0 { | 1337 | + | return name, value, ok |
| 1349 | - | return "", "", false | |||
| 1350 | - | } | |||
| 1351 | - | return entry[:i], entry[i+1:], true | |||
| 1352 | } | 1338 | } | ||
| 1353 | 1339 | ||||
| 1354 | func (d *Daemon) ptyCommand(env []string) (string, []string) { | 1340 | func (d *Daemon) ptyCommand(env []string) (string, []string) { | ||
M internal/usecase/daemon/session_purge_control_test.go +11 -26
| @@ -4,48 +4,33 @@ import ( | |||||
| 4 | "errors" | 4 | "errors" | ||
| 5 | "testing" | 5 | "testing" | ||
| 6 | 6 | ||||
| 7 | - | "github.com/stretchr/testify/mock" | |||
| 8 | "github.com/stretchr/testify/require" | 7 | "github.com/stretchr/testify/require" | ||
| 9 | 8 | ||||
| 10 | - | "github.com/bnema/vev/internal/ports" | 9 | + | "github.com/bnema/vev/internal/domain" |
| 10 | + | "github.com/bnema/vev/internal/persist" | |||
| 11 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | 11 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | ||
| 12 | ) | 12 | ) | ||
| 13 | 13 | ||||
| 14 | - | func TestHandleKillStoppedUsesDurableBothSourcePurge(t *testing.T) { | 14 | + | func TestStoppedPurgeMetadataFailureFencesCatalogue(t *testing.T) { |
| 15 | - | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 16 | - | repository := &retryablePurgeRepository{legacyErr: errors.New("legacy delete failed")} | |||
| 17 | - | WithSnapshotRepository(repository, repository)(d) | |||
| 18 | - | d.stopped["work"] = stoppedSession{name: "work"} | |||
| 19 | - | ||||
| 20 | - | transport := portsmocks.NewMockTransport(t) | |||
| 21 | - | transport.EXPECT().Send(mock.Anything).Return(nil).Once() | |||
| 22 | - | transport.EXPECT().Close().Return(nil).Once() | |||
| 23 | - | d.handleKill(transport, ports.Frame{Type: ports.MsgKill, Payload: ports.MarshalKill(ports.Kill{Name: "work"})}) | |||
| 24 | - | ||||
| 25 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy"}, repository.calls) | |||
| 26 | - | require.True(t, repository.tombstoned["work"]) | |||
| 27 | - | require.True(t, d.stopped["work"].purging) | |||
| 28 | - | } | |||
| 29 | - | ||||
| 30 | - | func TestStoppedPurgeMetadataFailureRemainsFencedForRetry(t *testing.T) { | |||
| 31 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 15 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 32 | repository := &retryablePurgeRepository{} | 16 | repository := &retryablePurgeRepository{} | ||
| 33 | - | WithSnapshotRepository(repository, repository)(d) | 17 | + | WithSnapshotRepository(repository)(d) |
| 34 | store, state := newMockStore(t) | 18 | store, state := newMockStore(t) | ||
| 35 | - | WithStore(store)(d) | 19 | + | WithStore(t, store)(d) |
| 20 | + | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}} | |||
| 21 | + | require.NoError(t, d.catalogue.Create(record)) | |||
| 36 | metadataErr := errors.New("metadata delete failed") | 22 | metadataErr := errors.New("metadata delete failed") | ||
| 37 | state.mu.Lock() | 23 | state.mu.Lock() | ||
| 38 | state.deleteErr = func(string) error { return metadataErr } | 24 | state.deleteErr = func(string) error { return metadataErr } | ||
| 39 | state.mu.Unlock() | 25 | state.mu.Unlock() | ||
| 40 | - | d.stopped["work"] = stoppedSession{name: "work"} | 26 | + | d.stopped["work"] = stoppedSession{name: "work", incarnation: record.IncarnationID, record: record} |
| 41 | 27 | ||||
| 42 | require.ErrorIs(t, d.retryStoppedPurge("work"), metadataErr) | 28 | require.ErrorIs(t, d.retryStoppedPurge("work"), metadataErr) | ||
| 43 | - | require.True(t, repository.tombstoned["work"]) | 29 | + | require.Empty(t, repository.calls, "snapshot deletion must wait for catalogue removal") |
| 44 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy"}, repository.calls) | |||
| 45 | 30 | ||||
| 46 | state.mu.Lock() | 31 | state.mu.Lock() | ||
| 47 | state.deleteErr = nil | 32 | state.deleteErr = nil | ||
| 48 | state.mu.Unlock() | 33 | state.mu.Unlock() | ||
| 49 | - | require.NoError(t, d.retryStoppedPurge("work")) | 34 | + | require.ErrorIs(t, d.retryStoppedPurge("work"), persist.ErrCatalogueDurability) |
| 50 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy", "tombstone", "incremental", "legacy", "clear tombstone"}, repository.calls) | 35 | + | require.Empty(t, repository.calls, "a fenced catalogue must not retry deletion") |
| 51 | } | 36 | } | ||
M internal/usecase/daemon/session_purge_retry_test.go +32 -80
| @@ -13,85 +13,37 @@ import ( | |||||
| 13 | ) | 13 | ) | ||
| 14 | 14 | ||||
| 15 | type retryablePurgeRepository struct { | 15 | type retryablePurgeRepository struct { | ||
| 16 | - | incrementalErr error | 16 | + | noOpSnapshotRepository |
| 17 | - | legacyErr error | 17 | + | deleteErr error |
| 18 | - | tombstoned map[string]bool | |||
| 19 | calls []string | 18 | calls []string | ||
| 20 | - | } | 19 | + | deadlineScoped []bool |
| 21 | - | 20 | + | } | |
| 22 | - | func (r *retryablePurgeRepository) Publish(context.Context, ports.SnapshotPublication) error { | 21 | + | |
| 23 | - | return nil | 22 | + | func (r *retryablePurgeRepository) DeleteIncarnation(ctx context.Context, _ domain.IncarnationID) error { |
| 24 | - | } | 23 | + | _, hasDeadline := ctx.Deadline() |
| 25 | - | func (r *retryablePurgeRepository) List(context.Context) ([]string, error) { return nil, nil } | 24 | + | r.deadlineScoped = append(r.deadlineScoped, hasDeadline) |
| 26 | - | func (r *retryablePurgeRepository) Load(context.Context, string) (ports.SnapshotGeneration, error) { | 25 | + | r.calls = append(r.calls, "delete incarnation") |
| 27 | - | return ports.SnapshotGeneration{}, errors.New("unused") | 26 | + | return r.deleteErr |
| 28 | - | } | 27 | + | } |
| 29 | - | func (r *retryablePurgeRepository) Maintain(context.Context) error { return nil } | 28 | + | |
| 30 | - | func (r *retryablePurgeRepository) Delete(_ context.Context, name string) error { | 29 | + | func TestLivePurgeLeavesFailedDirectoryDeletionForStartupGarbageCollection(t *testing.T) { |
| 31 | - | r.calls = append(r.calls, "incremental") | 30 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) |
| 32 | - | return r.incrementalErr | 31 | + | repository := &retryablePurgeRepository{deleteErr: errors.New("delete failed")} |
| 33 | - | } | 32 | + | WithSnapshotRepository(repository)(d) |
| 34 | - | func (r *retryablePurgeRepository) LoadLegacy(context.Context) ([]ports.LegacySnapshot, error) { | 33 | + | store, _ := newMockStore(t) |
| 35 | - | return nil, nil | 34 | + | WithStore(t, store)(d) |
| 36 | - | } | 35 | + | sess := newSnapshotTestSession(t, "work", false, "/work") |
| 37 | - | func (r *retryablePurgeRepository) DeleteVerifiedLegacy(ctx context.Context, blob ports.LegacySnapshot) error { | 36 | + | record := sess.persistRecordLocked(1) |
| 38 | - | return r.DeleteLegacy(ctx, blob.Name) | 37 | + | require.NoError(t, d.catalogue.Create(record)) |
| 39 | - | } | 38 | + | pty, ok := sess.tabs[0].panes["pane-1"].pty.(*portsmocks.MockPTY) |
| 40 | - | func (r *retryablePurgeRepository) DeleteLegacy(_ context.Context, name string) error { | 39 | + | require.True(t, ok) |
| 41 | - | r.calls = append(r.calls, "legacy") | 40 | + | pty.EXPECT().Close().Return(nil).Once() |
| 42 | - | return r.legacyErr | 41 | + | d.sessions = map[domain.SessionID]*session{sess.id: sess} |
| 43 | - | } | 42 | + | |
| 44 | - | func (r *retryablePurgeRepository) Tombstone(_ context.Context, name string) error { | 43 | + | require.Error(t, d.killSession(sess, ports.ReasonSessionKilled, true)) |
| 45 | - | if r.tombstoned == nil { | 44 | + | require.Equal(t, []string{"delete incarnation"}, repository.calls) |
| 46 | - | r.tombstoned = make(map[string]bool) | 45 | + | _, exists, err := d.catalogue.Record("work") |
| 47 | - | } | 46 | + | require.NoError(t, err) |
| 48 | - | r.calls = append(r.calls, "tombstone") | 47 | + | require.False(t, exists, "catalogue removal commits before best-effort directory deletion") |
| 49 | - | r.tombstoned[name] = true | 48 | + | require.Equal(t, []bool{true}, repository.deadlineScoped) |
| 50 | - | return nil | |||
| 51 | - | } | |||
| 52 | - | func (r *retryablePurgeRepository) DeleteTombstone(_ context.Context, name string) error { | |||
| 53 | - | r.calls = append(r.calls, "clear tombstone") | |||
| 54 | - | delete(r.tombstoned, name) | |||
| 55 | - | return nil | |||
| 56 | - | } | |||
| 57 | - | ||||
| 58 | - | func TestLivePurgeRetainsTombstoneAcrossPartialSourceDeletion(t *testing.T) { | |||
| 59 | - | for _, tt := range []struct { | |||
| 60 | - | name string | |||
| 61 | - | incremental error | |||
| 62 | - | legacy error | |||
| 63 | - | }{ | |||
| 64 | - | {name: "incremental failure", incremental: errors.New("incremental failed")}, | |||
| 65 | - | {name: "legacy failure", legacy: errors.New("legacy failed")}, | |||
| 66 | - | } { | |||
| 67 | - | t.Run(tt.name, func(t *testing.T) { | |||
| 68 | - | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 69 | - | repository := &retryablePurgeRepository{incrementalErr: tt.incremental, legacyErr: tt.legacy} | |||
| 70 | - | WithSnapshotRepository(repository, repository)(d) | |||
| 71 | - | sess := newSnapshotTestSession(t, "work", false, "/work") | |||
| 72 | - | pty, ok := sess.tabs[0].panes["pane-1"].pty.(*portsmocks.MockPTY) | |||
| 73 | - | require.True(t, ok, "snapshot test pane must use MockPTY") | |||
| 74 | - | pty.EXPECT().Close().Return(nil).Once() | |||
| 75 | - | d.sessions = map[domain.SessionID]*session{sess.id: sess} | |||
| 76 | - | ||||
| 77 | - | require.Error(t, d.killSession(sess, ports.ReasonSessionKilled, true)) | |||
| 78 | - | require.True(t, repository.tombstoned["work"], "partial purge must fence startup restore/import") | |||
| 79 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy"}, repository.calls) | |||
| 80 | - | d.mu.Lock() | |||
| 81 | - | stopped, ok := d.stopped["work"] | |||
| 82 | - | d.mu.Unlock() | |||
| 83 | - | require.True(t, ok) | |||
| 84 | - | require.True(t, stopped.purging) | |||
| 85 | - | ||||
| 86 | - | repository.incrementalErr = nil | |||
| 87 | - | repository.legacyErr = nil | |||
| 88 | - | require.NoError(t, d.retryStoppedPurge("work")) | |||
| 89 | - | require.False(t, repository.tombstoned["work"]) | |||
| 90 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy", "tombstone", "incremental", "legacy", "clear tombstone"}, repository.calls) | |||
| 91 | - | d.mu.Lock() | |||
| 92 | - | _, ok = d.stopped["work"] | |||
| 93 | - | d.mu.Unlock() | |||
| 94 | - | require.False(t, ok, "metadata is removed only after both source deletes and tombstone clear") | |||
| 95 | - | }) | |||
| 96 | - | } | |||
| 97 | } | 49 | } | ||
M internal/usecase/daemon/session_rename_purge_test.go +9 -24
| @@ -1,7 +1,6 @@ | |||||
| 1 | package daemon | 1 | package daemon | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | - | "errors" | |||
| 5 | "testing" | 4 | "testing" | ||
| 6 | 5 | ||||
| 7 | "github.com/stretchr/testify/require" | 6 | "github.com/stretchr/testify/require" | ||
| @@ -10,36 +9,22 @@ import ( | |||||
| 10 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | 9 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | ||
| 11 | ) | 10 | ) | ||
| 12 | 11 | ||||
| 13 | - | func TestRenamePurgesOldIncrementalAndLegacyBeforeCommittingNewIdentity(t *testing.T) { | 12 | + | func TestRenamePreservesIncarnationSnapshotSources(t *testing.T) { |
| 14 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 13 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 15 | repository := &retryablePurgeRepository{} | 14 | repository := &retryablePurgeRepository{} | ||
| 16 | - | WithSnapshotRepository(repository, repository)(d) | 15 | + | WithSnapshotRepository(repository)(d) |
| 17 | store, state := newMockStore(t) | 16 | store, state := newMockStore(t) | ||
| 18 | - | WithStore(store)(d) | 17 | + | WithStore(t, store)(d) |
| 19 | sess := newSnapshotTestSession(t, "old", false, "/work") | 18 | sess := newSnapshotTestSession(t, "old", false, "/work") | ||
| 20 | d.sessions = map[domain.SessionID]*session{sess.id: sess} | 19 | d.sessions = map[domain.SessionID]*session{sess.id: sess} | ||
| 21 | - | require.NoError(t, d.persist.Save(sess.persistRecordLocked(1))) | 20 | + | sess.mu.Lock() |
| 21 | + | record := sess.persistRecordLocked(1) | |||
| 22 | + | sess.mu.Unlock() | |||
| 23 | + | require.NoError(t, testPersister(t, d).Save(record)) | |||
| 22 | 24 | ||||
| 23 | require.NoError(t, d.renameSession(sess, "new")) | 25 | require.NoError(t, d.renameSession(sess, "new")) | ||
| 24 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy", "clear tombstone"}, repository.calls) | 26 | + | require.Empty(t, repository.calls, "rename must not delete incarnation-keyed snapshots") |
| 25 | require.False(t, state.has("old")) | 27 | require.False(t, state.has("old")) | ||
| 26 | require.True(t, state.has("new")) | 28 | require.True(t, state.has("new")) | ||
| 27 | - | require.False(t, repository.tombstoned["old"]) | 29 | + | require.Equal(t, sess.incarnation, state.record(t, "new").IncarnationID) |
| 28 | - | } | |||
| 29 | - | ||||
| 30 | - | func TestRenameLegacyFailureLeavesOldIdentityFenced(t *testing.T) { | |||
| 31 | - | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 32 | - | repository := &retryablePurgeRepository{legacyErr: errors.New("legacy delete failed")} | |||
| 33 | - | WithSnapshotRepository(repository, repository)(d) | |||
| 34 | - | store, state := newMockStore(t) | |||
| 35 | - | WithStore(store)(d) | |||
| 36 | - | sess := newSnapshotTestSession(t, "old", false, "/work") | |||
| 37 | - | d.sessions = map[domain.SessionID]*session{sess.id: sess} | |||
| 38 | - | require.NoError(t, d.persist.Save(sess.persistRecordLocked(1))) | |||
| 39 | - | ||||
| 40 | - | require.Error(t, d.renameSession(sess, "new")) | |||
| 41 | - | require.Equal(t, []string{"tombstone", "incremental", "legacy"}, repository.calls) | |||
| 42 | - | require.True(t, repository.tombstoned["old"]) | |||
| 43 | - | require.True(t, state.has("old")) | |||
| 44 | - | require.False(t, state.has("new")) | |||
| 45 | } | 30 | } | ||
M internal/usecase/daemon/session_test.go +160 -63
| @@ -772,8 +772,14 @@ func (s newStaticStore) Get(key []byte) ([]byte, bool) { | |||||
| 772 | v, ok := s[string(key)] | 772 | v, ok := s[string(key)] | ||
| 773 | return append([]byte(nil), v...), ok | 773 | return append([]byte(nil), v...), ok | ||
| 774 | } | 774 | } | ||
| 775 | - | func (s newStaticStore) Set(_, _ []byte) error { return nil } | 775 | + | func (s newStaticStore) Set(key, value []byte) error { |
| 776 | - | func (s newStaticStore) Delete(_ []byte) error { return nil } | 776 | + | s[string(key)] = append([]byte(nil), value...) |
| 777 | + | return nil | |||
| 778 | + | } | |||
| 779 | + | func (s newStaticStore) Delete(key []byte) error { | |||
| 780 | + | delete(s, string(key)) | |||
| 781 | + | return nil | |||
| 782 | + | } | |||
| 777 | func (s newStaticStore) Range(fn func(k, v []byte) bool) { | 783 | func (s newStaticStore) Range(fn func(k, v []byte) bool) { | ||
| 778 | for k, v := range s { | 784 | for k, v := range s { | ||
| 779 | if !fn([]byte(k), append([]byte(nil), v...)) { | 785 | if !fn([]byte(k), append([]byte(nil), v...)) { | ||
| @@ -781,24 +787,59 @@ func (s newStaticStore) Range(fn func(k, v []byte) bool) { | |||||
| 781 | } | 787 | } | ||
| 782 | } | 788 | } | ||
| 783 | } | 789 | } | ||
| 784 | - | func (s newStaticStore) Sync() error { return nil } | 790 | + | func (s newStaticStore) Sync() error { return nil } |
| 785 | - | func (s newStaticStore) Close() error { return nil } | 791 | + | func (s newStaticStore) CloseWithoutSync() error { return nil } |
| 792 | + | func (s newStaticStore) Close() error { return nil } | |||
| 793 | + | ||||
| 794 | + | func TestNamedSessionCreationIsAllowedWithNilStore(t *testing.T) { | |||
| 795 | + | pty, release := newBlockingPTY(t) | |||
| 796 | + | defer release() | |||
| 797 | + | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | |||
| 798 | + | WithStore(t, nil)(d) | |||
| 799 | + | ||||
| 800 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |||
| 801 | + | require.NoError(t, err) | |||
| 802 | + | require.Equal(t, "work", sess.name) | |||
| 803 | + | require.False(t, d.persistEnabled) | |||
| 804 | + | } | |||
| 786 | 805 | ||||
| 787 | func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) { | 806 | func TestDaemonLoadsPersistedSessionsAsStopped(t *testing.T) { | ||
| 788 | store, _ := newMockStore(t) | 807 | store, _ := newMockStore(t) | ||
| 789 | - | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | 808 | + | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store)) |
| 790 | - | require.NoError(t, seed.persist.Save(persist.Record{Name: "work", Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9})) | 809 | + | incarnation := domain.IncarnationID{1} |
| 810 | + | require.NoError(t, testPersister(t, seed).Save(persist.Record{Name: "work", IncarnationID: incarnation, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, LastUsedSeq: 9})) | |||
| 791 | 811 | ||||
| 792 | - | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | 812 | + | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store)) |
| 793 | d.mu.Lock() | 813 | d.mu.Lock() | ||
| 794 | stopped := d.stopped["work"] | 814 | stopped := d.stopped["work"] | ||
| 795 | d.mu.Unlock() | 815 | d.mu.Unlock() | ||
| 796 | - | require.Equal(t, stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, lastUsedSeq: 9}, stopped) | 816 | + | require.Equal(t, "work", stopped.name) |
| 817 | + | require.Equal(t, "/tmp/work", stopped.cwd) | |||
| 818 | + | require.Equal(t, int64(7), stopped.createdAt) | |||
| 819 | + | require.Equal(t, incarnation, stopped.incarnation) | |||
| 820 | + | require.Equal(t, uint64(9), stopped.lastUsedSeq) | |||
| 821 | + | require.Equal(t, ports.SessionStopped, stopped.state) | |||
| 822 | + | require.NotNil(t, stopped.restoreDone) | |||
| 797 | require.Equal(t, uint64(9), d.mruSeq.Load()) | 823 | require.Equal(t, uint64(9), d.mruSeq.Load()) | ||
| 798 | } | 824 | } | ||
| 799 | 825 | ||||
| 826 | + | // TestDaemonNewWithoutPersistenceLogsNoLoadWarning covers the default, | |||
| 827 | + | // persistence-free construction path (no WithStore/WithCatalogue option): | |||
| 828 | + | // testPersister(t, d) is the persist.New(nil) sentinel, so LoadAll now returns | |||
| 829 | + | // errPersistenceUnavailable instead of an empty slice. That must not surface | |||
| 830 | + | // as a "loading persisted sessions failed" warning, since nothing failed — | |||
| 831 | + | // persistence simply was never configured. | |||
| 832 | + | func TestDaemonNewWithoutPersistenceLogsNoLoadWarning(t *testing.T) { | |||
| 833 | + | var logBuf strings.Builder | |||
| 834 | + | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(&logBuf, nil))) | |||
| 835 | + | require.NotNil(t, d) | |||
| 836 | + | require.False(t, d.persistEnabled) | |||
| 837 | + | require.Empty(t, d.stopped) | |||
| 838 | + | require.NotContains(t, logBuf.String(), "loading persisted sessions failed") | |||
| 839 | + | } | |||
| 840 | + | ||||
| 800 | func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) { | 841 | func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) { | ||
| 801 | - | d := &Daemon{} | 842 | + | d := &Daemon{clock: stubClock{}} |
| 802 | sess := &session{} | 843 | sess := &session{} | ||
| 803 | const goroutines = 16 | 844 | const goroutines = 16 | ||
| 804 | const iterations = 1000 | 845 | const iterations = 1000 | ||
| @@ -845,9 +886,9 @@ func TestTouchMRUConcurrentUpdatesRemainMonotonic(t *testing.T) { | |||||
| 845 | func TestTouchMRUPersistsNamedButNotEphemeral(t *testing.T) { | 886 | func TestTouchMRUPersistsNamedButNotEphemeral(t *testing.T) { | ||
| 846 | store, state := newMockStore(t) | 887 | store, state := newMockStore(t) | ||
| 847 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 888 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 848 | - | WithStore(store)(d) | 889 | + | WithStore(t, store)(d) |
| 849 | - | named := &session{name: "work", tabs: []*tab{{}}, createdAt: 1} | 890 | + | named := &session{name: "work", tabs: []*tab{{}}, createdAt: 1, incarnation: domain.IncarnationID{1}} |
| 850 | - | require.NoError(t, d.persist.Save(persist.Record{Name: "work", Cwd: "/work", CreatedAt: 1, UpdatedAt: 1})) | 891 | + | require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "work", IncarnationID: named.incarnation, Cwd: "/work", CreatedAt: 1, UpdatedAt: 1})) |
| 851 | 892 | ||||
| 852 | d.touchMRU(named) | 893 | d.touchMRU(named) | ||
| 853 | require.Equal(t, named.mruAt.Load(), state.record(t, "work").LastUsedSeq) | 894 | require.Equal(t, named.mruAt.Load(), state.record(t, "work").LastUsedSeq) | ||
| @@ -865,11 +906,11 @@ func TestCreateSessionSeedsMRUFromStopped(t *testing.T) { | |||||
| 865 | defer release() | 906 | defer release() | ||
| 866 | store, state := newMockStore(t) | 907 | store, state := newMockStore(t) | ||
| 867 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 908 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 868 | - | WithStore(store)(d) | 909 | + | WithStore(t, store)(d) |
| 869 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 1, lastUsedSeq: 42} | 910 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 1, lastUsedSeq: 42} | ||
| 870 | d.mruSeq.Store(42) | 911 | d.mruSeq.Store(42) | ||
| 871 | 912 | ||||
| 872 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 913 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 873 | require.NoError(t, err) | 914 | require.NoError(t, err) | ||
| 874 | require.Equal(t, uint64(42), sess.mruAt.Load()) | 915 | require.Equal(t, uint64(42), sess.mruAt.Load()) | ||
| 875 | require.Equal(t, uint64(42), state.record(t, "work").LastUsedSeq) | 916 | require.Equal(t, uint64(42), state.record(t, "work").LastUsedSeq) | ||
| @@ -883,11 +924,14 @@ func TestCreateRenameKillPersistenceLifecycle(t *testing.T) { | |||||
| 883 | defer release2() | 924 | defer release2() | ||
| 884 | store, state := newMockStore(t) | 925 | store, state := newMockStore(t) | ||
| 885 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | 926 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | ||
| 886 | - | WithStore(store)(d) | 927 | + | WithStore(t, store)(d) |
| 887 | 928 | ||||
| 888 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 929 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 889 | require.NoError(t, err) | 930 | require.NoError(t, err) | ||
| 890 | require.True(t, state.has("work")) | 931 | require.True(t, state.has("work")) | ||
| 932 | + | created := state.record(t, "work") | |||
| 933 | + | require.NotEqual(t, domain.IncarnationID{}, created.IncarnationID) | |||
| 934 | + | require.NoError(t, created.Validate()) | |||
| 891 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | 935 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | ||
| 892 | require.NoError(t, d.createTab(sess, sz)) | 936 | require.NoError(t, d.createTab(sess, sz)) | ||
| 893 | require.NoError(t, d.renameTab(sess, sess.tabs[1], "logs")) | 937 | require.NoError(t, d.renameTab(sess, sess.tabs[1], "logs")) | ||
| @@ -895,7 +939,10 @@ func TestCreateRenameKillPersistenceLifecycle(t *testing.T) { | |||||
| 895 | require.NoError(t, d.renameSession(sess, "renamed")) | 939 | require.NoError(t, d.renameSession(sess, "renamed")) | ||
| 896 | require.False(t, state.has("work")) | 940 | require.False(t, state.has("work")) | ||
| 897 | require.True(t, state.has("renamed")) | 941 | require.True(t, state.has("renamed")) | ||
| 898 | - | require.Equal(t, []string{"shell", "logs"}, state.record(t, "renamed").TabNames) | 942 | + | renamed := state.record(t, "renamed") |
| 943 | + | require.Equal(t, []string{"shell", "logs"}, renamed.TabNames) | |||
| 944 | + | require.Equal(t, created.IncarnationID, renamed.IncarnationID) | |||
| 945 | + | require.NoError(t, renamed.Validate()) | |||
| 899 | 946 | ||||
| 900 | require.NoError(t, d.killSession(sess, ports.ReasonSessionKilled, true)) | 947 | require.NoError(t, d.killSession(sess, ports.ReasonSessionKilled, true)) | ||
| 901 | require.False(t, state.has("renamed")) | 948 | require.False(t, state.has("renamed")) | ||
| @@ -907,13 +954,13 @@ func TestRenameTabPersistsForNamedSession(t *testing.T) { | |||||
| 907 | defer release() | 954 | defer release() | ||
| 908 | store, _ := newMockStore(t) | 955 | store, _ := newMockStore(t) | ||
| 909 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 956 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 910 | - | WithStore(store)(d) | 957 | + | WithStore(t, store)(d) |
| 911 | 958 | ||||
| 912 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 959 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 913 | require.NoError(t, err) | 960 | require.NoError(t, err) | ||
| 914 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | 961 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | ||
| 915 | 962 | ||||
| 916 | - | records, err := d.persist.LoadAll() | 963 | + | records, err := testPersister(t, d).LoadAll() |
| 917 | require.NoError(t, err) | 964 | require.NoError(t, err) | ||
| 918 | require.Len(t, records, 1) | 965 | require.Len(t, records, 1) | ||
| 919 | require.Equal(t, "work", records[0].Name) | 966 | require.Equal(t, "work", records[0].Name) | ||
| @@ -923,6 +970,21 @@ func TestRenameTabPersistsForNamedSession(t *testing.T) { | |||||
| 923 | require.Equal(t, []string{"shell"}, records[0].TabNames) | 970 | require.Equal(t, []string{"shell"}, records[0].TabNames) | ||
| 924 | } | 971 | } | ||
| 925 | 972 | ||||
| 973 | + | func TestRenameTabBuffersMetadataUnderStateLocks(t *testing.T) { | |||
| 974 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 975 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |||
| 976 | + | d.sessions[sess.id] = sess | |||
| 977 | + | catalogue := portsmocks.NewMockCatalogue(t) | |||
| 978 | + | WithCatalogue(catalogue, nil)(d) | |||
| 979 | + | catalogue.EXPECT().UpdateMetadata(mock.Anything).RunAndReturn(func(domain.CatalogueMetadataUpdate) error { | |||
| 980 | + | require.False(t, d.mu.TryLock(), "daemon mutex must guard the metadata snapshot") | |||
| 981 | + | require.False(t, sess.mu.TryLock(), "session mutex must guard the metadata snapshot") | |||
| 982 | + | return nil | |||
| 983 | + | }).Once() | |||
| 984 | + | ||||
| 985 | + | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | |||
| 986 | + | } | |||
| 987 | + | ||||
| 926 | func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) { | 988 | func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) { | ||
| 927 | tests := []struct { | 989 | tests := []struct { | ||
| 928 | name string | 990 | name string | ||
| @@ -943,10 +1005,10 @@ func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) { | |||||
| 943 | } | 1005 | } | ||
| 944 | store, _ := newMockStore(t) | 1006 | store, _ := newMockStore(t) | ||
| 945 | d := newTestDaemon(t, newFactorySeq(t, ptys...), stubClock{}) | 1007 | d := newTestDaemon(t, newFactorySeq(t, ptys...), stubClock{}) | ||
| 946 | - | WithStore(store)(d) | 1008 | + | WithStore(t, store)(d) |
| 947 | sz := domain.Size{Cols: 80, Rows: 24} | 1009 | sz := domain.Size{Cols: 80, Rows: 24} | ||
| 948 | 1010 | ||||
| 949 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 1011 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 950 | require.NoError(t, err) | 1012 | require.NoError(t, err) | ||
| 951 | require.NoError(t, d.createTab(sess, sz)) | 1013 | require.NoError(t, d.createTab(sess, sz)) | ||
| 952 | require.NoError(t, d.createTab(sess, sz)) | 1014 | require.NoError(t, d.createTab(sess, sz)) | ||
| @@ -955,7 +1017,7 @@ func TestTabNamePersistenceTracksTabIndexShifts(t *testing.T) { | |||||
| 955 | } | 1017 | } | ||
| 956 | 1018 | ||||
| 957 | require.NoError(t, d.closeTab(sess, sess.tabs[tt.closeIndex], false)) | 1019 | require.NoError(t, d.closeTab(sess, sess.tabs[tt.closeIndex], false)) | ||
| 958 | - | records, err := d.persist.LoadAll() | 1020 | + | records, err := testPersister(t, d).LoadAll() |
| 959 | require.NoError(t, err) | 1021 | require.NoError(t, err) | ||
| 960 | require.Len(t, records, 1) | 1022 | require.Len(t, records, 1) | ||
| 961 | require.Equal(t, tt.want, records[0].TabNames) | 1023 | require.Equal(t, tt.want, records[0].TabNames) | ||
| @@ -992,15 +1054,15 @@ func TestRenameTabDoesNotPersistForEphemeralSession(t *testing.T) { | |||||
| 992 | defer release() | 1054 | defer release() | ||
| 993 | store, state := newMockStore(t) | 1055 | store, state := newMockStore(t) | ||
| 994 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1056 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 995 | - | WithStore(store)(d) | 1057 | + | WithStore(t, store)(d) |
| 996 | 1058 | ||||
| 997 | - | sess, err := d.createSessionLocked("0", true, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 1059 | + | sess, err := createSessionForTest(d, "0", true, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 998 | require.NoError(t, err) | 1060 | require.NoError(t, err) | ||
| 999 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | 1061 | require.NoError(t, d.renameTab(sess, sess.tabs[0], "shell")) | ||
| 1000 | require.Equal(t, "shell", sess.tabs[0].name) | 1062 | require.Equal(t, "shell", sess.tabs[0].name) | ||
| 1001 | 1063 | ||||
| 1002 | require.False(t, state.has("0")) | 1064 | require.False(t, state.has("0")) | ||
| 1003 | - | records, err := d.persist.LoadAll() | 1065 | + | records, err := testPersister(t, d).LoadAll() |
| 1004 | require.NoError(t, err) | 1066 | require.NoError(t, err) | ||
| 1005 | require.Empty(t, records) | 1067 | require.Empty(t, records) | ||
| 1006 | } | 1068 | } | ||
| @@ -1013,8 +1075,8 @@ func TestAttachRestoresPersistedTabNames(t *testing.T) { | |||||
| 1013 | defer release2() | 1075 | defer release2() | ||
| 1014 | store, _ := newMockStore(t) | 1076 | store, _ := newMockStore(t) | ||
| 1015 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | 1077 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | ||
| 1016 | - | WithStore(store)(d) | 1078 | + | WithStore(t, store)(d) |
| 1017 | - | require.NoError(t, d.persist.Save(persist.Record{Name: "work", Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}})) | 1079 | + | require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "work", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/work", CreatedAt: 7, UpdatedAt: 8, TabNames: []string{"shell", "logs"}})) |
| 1018 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}} | 1080 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/tmp/work", createdAt: 7, tabNames: []string{"shell", "logs"}} | ||
| 1019 | tr := portsmocks.NewMockTransport(t) | 1081 | tr := portsmocks.NewMockTransport(t) | ||
| 1020 | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | 1082 | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | ||
| @@ -1034,10 +1096,10 @@ func TestEphemeralRenamePromotesAndStoppedCollisionRejected(t *testing.T) { | |||||
| 1034 | defer release() | 1096 | defer release() | ||
| 1035 | store, state := newMockStore(t) | 1097 | store, state := newMockStore(t) | ||
| 1036 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1098 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1037 | - | WithStore(store)(d) | 1099 | + | WithStore(t, store)(d) |
| 1038 | d.stopped["taken"] = stoppedSession{name: "taken", cwd: "/tmp", createdAt: 1} | 1100 | d.stopped["taken"] = stoppedSession{name: "taken", cwd: "/tmp", createdAt: 1} | ||
| 1039 | 1101 | ||||
| 1040 | - | sess, err := d.createSessionLocked("0", true, "/tmp/e", sz, terminalEnv{}, d.baseEnv) | 1102 | + | sess, err := createSessionForTest(d, "0", true, "/tmp/e", sz, terminalEnv{}, d.baseEnv) |
| 1041 | require.NoError(t, err) | 1103 | require.NoError(t, err) | ||
| 1042 | require.False(t, state.has("0")) | 1104 | require.False(t, state.has("0")) | ||
| 1043 | require.EqualError(t, d.renameSession(sess, "taken"), "name already in use") | 1105 | require.EqualError(t, d.renameSession(sess, "taken"), "name already in use") | ||
| @@ -1051,9 +1113,9 @@ func TestEphemeralPromotionAssignsAndPersistsLifecycleIdentity(t *testing.T) { | |||||
| 1051 | store, state := newMockStore(t) | 1113 | store, state := newMockStore(t) | ||
| 1052 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}} | 1114 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}} | ||
| 1053 | d := newTestDaemon(t, newFactory(t, p), clock) | 1115 | d := newTestDaemon(t, newFactory(t, p), clock) | ||
| 1054 | - | WithStore(store)(d) | 1116 | + | WithStore(t, store)(d) |
| 1055 | 1117 | ||||
| 1056 | - | sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | 1118 | + | sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) |
| 1057 | require.NoError(t, err) | 1119 | require.NoError(t, err) | ||
| 1058 | require.Zero(t, sess.createdAt, "regression fixture: ephemerals have no lifecycle identity") | 1120 | require.Zero(t, sess.createdAt, "regression fixture: ephemerals have no lifecycle identity") | ||
| 1059 | 1121 | ||||
| @@ -1078,9 +1140,9 @@ func TestEphemeralPromotionLifecyclePreventsStaleSameNamePaletteTarget(t *testin | |||||
| 1078 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}} | 1140 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}} | ||
| 1079 | d.clock = clock | 1141 | d.clock = clock | ||
| 1080 | d.ptys = newFactorySeq(t, firstPTY, secondPTY) | 1142 | d.ptys = newFactorySeq(t, firstPTY, secondPTY) | ||
| 1081 | - | WithStore(store)(d) | 1143 | + | WithStore(t, store)(d) |
| 1082 | 1144 | ||||
| 1083 | - | first, err := d.createSessionLocked("0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv) | 1145 | + | first, err := createSessionForTest(d, "0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv) |
| 1084 | require.NoError(t, err) | 1146 | require.NoError(t, err) | ||
| 1085 | require.NoError(t, d.renameSession(first, "named")) | 1147 | require.NoError(t, d.renameSession(first, "named")) | ||
| 1086 | staleCreatedAt := first.createdAt | 1148 | staleCreatedAt := first.createdAt | ||
| @@ -1088,7 +1150,7 @@ func TestEphemeralPromotionLifecyclePreventsStaleSameNamePaletteTarget(t *testin | |||||
| 1088 | require.NoError(t, d.killSession(first, ports.ReasonSessionKilled, true)) | 1150 | require.NoError(t, d.killSession(first, ports.ReasonSessionKilled, true)) | ||
| 1089 | d.sessWg.Wait() | 1151 | d.sessWg.Wait() | ||
| 1090 | 1152 | ||||
| 1091 | - | second, err := d.createSessionLocked("0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv) | 1153 | + | second, err := createSessionForTest(d, "0", true, "/tmp", ac.size, terminalEnv{}, d.baseEnv) |
| 1092 | require.NoError(t, err) | 1154 | require.NoError(t, err) | ||
| 1093 | require.NoError(t, d.renameSession(second, "named")) | 1155 | require.NoError(t, d.renameSession(second, "named")) | ||
| 1094 | require.NotEqual(t, staleCreatedAt, second.createdAt) | 1156 | require.NotEqual(t, staleCreatedAt, second.createdAt) | ||
| @@ -1105,10 +1167,10 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T) | |||||
| 1105 | defer release() | 1167 | defer release() | ||
| 1106 | store, state := newMockStore(t) | 1168 | store, state := newMockStore(t) | ||
| 1107 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1169 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1108 | - | WithStore(store)(d) | 1170 | + | WithStore(t, store)(d) |
| 1109 | d.lastAllocatedCreatedAt = math.MaxInt64 | 1171 | d.lastAllocatedCreatedAt = math.MaxInt64 | ||
| 1110 | 1172 | ||||
| 1111 | - | sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | 1173 | + | sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) |
| 1112 | require.NoError(t, err) | 1174 | require.NoError(t, err) | ||
| 1113 | require.EqualError(t, d.renameSession(sess, "named"), "daemon: lifecycle identities exhausted") | 1175 | require.EqualError(t, d.renameSession(sess, "named"), "daemon: lifecycle identities exhausted") | ||
| 1114 | require.Equal(t, "0", sess.name) | 1176 | require.Equal(t, "0", sess.name) | ||
| @@ -1123,17 +1185,22 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T) | |||||
| 1123 | defer release() | 1185 | defer release() | ||
| 1124 | store := portsmocks.NewMockStore(t) | 1186 | store := portsmocks.NewMockStore(t) | ||
| 1125 | var attempted map[string][]byte | 1187 | var attempted map[string][]byte | ||
| 1188 | + | store.EXPECT().Range(mock.Anything).Run(func(func([]byte, []byte) bool) {}).Times(2) | |||
| 1189 | + | store.EXPECT().Get([]byte("named")).Return(nil, false).Times(3) | |||
| 1126 | store.EXPECT().Set(mock.Anything, mock.Anything).RunAndReturn(func(key, value []byte) error { | 1190 | store.EXPECT().Set(mock.Anything, mock.Anything).RunAndReturn(func(key, value []byte) error { | ||
| 1127 | attempted = map[string][]byte{string(key): append([]byte(nil), value...)} | 1191 | attempted = map[string][]byte{string(key): append([]byte(nil), value...)} | ||
| 1128 | return errors.New("disk full") | 1192 | return errors.New("disk full") | ||
| 1129 | }).Once() | 1193 | }).Once() | ||
| 1194 | + | store.EXPECT().Delete([]byte("named")).Return(nil).Once() | |||
| 1130 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}} | 1195 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100), time.Unix(0, 100)}} | ||
| 1131 | d := newTestDaemon(t, newFactory(t, p), clock) | 1196 | d := newTestDaemon(t, newFactory(t, p), clock) | ||
| 1132 | - | WithStore(store)(d) | 1197 | + | WithStore(t, store)(d) |
| 1133 | 1198 | ||||
| 1134 | - | sess, err := d.createSessionLocked("0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | 1199 | + | sess, err := createSessionForTest(d, "0", true, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) |
| 1135 | require.NoError(t, err) | 1200 | require.NoError(t, err) | ||
| 1136 | - | require.EqualError(t, d.renameSession(sess, "named"), "disk full") | 1201 | + | err = d.renameSession(sess, "named") |
| 1202 | + | require.ErrorIs(t, err, persist.ErrCatalogueDurability) | |||
| 1203 | + | require.ErrorContains(t, err, "disk full") | |||
| 1137 | require.Equal(t, "0", sess.name) | 1204 | require.Equal(t, "0", sess.name) | ||
| 1138 | require.True(t, sess.ephemeral) | 1205 | require.True(t, sess.ephemeral) | ||
| 1139 | require.Zero(t, sess.createdAt) | 1206 | require.Zero(t, sess.createdAt) | ||
| @@ -1143,10 +1210,8 @@ func TestEphemeralPromotionLifecycleFailuresLeaveStateRollbackSafe(t *testing.T) | |||||
| 1143 | require.Len(t, records, 1) | 1210 | require.Len(t, records, 1) | ||
| 1144 | require.Equal(t, int64(100), records[0].CreatedAt, "a promotion must never attempt to persist a zero lifecycle identity") | 1211 | require.Equal(t, int64(100), records[0].CreatedAt, "a promotion must never attempt to persist a zero lifecycle identity") | ||
| 1145 | 1212 | ||||
| 1146 | - | store.EXPECT().Set(mock.Anything, mock.Anything).Return(nil).Once() | 1213 | + | require.ErrorIs(t, d.renameSession(sess, "named"), persist.ErrCatalogueDurability) |
| 1147 | - | store.EXPECT().Sync().Return(nil).Once() | 1214 | + | require.Equal(t, int64(101), d.lastAllocatedCreatedAt, "the lifecycle identity is allocated before the fenced catalogue rejects promotion") |
| 1148 | - | require.NoError(t, d.renameSession(sess, "named")) | |||
| 1149 | - | require.Equal(t, int64(101), sess.createdAt, "the high-water mark must remain monotonic after a failed promotion") | |||
| 1150 | }) | 1215 | }) | ||
| 1151 | } | 1216 | } | ||
| 1152 | 1217 | ||||
| @@ -1156,11 +1221,11 @@ func TestRefreshSessionCwdTouchesOnlyOnChange(t *testing.T) { | |||||
| 1156 | defer release() | 1221 | defer release() | ||
| 1157 | store, state := newMockStore(t) | 1222 | store, state := newMockStore(t) | ||
| 1158 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1223 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1159 | - | WithStore(store)(d) | 1224 | + | WithStore(t, store)(d) |
| 1160 | cwd := "/tmp/work" | 1225 | cwd := "/tmp/work" | ||
| 1161 | WithCwdReader(func(int) (string, error) { return cwd, nil })(d) | 1226 | WithCwdReader(func(int) (string, error) { return cwd, nil })(d) | ||
| 1162 | 1227 | ||||
| 1163 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 1228 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 1164 | require.NoError(t, err) | 1229 | require.NoError(t, err) | ||
| 1165 | state.mu.Lock() | 1230 | state.mu.Lock() | ||
| 1166 | setsAfterCreate := state.sets | 1231 | setsAfterCreate := state.sets | ||
| @@ -1184,7 +1249,7 @@ func TestAttachResumesStoppedSessionFromStoredCwd(t *testing.T) { | |||||
| 1184 | defer release() | 1249 | defer release() | ||
| 1185 | store, _ := newMockStore(t) | 1250 | store, _ := newMockStore(t) | ||
| 1186 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1251 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1187 | - | WithStore(store)(d) | 1252 | + | WithStore(t, store)(d) |
| 1188 | cwd := t.TempDir() | 1253 | cwd := t.TempDir() | ||
| 1189 | d.stopped["work"] = stoppedSession{name: "work", cwd: cwd, createdAt: 1} | 1254 | d.stopped["work"] = stoppedSession{name: "work", cwd: cwd, createdAt: 1} | ||
| 1190 | tr := portsmocks.NewMockTransport(t) | 1255 | tr := portsmocks.NewMockTransport(t) | ||
| @@ -1208,7 +1273,7 @@ func TestAttachStoppedMissingCwdFallsBackToHome(t *testing.T) { | |||||
| 1208 | defer release() | 1273 | defer release() | ||
| 1209 | store, _ := newMockStore(t) | 1274 | store, _ := newMockStore(t) | ||
| 1210 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1275 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1211 | - | WithStore(store)(d) | 1276 | + | WithStore(t, store)(d) |
| 1212 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/definitely/missing/vev", createdAt: 1} | 1277 | d.stopped["work"] = stoppedSession{name: "work", cwd: "/definitely/missing/vev", createdAt: 1} | ||
| 1213 | tr := portsmocks.NewMockTransport(t) | 1278 | tr := portsmocks.NewMockTransport(t) | ||
| 1214 | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | 1279 | tr.EXPECT().Send(mock.Anything).Return(nil).Maybe() | ||
| @@ -1222,8 +1287,8 @@ func TestAttachStoppedMissingCwdFallsBackToHome(t *testing.T) { | |||||
| 1222 | func TestPickerStoppedTargetKillPurges(t *testing.T) { | 1287 | func TestPickerStoppedTargetKillPurges(t *testing.T) { | ||
| 1223 | store, state := newMockStore(t) | 1288 | store, state := newMockStore(t) | ||
| 1224 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 1289 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 1225 | - | WithStore(store)(d) | 1290 | + | WithStore(t, store)(d) |
| 1226 | - | require.NoError(t, d.persist.Save(persist.Record{Name: "old", Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1})) | 1291 | + | require.NoError(t, testPersister(t, d).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 1, UpdatedAt: 1})) |
| 1227 | d.stopped["old"] = stoppedSession{name: "old", cwd: "/tmp", createdAt: 1} | 1292 | d.stopped["old"] = stoppedSession{name: "old", cwd: "/tmp", createdAt: 1} | ||
| 1228 | require.NoError(t, d.killPickerTarget(picker.Target{Name: "old", Stopped: true})) | 1293 | require.NoError(t, d.killPickerTarget(picker.Target{Name: "old", Stopped: true})) | ||
| 1229 | require.False(t, state.has("old")) | 1294 | require.False(t, state.has("old")) | ||
| @@ -1257,7 +1322,7 @@ func TestNewSessionAssignsStableIDsAndChildEnv(t *testing.T) { | |||||
| 1257 | d := newTestDaemon(t, f, stubClock{}) | 1322 | d := newTestDaemon(t, f, stubClock{}) | ||
| 1258 | d.baseEnv = []string{"KEEP=1", "TERM=old", "COLORTERM=old", "TERM_PROGRAM=old", "VEV=old"} | 1323 | d.baseEnv = []string{"KEEP=1", "TERM=old", "COLORTERM=old", "TERM_PROGRAM=old", "VEV=old"} | ||
| 1259 | 1324 | ||||
| 1260 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{TrueColor: true}, d.baseEnv) | 1325 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{TrueColor: true}, d.baseEnv) |
| 1261 | require.NoError(t, err) | 1326 | require.NoError(t, err) | ||
| 1262 | defer func() { | 1327 | defer func() { | ||
| 1263 | _ = d.killSession(sess, ports.ReasonServerShutdown, false) | 1328 | _ = d.killSession(sess, ports.ReasonServerShutdown, false) | ||
| @@ -1313,7 +1378,7 @@ func TestRenameSessionUnsafeNameRejected(t *testing.T) { | |||||
| 1313 | p, release := newBlockingPTY(t) | 1378 | p, release := newBlockingPTY(t) | ||
| 1314 | defer release() | 1379 | defer release() | ||
| 1315 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1380 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1316 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 1381 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 1317 | require.NoError(t, err) | 1382 | require.NoError(t, err) | ||
| 1318 | require.ErrorIs(t, d.renameSession(sess, "my work"), domain.ErrInvalidSessionName) | 1383 | require.ErrorIs(t, d.renameSession(sess, "my work"), domain.ErrInvalidSessionName) | ||
| 1319 | } | 1384 | } | ||
| @@ -1323,7 +1388,7 @@ func TestCreateTabPtyFailureIsUserError(t *testing.T) { | |||||
| 1323 | p, release := newBlockingPTY(t) | 1388 | p, release := newBlockingPTY(t) | ||
| 1324 | defer release() | 1389 | defer release() | ||
| 1325 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 1390 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 1326 | - | sess, err := d.createSessionLocked("work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) | 1391 | + | sess, err := createSessionForTest(d, "work", false, "/tmp/work", sz, terminalEnv{}, d.baseEnv) |
| 1327 | require.NoError(t, err) | 1392 | require.NoError(t, err) | ||
| 1328 | 1393 | ||||
| 1329 | cause := errors.New("fork/exec: no such file") | 1394 | cause := errors.New("fork/exec: no such file") | ||
| @@ -1350,12 +1415,12 @@ func TestNaturalExitStoppedButExplicitKillPurges(t *testing.T) { | |||||
| 1350 | defer release2() | 1415 | defer release2() | ||
| 1351 | store, state := newMockStore(t) | 1416 | store, state := newMockStore(t) | ||
| 1352 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | 1417 | d := newTestDaemon(t, newFactorySeq(t, p1, p2), stubClock{}) | ||
| 1353 | - | WithStore(store)(d) | 1418 | + | WithStore(t, store)(d) |
| 1354 | WithCwdReader(func(int) (string, error) { return "/tmp/latest", nil })(d) | 1419 | WithCwdReader(func(int) (string, error) { return "/tmp/latest", nil })(d) | ||
| 1355 | 1420 | ||||
| 1356 | - | natural, err := d.createSessionLocked("natural", false, "/tmp/old", sz, terminalEnv{}, d.baseEnv) | 1421 | + | natural, err := createSessionForTest(d, "natural", false, "/tmp/old", sz, terminalEnv{}, d.baseEnv) |
| 1357 | require.NoError(t, err) | 1422 | require.NoError(t, err) | ||
| 1358 | - | other, err := d.createSessionLocked("other", false, "/tmp/other", sz, terminalEnv{}, d.baseEnv) | 1423 | + | other, err := createSessionForTest(d, "other", false, "/tmp/other", sz, terminalEnv{}, d.baseEnv) |
| 1359 | require.NoError(t, err) | 1424 | require.NoError(t, err) | ||
| 1360 | _ = d.killSession(natural, ports.ReasonSessionKilled, false) | 1425 | _ = d.killSession(natural, ports.ReasonSessionKilled, false) | ||
| 1361 | require.True(t, state.has("natural")) | 1426 | require.True(t, state.has("natural")) | ||
| @@ -1671,7 +1736,7 @@ func TestNamedSessionLifecycleTimestampsAreMonotonicAcrossClockRegression(t *tes | |||||
| 1671 | 1736 | ||||
| 1672 | var got []int64 | 1737 | var got []int64 | ||
| 1673 | for _, name := range []string{"one", "two", "three"} { | 1738 | for _, name := range []string{"one", "two", "three"} { | ||
| 1674 | - | sess, err := d.createSessionLocked(name, false, "/tmp", sz, terminalEnv{}, d.baseEnv) | 1739 | + | sess, err := createSessionForTest(d, name, false, "/tmp", sz, terminalEnv{}, d.baseEnv) |
| 1675 | require.NoError(t, err) | 1740 | require.NoError(t, err) | ||
| 1676 | got = append(got, sess.createdAt) | 1741 | got = append(got, sess.createdAt) | ||
| 1677 | } | 1742 | } | ||
| @@ -1684,7 +1749,7 @@ func TestNamedSessionLifecycleExhaustionDoesNotMutateSessionState(t *testing.T) | |||||
| 1684 | d.nextID = 17 | 1749 | d.nextID = 17 | ||
| 1685 | d.stopped["retained"] = stoppedSession{name: "retained", cwd: "/tmp", createdAt: 9} | 1750 | d.stopped["retained"] = stoppedSession{name: "retained", cwd: "/tmp", createdAt: 9} | ||
| 1686 | 1751 | ||||
| 1687 | - | sess, err := d.createSessionLocked("new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | 1752 | + | sess, err := createSessionForTest(d, "new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) |
| 1688 | 1753 | ||||
| 1689 | require.ErrorContains(t, err, "lifecycle identities exhausted") | 1754 | require.ErrorContains(t, err, "lifecycle identities exhausted") | ||
| 1690 | require.Nil(t, sess) | 1755 | require.Nil(t, sess) | ||
| @@ -1700,20 +1765,52 @@ func TestNamedSessionLifecycleExhaustionDoesNotMutateSessionState(t *testing.T) | |||||
| 1700 | require.Equal(t, stoppedSession{name: "retained", cwd: "/tmp", createdAt: 9}, d.stopped["retained"]) | 1765 | require.Equal(t, stoppedSession{name: "retained", cwd: "/tmp", createdAt: 9}, d.stopped["retained"]) | ||
| 1701 | } | 1766 | } | ||
| 1702 | 1767 | ||||
| 1768 | + | func TestCatalogueRecordsConstructExpectedSessionRegistry(t *testing.T) { | |||
| 1769 | + | alphaRef := &domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}} | |||
| 1770 | + | workRef := &domain.CheckpointRef{Generation: 2, ManifestDigest: [32]byte{2}} | |||
| 1771 | + | records := []domain.CatalogueRecord{ | |||
| 1772 | + | {Name: "alpha", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp/alpha", CreatedAt: 7, LastUsedSeq: 11, TabNames: []string{"shell"}, Committed: alphaRef}, | |||
| 1773 | + | {Name: "work", IncarnationID: domain.IncarnationID{2}, Cwd: "/tmp/work", CreatedAt: 9, LastUsedSeq: 13, TabNames: []string{"editor", "logs"}, Committed: workRef, DegradedReason: "uncertain legacy checkpoint"}, | |||
| 1774 | + | } | |||
| 1775 | + | ||||
| 1776 | + | d := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithCatalogue(persist.New(nil), records)) | |||
| 1777 | + | ||||
| 1778 | + | alpha := d.stopped["alpha"] | |||
| 1779 | + | require.Equal(t, records[0], alpha.record) | |||
| 1780 | + | require.Equal(t, ports.SessionStopped, alpha.state) | |||
| 1781 | + | require.NotNil(t, alpha.restoreDone) | |||
| 1782 | + | select { | |||
| 1783 | + | case <-alpha.restoreDone: | |||
| 1784 | + | t.Fatal("healthy catalogue record must begin in restoring state") | |||
| 1785 | + | default: | |||
| 1786 | + | } | |||
| 1787 | + | work := d.stopped["work"] | |||
| 1788 | + | require.Equal(t, records[1], work.record) | |||
| 1789 | + | require.Equal(t, ports.SessionBroken, work.state) | |||
| 1790 | + | require.NotNil(t, work.restoreDone) | |||
| 1791 | + | select { | |||
| 1792 | + | case <-work.restoreDone: | |||
| 1793 | + | default: | |||
| 1794 | + | t.Fatal("degraded catalogue record must not wait for restoration") | |||
| 1795 | + | } | |||
| 1796 | + | require.Equal(t, int64(9), d.lastAllocatedCreatedAt) | |||
| 1797 | + | require.Equal(t, uint64(13), d.mruSeq.Load()) | |||
| 1798 | + | } | |||
| 1799 | + | ||||
| 1703 | func TestNamedSessionLifecycleTimestampStartsAfterPersistedHighWaterMark(t *testing.T) { | 1800 | func TestNamedSessionLifecycleTimestampStartsAfterPersistedHighWaterMark(t *testing.T) { | ||
| 1704 | store, _ := newMockStore(t) | 1801 | store, _ := newMockStore(t) | ||
| 1705 | - | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | 1802 | + | seed := New(nil, stubClock{}, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store)) |
| 1706 | - | require.NoError(t, seed.persist.Save(persist.Record{Name: "old", Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900})) | 1803 | + | require.NoError(t, testPersister(t, seed).Save(persist.Record{Name: "old", IncarnationID: domain.IncarnationID{1}, Cwd: "/tmp", CreatedAt: 900, UpdatedAt: 900})) |
| 1707 | 1804 | ||||
| 1708 | p, release := newBlockingPTY(t) | 1805 | p, release := newBlockingPTY(t) | ||
| 1709 | defer release() | 1806 | defer release() | ||
| 1710 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}} | 1807 | clock := &lifecycleClock{nows: []time.Time{time.Unix(0, 100)}} | ||
| 1711 | // Constructing with persistence must establish the lifecycle high-water mark. | 1808 | // Constructing with persistence must establish the lifecycle high-water mark. | ||
| 1712 | - | d := New(newFactory(t, p), clock, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(store)) | 1809 | + | d := New(newFactory(t, p), clock, slog.New(slog.NewTextHandler(io.Discard, nil)), WithStore(t, store)) |
| 1713 | d.serveCtx, d.serveCancel = context.WithCancel(context.Background()) | 1810 | d.serveCtx, d.serveCancel = context.WithCancel(context.Background()) | ||
| 1714 | t.Cleanup(d.serveCancel) | 1811 | t.Cleanup(d.serveCancel) | ||
| 1715 | 1812 | ||||
| 1716 | - | sess, err := d.createSessionLocked("new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | 1813 | + | sess, err := createSessionForTest(d, "new", false, "/tmp", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) |
| 1717 | require.NoError(t, err) | 1814 | require.NoError(t, err) | ||
| 1718 | require.Equal(t, int64(901), sess.createdAt) | 1815 | require.Equal(t, int64(901), sess.createdAt) | ||
| 1719 | } | 1816 | } | ||
| @@ -1725,7 +1822,7 @@ func TestResumingStoppedSessionPreservesLifecycleIdentityInPersistence(t *testin | |||||
| 1725 | defer releaseTarget() | 1822 | defer releaseTarget() | ||
| 1726 | d, from, ac, _ := newManualSessionWithPTYs(t, fromPTY) | 1823 | d, from, ac, _ := newManualSessionWithPTYs(t, fromPTY) | ||
| 1727 | store, state := newMockStore(t) | 1824 | store, state := newMockStore(t) | ||
| 1728 | - | WithStore(store)(d) | 1825 | + | WithStore(t, store)(d) |
| 1729 | d.ptys = newFactory(t, targetPTY) | 1826 | d.ptys = newFactory(t, targetPTY) | ||
| 1730 | d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp", createdAt: 77} | 1827 | d.stopped["stopped"] = stoppedSession{name: "stopped", cwd: "/tmp", createdAt: 77} | ||
| 1731 | 1828 | ||||
M internal/usecase/daemon/snapshot.go +10 -0
| @@ -7,6 +7,7 @@ import ( | |||||
| 7 | "sync" | 7 | "sync" | ||
| 8 | "time" | 8 | "time" | ||
| 9 | 9 | ||||
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 10 | "github.com/bnema/vev/internal/usecase/layout" | 11 | "github.com/bnema/vev/internal/usecase/layout" | ||
| 11 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | 12 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 12 | "github.com/bnema/vev/pkg/vt" | 13 | "github.com/bnema/vev/pkg/vt" | ||
| @@ -35,14 +36,20 @@ type snapshotCapture struct { | |||||
| 35 | session *session | 36 | session *session | ||
| 36 | attemptKind snapshotAttemptKind | 37 | attemptKind snapshotAttemptKind | ||
| 37 | generation uint64 // repository publication generation | 38 | generation uint64 // repository publication generation | ||
| 39 | + | parentCheckpoint *domain.CheckpointRef | |||
| 40 | + | checkpoint domain.CheckpointRef // set by the encoder before publication | |||
| 38 | mutationRevision uint64 | 41 | mutationRevision uint64 | ||
| 39 | name string | 42 | name string | ||
| 43 | + | incarnation domain.IncarnationID | |||
| 40 | createdAt uint64 | 44 | createdAt uint64 | ||
| 41 | active uint16 | 45 | active uint16 | ||
| 42 | tabs []snapshotCaptureTab | 46 | tabs []snapshotCaptureTab | ||
| 43 | publicationContext context.Context | 47 | publicationContext context.Context | ||
| 44 | sealedRefs map[*vt.HistoryChunk]snapcodec.ObjectRef // set by the single encoder worker | 48 | sealedRefs map[*vt.HistoryChunk]snapcodec.ObjectRef // set by the single encoder worker | ||
| 45 | coordinatorDiscarded bool // guarded by session.snapshotMu | 49 | coordinatorDiscarded bool // guarded by session.snapshotMu | ||
| 50 | + | // normalWorkerAdmitted is set before snapshotJobs admission and immutable | |||
| 51 | + | // until completion; final-queue captures leave it false. | |||
| 52 | + | normalWorkerAdmitted bool | |||
| 46 | finishOnce sync.Once | 53 | finishOnce sync.Once | ||
| 47 | } | 54 | } | ||
| 48 | 55 | ||||
| @@ -94,3 +101,6 @@ const ( | |||||
| 94 | snapshotInterval = 2 * time.Minute | 101 | snapshotInterval = 2 * time.Minute | ||
| 95 | snapshotFinalFlushTimeout = time.Second | 102 | snapshotFinalFlushTimeout = time.Second | ||
| 96 | ) | 103 | ) | ||
| 104 | + | ||||
| 105 | + | // SnapshotShutdownTimeout reports the shared final-checkpoint shutdown budget. | |||
| 106 | + | func SnapshotShutdownTimeout() time.Duration { return snapshotFinalFlushTimeout } | |||
M internal/usecase/daemon/snapshot_acceptance_test.go +4 -3
| @@ -63,9 +63,10 @@ func snapshotCacheHistory(t *testing.T, chunks int, first rune) vt.HistorySnapsh | |||||
| 63 | 63 | ||||
| 64 | func snapshotCacheCapture(sess *session, history vt.HistorySnapshotView, generation uint64) *snapshotCapture { | 64 | func snapshotCacheCapture(sess *session, history vt.HistorySnapshotView, generation uint64) *snapshotCapture { | ||
| 65 | return &snapshotCapture{ | 65 | return &snapshotCapture{ | ||
| 66 | - | session: sess, | 66 | + | session: sess, |
| 67 | - | name: sess.name, | 67 | + | name: sess.name, |
| 68 | - | generation: generation, | 68 | + | incarnation: sess.incarnation, |
| 69 | + | generation: generation, | |||
| 69 | tabs: []snapshotCaptureTab{{ | 70 | tabs: []snapshotCaptureTab{{ | ||
| 70 | stableID: "tab", | 71 | stableID: "tab", | ||
| 71 | cols: 1, | 72 | cols: 1, | ||
M internal/usecase/daemon/snapshot_capture.go +6 -5
| @@ -8,11 +8,12 @@ import snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |||||
| 8 | func (d *Daemon) captureSnapshotState(sess *session, generation uint64) (*snapshotCapture, bool) { | 8 | func (d *Daemon) captureSnapshotState(sess *session, generation uint64) (*snapshotCapture, bool) { | ||
| 9 | sess.mu.Lock() | 9 | sess.mu.Lock() | ||
| 10 | capture := &snapshotCapture{ | 10 | capture := &snapshotCapture{ | ||
| 11 | - | session: sess, | 11 | + | session: sess, |
| 12 | - | generation: generation, | 12 | + | generation: generation, |
| 13 | - | name: sess.name, | 13 | + | name: sess.name, |
| 14 | - | createdAt: uint64(sess.createdAt), | 14 | + | incarnation: sess.incarnation, |
| 15 | - | active: uint16(max(sess.active, 0)), | 15 | + | createdAt: uint64(sess.createdAt), |
| 16 | + | active: uint16(max(sess.active, 0)), | |||
| 16 | } | 17 | } | ||
| 17 | ephemeral := sess.ephemeral | 18 | ephemeral := sess.ephemeral | ||
| 18 | fallbackCwd := sess.cwd | 19 | fallbackCwd := sess.cwd | ||
M internal/usecase/daemon/snapshot_coordinator.go +23 -63
| @@ -1,23 +1,11 @@ | |||||
| 1 | package daemon | 1 | package daemon | ||
| 2 | 2 | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "time" | |||
| 6 | - | ) | |||
| 7 | - | ||||
| 8 | - | // snapshotCoordinatorQuarantine identifies one stop request. A later teardown | |||
| 9 | - | // can supersede it, in which case only that later owner may resume publication. | |||
| 10 | - | type snapshotCoordinatorQuarantine struct { | |||
| 11 | - | done <-chan struct{} | |||
| 12 | - | epoch uint64 | |||
| 13 | - | } | |||
| 14 | - | ||||
| 15 | // quarantineSnapshotCoordinator prevents a deleted session from starting any | 3 | // quarantineSnapshotCoordinator prevents a deleted session from starting any | ||
| 16 | // further publication. The returned join channel is closed after an already | 4 | // further publication. The returned join channel is closed after an already | ||
| 17 | // started repository call returns; callers must wait without daemon or session | 5 | // started repository call returns; callers must wait without daemon or session | ||
| 18 | // locks before deleting repository state. | 6 | // locks before deleting repository state. | ||
| 19 | func quarantineSnapshotCoordinator(sess *session) <-chan struct{} { | 7 | func quarantineSnapshotCoordinator(sess *session) <-chan struct{} { | ||
| 20 | - | return quarantineSnapshotCoordinatorWithEpoch(sess).done | 8 | + | return quarantineSnapshotCoordinatorWithOptions(sess, false) |
| 21 | } | 9 | } | ||
| 22 | 10 | ||||
| 23 | // quarantineSnapshotCoordinatorRetainingQueuedCapture is used after a forced | 11 | // quarantineSnapshotCoordinatorRetainingQueuedCapture is used after a forced | ||
| @@ -26,22 +14,16 @@ func quarantineSnapshotCoordinator(sess *session) <-chan struct{} { | |||||
| 26 | // quarantined, and the worker eventually discards it after its blocked call | 14 | // quarantined, and the worker eventually discards it after its blocked call | ||
| 27 | // returns. | 15 | // returns. | ||
| 28 | func quarantineSnapshotCoordinatorRetainingQueuedCapture(sess *session) <-chan struct{} { | 16 | func quarantineSnapshotCoordinatorRetainingQueuedCapture(sess *session) <-chan struct{} { | ||
| 29 | - | return quarantineSnapshotCoordinatorWithOptions(sess, true).done | 17 | + | return quarantineSnapshotCoordinatorWithOptions(sess, true) |
| 30 | - | } | |||
| 31 | - | ||||
| 32 | - | func quarantineSnapshotCoordinatorWithEpoch(sess *session) snapshotCoordinatorQuarantine { | |||
| 33 | - | return quarantineSnapshotCoordinatorWithOptions(sess, false) | |||
| 34 | } | 18 | } | ||
| 35 | 19 | ||||
| 36 | - | func quarantineSnapshotCoordinatorWithOptions(sess *session, retainQueuedCapture bool) snapshotCoordinatorQuarantine { | 20 | + | func quarantineSnapshotCoordinatorWithOptions(sess *session, retainQueuedCapture bool) <-chan struct{} { |
| 37 | if sess == nil { | 21 | if sess == nil { | ||
| 38 | done := make(chan struct{}) | 22 | done := make(chan struct{}) | ||
| 39 | close(done) | 23 | close(done) | ||
| 40 | - | return snapshotCoordinatorQuarantine{done: done} | 24 | + | return done |
| 41 | } | 25 | } | ||
| 42 | sess.snapshotMu.Lock() | 26 | sess.snapshotMu.Lock() | ||
| 43 | - | sess.snapshotQuarantineEpoch++ | |||
| 44 | - | epoch := sess.snapshotQuarantineEpoch | |||
| 45 | sess.snapshotQuarantined = true | 27 | sess.snapshotQuarantined = true | ||
| 46 | sess.snapEligible.Store(false) | 28 | sess.snapEligible.Store(false) | ||
| 47 | if sess.snapshotPublicationCancel != nil { | 29 | if sess.snapshotPublicationCancel != nil { | ||
| @@ -65,47 +47,7 @@ func quarantineSnapshotCoordinatorWithOptions(sess *session, retainQueuedCapture | |||||
| 65 | } | 47 | } | ||
| 66 | sess.signalSnapshotChangedLocked() | 48 | sess.signalSnapshotChangedLocked() | ||
| 67 | sess.snapshotMu.Unlock() | 49 | sess.snapshotMu.Unlock() | ||
| 68 | - | return snapshotCoordinatorQuarantine{done: done, epoch: epoch} | 50 | + | return done |
| 69 | - | } | |||
| 70 | - | ||||
| 71 | - | // resumeSnapshotCoordinatorForNewIdentity starts a fresh repository lineage | |||
| 72 | - | // only after rename committed its new in-memory name. The epoch prevents a | |||
| 73 | - | // concurrent teardown from being undone by a late rename completion. | |||
| 74 | - | func resumeSnapshotCoordinatorForNewIdentity(sess *session, quarantine snapshotCoordinatorQuarantine) bool { | |||
| 75 | - | if sess == nil { | |||
| 76 | - | return false | |||
| 77 | - | } | |||
| 78 | - | sess.snapshotMu.Lock() | |||
| 79 | - | if !sess.snapshotQuarantined || sess.snapshotQuarantineEpoch != quarantine.epoch { | |||
| 80 | - | sess.snapshotMu.Unlock() | |||
| 81 | - | return false | |||
| 82 | - | } | |||
| 83 | - | if sess.snapshotPublicationCancel != nil { | |||
| 84 | - | sess.snapshotPublicationCancel() | |||
| 85 | - | } | |||
| 86 | - | sess.snapshotPublicationContext, sess.snapshotPublicationCancel = context.WithCancel(context.Background()) | |||
| 87 | - | sess.snapshotQuarantined = false | |||
| 88 | - | // A repository name owns its own generation stream. Reset all scheduler | |||
| 89 | - | // state while eligibility remains false, then atomically make generation 1 | |||
| 90 | - | // dirty and wake the scheduler. | |||
| 91 | - | sess.snapshotGeneration = 1 | |||
| 92 | - | sess.snapshotPublishedGeneration = 0 | |||
| 93 | - | sess.snapshotPublishedMutationRevision = 0 | |||
| 94 | - | sess.snapshotForcedGeneration = 0 | |||
| 95 | - | sess.snapshotNextEligibleAt = time.Time{} | |||
| 96 | - | sess.snapshotChunkCache = newSnapshotChunkCache(snapshotChunkCacheLimit) | |||
| 97 | - | sess.snapDirty.Store(true) | |||
| 98 | - | sess.snapEligible.Store(true) | |||
| 99 | - | sess.signalSnapshotChangedLocked() | |||
| 100 | - | wake := sess.snapshotWake | |||
| 101 | - | sess.snapshotMu.Unlock() | |||
| 102 | - | if wake != nil { | |||
| 103 | - | select { | |||
| 104 | - | case wake <- struct{}{}: | |||
| 105 | - | default: | |||
| 106 | - | } | |||
| 107 | - | } | |||
| 108 | - | return true | |||
| 109 | } | 51 | } | ||
| 110 | 52 | ||||
| 111 | func markSnapshotDirty(sess *session) { | 53 | func markSnapshotDirty(sess *session) { | ||
| @@ -166,6 +108,14 @@ func (d *Daemon) waitForSnapshotGenerationWithDeadline(sess *session, generation | |||||
| 166 | return d.waitForSnapshotGeneration(sess, generation) | 108 | return d.waitForSnapshotGeneration(sess, generation) | ||
| 167 | } | 109 | } | ||
| 168 | for { | 110 | for { | ||
| 111 | + | // Once the shared deadline is observable, timeout wins even if publication | |||
| 112 | + | // completion becomes ready concurrently. This preserves the timed-out | |||
| 113 | + | // session identity and avoids nondeterministic success after the budget. | |||
| 114 | + | select { | |||
| 115 | + | case <-deadline.Done(): | |||
| 116 | + | return false | |||
| 117 | + | default: | |||
| 118 | + | } | |||
| 169 | sess.snapshotMu.Lock() | 119 | sess.snapshotMu.Lock() | ||
| 170 | published := sess.snapshotPublishedMutationRevision >= generation | 120 | published := sess.snapshotPublishedMutationRevision >= generation | ||
| 171 | changed := sess.snapshotChangeLocked() | 121 | changed := sess.snapshotChangeLocked() | ||
| @@ -238,6 +188,11 @@ func (d *Daemon) finishSnapshotCapture(capture *snapshotCapture, succeeded bool) | |||||
| 238 | mutationRevision = capture.generation | 188 | mutationRevision = capture.generation | ||
| 239 | } | 189 | } | ||
| 240 | if succeeded { | 190 | if succeeded { | ||
| 191 | + | if capture.generation >= capture.session.snapshotPublishedGeneration && | |||
| 192 | + | capture.checkpoint.Generation == capture.generation && capture.checkpoint.ManifestDigest != ([32]byte{}) { | |||
| 193 | + | checkpoint := capture.checkpoint | |||
| 194 | + | capture.session.snapshotPublishedCheckpoint = &checkpoint | |||
| 195 | + | } | |||
| 241 | capture.session.snapshotPublishedGeneration = max(capture.session.snapshotPublishedGeneration, capture.generation) | 196 | capture.session.snapshotPublishedGeneration = max(capture.session.snapshotPublishedGeneration, capture.generation) | ||
| 242 | capture.session.snapshotPublishedMutationRevision = max(capture.session.snapshotPublishedMutationRevision, mutationRevision) | 197 | capture.session.snapshotPublishedMutationRevision = max(capture.session.snapshotPublishedMutationRevision, mutationRevision) | ||
| 243 | } | 198 | } | ||
| @@ -260,6 +215,11 @@ func (d *Daemon) finishSnapshotCapture(capture *snapshotCapture, succeeded bool) | |||||
| 260 | capture.session.signalSnapshotChangedLocked() | 215 | capture.session.signalSnapshotChangedLocked() | ||
| 261 | wake := capture.session.snapshotWake | 216 | wake := capture.session.snapshotWake | ||
| 262 | capture.session.snapshotMu.Unlock() | 217 | capture.session.snapshotMu.Unlock() | ||
| 218 | + | if capture.normalWorkerAdmitted { | |||
| 219 | + | d.snapshotWorkerMu.Lock() | |||
| 220 | + | delete(d.snapshotAdmitted, capture) | |||
| 221 | + | d.snapshotWorkerMu.Unlock() | |||
| 222 | + | } | |||
| 263 | if wake != nil { | 223 | if wake != nil { | ||
| 264 | select { | 224 | select { | ||
| 265 | case wake <- struct{}{}: | 225 | case wake <- struct{}{}: | ||
M internal/usecase/daemon/snapshot_generation_test.go +2 -2
| @@ -17,7 +17,7 @@ import ( | |||||
| 17 | func TestSnapshotMultipleMutationsPublishNextRepositoryGeneration(t *testing.T) { | 17 | func TestSnapshotMultipleMutationsPublishNextRepositoryGeneration(t *testing.T) { | ||
| 18 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 18 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 19 | repository := portsmocks.NewMockSnapshotRepository(t) | 19 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 20 | - | WithSnapshotRepository(repository, nil)(d) | 20 | + | WithSnapshotRepository(repository)(d) |
| 21 | startSnapshotEncodeWorker(t, d) | 21 | startSnapshotEncodeWorker(t, d) | ||
| 22 | 22 | ||||
| 23 | publications := make(chan ports.SnapshotPublication, 1) | 23 | publications := make(chan ports.SnapshotPublication, 1) | ||
| @@ -44,7 +44,7 @@ func TestSnapshotMultipleMutationsPublishNextRepositoryGeneration(t *testing.T) | |||||
| 44 | func TestSnapshotRetryKeepsRepositoryGeneration(t *testing.T) { | 44 | func TestSnapshotRetryKeepsRepositoryGeneration(t *testing.T) { | ||
| 45 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 45 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 46 | repository := portsmocks.NewMockSnapshotRepository(t) | 46 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 47 | - | WithSnapshotRepository(repository, nil)(d) | 47 | + | WithSnapshotRepository(repository)(d) |
| 48 | startSnapshotEncodeWorker(t, d) | 48 | startSnapshotEncodeWorker(t, d) | ||
| 49 | 49 | ||||
| 50 | publications := make(chan ports.SnapshotPublication, 2) | 50 | publications := make(chan ports.SnapshotPublication, 2) | ||
D internal/usecase/daemon/snapshot_import_test.go +0 -215
| @@ -1,215 +0,0 @@ | |||||
| 1 | - | package daemon | |||
| 2 | - | ||||
| 3 | - | import ( | |||
| 4 | - | "context" | |||
| 5 | - | "encoding/base64" | |||
| 6 | - | "errors" | |||
| 7 | - | "os" | |||
| 8 | - | "path/filepath" | |||
| 9 | - | "testing" | |||
| 10 | - | ||||
| 11 | - | "github.com/stretchr/testify/require" | |||
| 12 | - | ||||
| 13 | - | "github.com/bnema/vev/internal/ports" | |||
| 14 | - | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | |||
| 15 | - | ) | |||
| 16 | - | ||||
| 17 | - | type snapshotAcceptanceLegacySource struct { | |||
| 18 | - | blobs []ports.LegacySnapshot | |||
| 19 | - | loadCalls int | |||
| 20 | - | deleteCalls []string | |||
| 21 | - | deleteErr error | |||
| 22 | - | } | |||
| 23 | - | ||||
| 24 | - | func (s *snapshotAcceptanceLegacySource) LoadLegacy(ctx context.Context) ([]ports.LegacySnapshot, error) { | |||
| 25 | - | if err := ctx.Err(); err != nil { | |||
| 26 | - | return nil, err | |||
| 27 | - | } | |||
| 28 | - | s.loadCalls++ | |||
| 29 | - | out := make([]ports.LegacySnapshot, len(s.blobs)) | |||
| 30 | - | for i, blob := range s.blobs { | |||
| 31 | - | out[i] = ports.LegacySnapshot{Name: blob.Name, Data: append([]byte(nil), blob.Data...)} | |||
| 32 | - | } | |||
| 33 | - | return out, nil | |||
| 34 | - | } | |||
| 35 | - | ||||
| 36 | - | func (s *snapshotAcceptanceLegacySource) DeleteVerifiedLegacy(ctx context.Context, blob ports.LegacySnapshot) error { | |||
| 37 | - | return s.DeleteLegacy(ctx, blob.Name) | |||
| 38 | - | } | |||
| 39 | - | ||||
| 40 | - | func (s *snapshotAcceptanceLegacySource) DeleteLegacy(ctx context.Context, name string) error { | |||
| 41 | - | if err := ctx.Err(); err != nil { | |||
| 42 | - | return err | |||
| 43 | - | } | |||
| 44 | - | s.deleteCalls = append(s.deleteCalls, name) | |||
| 45 | - | if s.deleteErr != nil { | |||
| 46 | - | return s.deleteErr | |||
| 47 | - | } | |||
| 48 | - | for i, blob := range s.blobs { | |||
| 49 | - | if blob.Name == name { | |||
| 50 | - | s.blobs = append(s.blobs[:i], s.blobs[i+1:]...) | |||
| 51 | - | break | |||
| 52 | - | } | |||
| 53 | - | } | |||
| 54 | - | return nil | |||
| 55 | - | } | |||
| 56 | - | ||||
| 57 | - | func legacyAcceptanceSnapshot(t *testing.T) snapcodec.Session { | |||
| 58 | - | t.Helper() | |||
| 59 | - | data, err := os.ReadFile(filepath.Join("testdata", "legacy-v3-empty.b64")) | |||
| 60 | - | require.NoError(t, err) | |||
| 61 | - | encoded, err := base64.StdEncoding.DecodeString(string(data)) | |||
| 62 | - | require.NoError(t, err) | |||
| 63 | - | snapshot, err := snapcodec.Unmarshal(encoded) | |||
| 64 | - | require.NoError(t, err) | |||
| 65 | - | return snapshot | |||
| 66 | - | } | |||
| 67 | - | ||||
| 68 | - | func legacyAcceptanceBlob(t *testing.T, snapshot snapcodec.Session) ports.LegacySnapshot { | |||
| 69 | - | t.Helper() | |||
| 70 | - | fixture := legacyAcceptanceSnapshot(t) | |||
| 71 | - | require.Equal(t, fixture, snapshot, "legacy fixture must be the imported session") | |||
| 72 | - | data, err := os.ReadFile(filepath.Join("testdata", "legacy-v3-empty.b64")) | |||
| 73 | - | require.NoError(t, err) | |||
| 74 | - | encoded, err := base64.StdEncoding.DecodeString(string(data)) | |||
| 75 | - | require.NoError(t, err) | |||
| 76 | - | return ports.LegacySnapshot{Name: snapshot.Name, Data: encoded} | |||
| 77 | - | } | |||
| 78 | - | ||||
| 79 | - | func TestLegacyPublicationMarshalObjectErrorsIdentifyPaneAndStage(t *testing.T) { | |||
| 80 | - | for _, tt := range []struct { | |||
| 81 | - | name string | |||
| 82 | - | mutate func(*snapcodec.Pane) | |||
| 83 | - | stage string | |||
| 84 | - | }{ | |||
| 85 | - | {name: "sealed chunk", mutate: func(pane *snapcodec.Pane) { pane.SealedChunks = [][]byte{nil} }, stage: "sealed chunk 0"}, | |||
| 86 | - | {name: "tail", mutate: func(pane *snapcodec.Pane) { pane.Tail = nil }, stage: "tail"}, | |||
| 87 | - | {name: "visible", mutate: func(pane *snapcodec.Pane) { pane.Visible = nil }, stage: "visible"}, | |||
| 88 | - | } { | |||
| 89 | - | t.Run(tt.name, func(t *testing.T) { | |||
| 90 | - | pane := snapcodec.Pane{ID: "pane-1", StableID: "pane-stable", Tail: []byte("tail"), Visible: []byte("visible")} | |||
| 91 | - | tt.mutate(&pane) | |||
| 92 | - | snapshot := snapcodec.Session{Name: "legacy", Tabs: []snapcodec.Tab{{StableID: "tab-1", Panes: []snapcodec.Pane{pane}}}} | |||
| 93 | - | ||||
| 94 | - | _, err := legacyPublication(snapshot) | |||
| 95 | - | require.Error(t, err) | |||
| 96 | - | require.ErrorContains(t, err, `tab "tab-1"`) | |||
| 97 | - | require.ErrorContains(t, err, `pane "pane-1"`) | |||
| 98 | - | require.ErrorContains(t, err, tt.stage) | |||
| 99 | - | require.ErrorContains(t, err, "marshal object") | |||
| 100 | - | }) | |||
| 101 | - | } | |||
| 102 | - | } | |||
| 103 | - | ||||
| 104 | - | func TestImportLegacyPublishesCompleteGenerationBeforeDeletingSource(t *testing.T) { | |||
| 105 | - | snapshot := legacyAcceptanceSnapshot(t) | |||
| 106 | - | repository := &snapshotAcceptanceRepository{generations: make(map[string]ports.SnapshotGeneration)} | |||
| 107 | - | legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{legacyAcceptanceBlob(t, snapshot)}} | |||
| 108 | - | d := newTestDaemon(t, nil, stubClock{}) | |||
| 109 | - | WithSnapshotRepository(repository, legacy)(d) | |||
| 110 | - | expected, verifyErr := legacyPublication(snapshot) | |||
| 111 | - | require.NoError(t, verifyErr) | |||
| 112 | - | _, verifyErr = snapcodec.Unmarshal(legacy.blobs[0].Data) | |||
| 113 | - | require.NoError(t, verifyErr) | |||
| 114 | - | require.NoError(t, verifyLegacyImportGeneration(acceptanceGeneration(t, snapshot, 1), expected)) | |||
| 115 | - | ||||
| 116 | - | d.importLegacySnapshots(context.Background()) | |||
| 117 | - | ||||
| 118 | - | require.Len(t, repository.publishes, 1) | |||
| 119 | - | publication := repository.publishes[0] | |||
| 120 | - | require.Equal(t, snapshot.Name, publication.Name) | |||
| 121 | - | require.EqualValues(t, 1, publication.Generation) | |||
| 122 | - | manifest, err := snapcodec.UnmarshalManifest(publication.Manifest) | |||
| 123 | - | require.NoError(t, err) | |||
| 124 | - | require.Equal(t, uint64(1), manifest.Generation) | |||
| 125 | - | require.Empty(t, manifest.Tabs) | |||
| 126 | - | for _, object := range publication.Objects { | |||
| 127 | - | kind, _, err := snapcodec.UnmarshalObject(object.Data) | |||
| 128 | - | require.NoError(t, err) | |||
| 129 | - | require.Contains(t, []snapcodec.ObjectKind{snapcodec.HistoryChunk, snapcodec.HistoryTail, snapcodec.Visible}, kind) | |||
| 130 | - | require.Equal(t, object.Digest, snapcodec.ManifestDigest(object.Data)) | |||
| 131 | - | } | |||
| 132 | - | require.Equal(t, []string{snapshot.Name}, legacy.deleteCalls) | |||
| 133 | - | require.Empty(t, legacy.blobs) | |||
| 134 | - | } | |||
| 135 | - | ||||
| 136 | - | func TestImportLegacyExistingIncrementalSkipsLegacy(t *testing.T) { | |||
| 137 | - | snapshot := legacyAcceptanceSnapshot(t) | |||
| 138 | - | repository := &snapshotAcceptanceRepository{names: []string{snapshot.Name}, generations: make(map[string]ports.SnapshotGeneration)} | |||
| 139 | - | legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{legacyAcceptanceBlob(t, snapshot)}} | |||
| 140 | - | d := newTestDaemon(t, nil, stubClock{}) | |||
| 141 | - | WithSnapshotRepository(repository, legacy)(d) | |||
| 142 | - | ||||
| 143 | - | d.importLegacySnapshots(context.Background()) | |||
| 144 | - | ||||
| 145 | - | require.Empty(t, repository.publishes) | |||
| 146 | - | require.Empty(t, legacy.deleteCalls) | |||
| 147 | - | require.Len(t, legacy.blobs, 1) | |||
| 148 | - | } | |||
| 149 | - | ||||
| 150 | - | func TestImportLegacyRetainsSourceUntilExactReloadVerification(t *testing.T) { | |||
| 151 | - | snapshot := legacyAcceptanceSnapshot(t) | |||
| 152 | - | for _, test := range []struct { | |||
| 153 | - | name string | |||
| 154 | - | publish error | |||
| 155 | - | load error | |||
| 156 | - | mutate func(ports.SnapshotGeneration, ports.SnapshotPublication) ports.SnapshotGeneration | |||
| 157 | - | }{ | |||
| 158 | - | {"publish failure", errors.New("publish failed"), nil, nil}, | |||
| 159 | - | {"reload failure", nil, errors.New("reload failed"), nil}, | |||
| 160 | - | {"manifest metadata mismatch", nil, nil, func(g ports.SnapshotGeneration, _ ports.SnapshotPublication) ports.SnapshotGeneration { | |||
| 161 | - | g.Generation++ | |||
| 162 | - | return g | |||
| 163 | - | }}, | |||
| 164 | - | {"manifest bytes mismatch", nil, nil, func(g ports.SnapshotGeneration, _ ports.SnapshotPublication) ports.SnapshotGeneration { | |||
| 165 | - | g.Manifest[len(g.Manifest)-1] ^= 1 | |||
| 166 | - | return g | |||
| 167 | - | }}, | |||
| 168 | - | } { | |||
| 169 | - | t.Run(test.name, func(t *testing.T) { | |||
| 170 | - | repository := &snapshotAcceptanceRepository{generations: make(map[string]ports.SnapshotGeneration), publishErr: test.publish, loadErr: test.load, loadMutate: test.mutate} | |||
| 171 | - | legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{legacyAcceptanceBlob(t, snapshot)}} | |||
| 172 | - | d := newTestDaemon(t, nil, stubClock{}) | |||
| 173 | - | WithSnapshotRepository(repository, legacy)(d) | |||
| 174 | - | d.importLegacySnapshots(context.Background()) | |||
| 175 | - | require.Empty(t, legacy.deleteCalls) | |||
| 176 | - | require.Len(t, legacy.blobs, 1) | |||
| 177 | - | }) | |||
| 178 | - | } | |||
| 179 | - | } | |||
| 180 | - | ||||
| 181 | - | func TestImportLegacyDeleteFailureRetriesWithoutRepublishing(t *testing.T) { | |||
| 182 | - | snapshot := legacyAcceptanceSnapshot(t) | |||
| 183 | - | repository := &snapshotAcceptanceRepository{generations: make(map[string]ports.SnapshotGeneration)} | |||
| 184 | - | legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{legacyAcceptanceBlob(t, snapshot)}, deleteErr: errors.New("sync failed")} | |||
| 185 | - | d := newTestDaemon(t, nil, stubClock{}) | |||
| 186 | - | WithSnapshotRepository(repository, legacy)(d) | |||
| 187 | - | ||||
| 188 | - | d.importLegacySnapshots(context.Background()) | |||
| 189 | - | legacy.deleteErr = nil | |||
| 190 | - | d.importLegacySnapshots(context.Background()) | |||
| 191 | - | ||||
| 192 | - | require.Len(t, repository.publishes, 1, "a retry must only retry source deletion") | |||
| 193 | - | require.Equal(t, []string{snapshot.Name, snapshot.Name}, legacy.deleteCalls) | |||
| 194 | - | require.Empty(t, legacy.blobs) | |||
| 195 | - | } | |||
| 196 | - | ||||
| 197 | - | func TestImportLegacyCancellationAndPerSessionContinuation(t *testing.T) { | |||
| 198 | - | bad := legacyAcceptanceSnapshot(t) | |||
| 199 | - | good := legacyAcceptanceSnapshot(t) | |||
| 200 | - | badBlob := legacyAcceptanceBlob(t, bad) | |||
| 201 | - | badBlob.Data = []byte("not-a-v3-snapshot") | |||
| 202 | - | repository := &snapshotAcceptanceRepository{generations: make(map[string]ports.SnapshotGeneration)} | |||
| 203 | - | legacy := &snapshotAcceptanceLegacySource{blobs: []ports.LegacySnapshot{badBlob, legacyAcceptanceBlob(t, good)}} | |||
| 204 | - | d := newTestDaemon(t, nil, stubClock{}) | |||
| 205 | - | WithSnapshotRepository(repository, legacy)(d) | |||
| 206 | - | ||||
| 207 | - | d.importLegacySnapshots(context.Background()) | |||
| 208 | - | require.Len(t, repository.publishes, 1, "a corrupt session must not block later sessions") | |||
| 209 | - | require.Equal(t, []string{good.Name}, legacy.deleteCalls) | |||
| 210 | - | ||||
| 211 | - | cancelled, cancel := context.WithCancel(context.Background()) | |||
| 212 | - | cancel() | |||
| 213 | - | d.importLegacySnapshots(cancelled) | |||
| 214 | - | require.Equal(t, 1, legacy.loadCalls, "a canceled import must not touch the legacy source") | |||
| 215 | - | } | |||
M internal/usecase/daemon/snapshot_incremental.go +4 -2
| @@ -3,6 +3,7 @@ package daemon | |||||
| 3 | import ( | 3 | import ( | ||
| 4 | "fmt" | 4 | "fmt" | ||
| 5 | 5 | ||||
| 6 | + | "github.com/bnema/vev/internal/domain" | |||
| 6 | "github.com/bnema/vev/internal/ports" | 7 | "github.com/bnema/vev/internal/ports" | ||
| 7 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | 8 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 8 | "github.com/bnema/vev/pkg/vt" | 9 | "github.com/bnema/vev/pkg/vt" | ||
| @@ -113,7 +114,7 @@ func (d *Daemon) incrementalPublication(capture *snapshotCapture) (ports.Snapsho | |||||
| 113 | if err := prepareSnapshotChunkCache(capture); err != nil { | 114 | if err := prepareSnapshotChunkCache(capture); err != nil { | ||
| 114 | return ports.SnapshotPublication{}, err | 115 | return ports.SnapshotPublication{}, err | ||
| 115 | } | 116 | } | ||
| 116 | - | manifest := snapcodec.Manifest{Generation: capture.generation, Name: capture.name, CreatedAt: capture.createdAt, Active: capture.active, Tabs: make([]snapcodec.ManifestTab, 0, len(capture.tabs))} | 117 | + | manifest := snapcodec.Manifest{Generation: capture.generation, IncarnationID: capture.incarnation, ParentCheckpoint: capture.parentCheckpoint, Name: capture.name, CreatedAt: capture.createdAt, Active: capture.active, Tabs: make([]snapcodec.ManifestTab, 0, len(capture.tabs))} |
| 117 | objects := make([]ports.SnapshotObject, 0) | 118 | objects := make([]ports.SnapshotObject, 0) | ||
| 118 | capture.sealedRefs = make(map[*vt.HistoryChunk]snapcodec.ObjectRef) | 119 | capture.sealedRefs = make(map[*vt.HistoryChunk]snapcodec.ObjectRef) | ||
| 119 | for _, tab := range capture.tabs { | 120 | for _, tab := range capture.tabs { | ||
| @@ -160,7 +161,8 @@ func (d *Daemon) incrementalPublication(capture *snapshotCapture) (ports.Snapsho | |||||
| 160 | if err != nil { | 161 | if err != nil { | ||
| 161 | return ports.SnapshotPublication{}, err | 162 | return ports.SnapshotPublication{}, err | ||
| 162 | } | 163 | } | ||
| 163 | - | return ports.SnapshotPublication{Name: capture.name, Generation: capture.generation, Manifest: encoded, Objects: objects}, nil | 164 | + | capture.checkpoint = domain.CheckpointRef{Generation: capture.generation, ManifestDigest: snapcodec.ManifestDigest(encoded)} |
| 165 | + | return ports.SnapshotPublication{IncarnationID: capture.incarnation, Name: capture.name, Generation: capture.generation, ParentCheckpoint: capture.parentCheckpoint, Manifest: encoded, Objects: objects}, nil | |||
| 164 | } | 166 | } | ||
| 165 | 167 | ||||
| 166 | // marshalSnapshotTail selects the canonical empty-tail encoding before any | 168 | // marshalSnapshotTail selects the canonical empty-tail encoding before any | ||
M internal/usecase/daemon/snapshot_incremental_test.go +8 -1
| @@ -6,6 +6,7 @@ import ( | |||||
| 6 | 6 | ||||
| 7 | "github.com/stretchr/testify/require" | 7 | "github.com/stretchr/testify/require" | ||
| 8 | 8 | ||||
| 9 | + | "github.com/bnema/vev/internal/domain" | |||
| 9 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | 10 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 10 | "github.com/bnema/vev/pkg/renderer" | 11 | "github.com/bnema/vev/pkg/renderer" | ||
| 11 | "github.com/bnema/vev/pkg/vt" | 12 | "github.com/bnema/vev/pkg/vt" | ||
| @@ -124,12 +125,14 @@ func TestIncrementalPublicationReusesSealedChunkObject(t *testing.T) { | |||||
| 124 | visible := vt.NewScreen(1, 1).PrimaryVisibleSnapshot() | 125 | visible := vt.NewScreen(1, 1).PrimaryVisibleSnapshot() | ||
| 125 | d := New(nil, nil, nil) | 126 | d := New(nil, nil, nil) | ||
| 126 | sess := newSnapshotTestSession(t, "work", false, "/work") | 127 | sess := newSnapshotTestSession(t, "work", false, "/work") | ||
| 127 | - | capture := &snapshotCapture{session: sess, name: "work", generation: 1, tabs: []snapshotCaptureTab{{stableID: "t", cols: 1, rows: 1, panes: []snapshotCapturePane{{id: "p", stableID: "p", sealed: view, tail: view.Tail(), visible: visible}}}}} | 128 | + | capture := &snapshotCapture{session: sess, name: "work", incarnation: domain.IncarnationID{1}, generation: 1, tabs: []snapshotCaptureTab{{stableID: "t", cols: 1, rows: 1, panes: []snapshotCapturePane{{id: "p", stableID: "p", sealed: view, tail: view.Tail(), visible: visible}}}}} |
| 128 | first, err := d.incrementalPublication(capture) | 129 | first, err := d.incrementalPublication(capture) | ||
| 129 | if err != nil { | 130 | if err != nil { | ||
| 130 | t.Fatal(err) | 131 | t.Fatal(err) | ||
| 131 | } | 132 | } | ||
| 133 | + | firstRef := domain.CheckpointRef{Generation: first.Generation, ManifestDigest: snapcodec.ManifestDigest(first.Manifest)} | |||
| 132 | capture.generation = 2 | 134 | capture.generation = 2 | ||
| 135 | + | capture.parentCheckpoint = &firstRef | |||
| 133 | second, err := d.incrementalPublication(capture) | 136 | second, err := d.incrementalPublication(capture) | ||
| 134 | if err != nil { | 137 | if err != nil { | ||
| 135 | t.Fatal(err) | 138 | t.Fatal(err) | ||
| @@ -137,6 +140,10 @@ func TestIncrementalPublicationReusesSealedChunkObject(t *testing.T) { | |||||
| 137 | if len(first.Objects) == 0 || len(second.Objects) == 0 { | 140 | if len(first.Objects) == 0 || len(second.Objects) == 0 { | ||
| 138 | t.Fatal("publication omitted required objects") | 141 | t.Fatal("publication omitted required objects") | ||
| 139 | } | 142 | } | ||
| 143 | + | require.Equal(t, &firstRef, second.ParentCheckpoint) | |||
| 144 | + | manifest, err := snapcodec.UnmarshalManifest(second.Manifest) | |||
| 145 | + | require.NoError(t, err) | |||
| 146 | + | require.Equal(t, &firstRef, manifest.ParentCheckpoint) | |||
| 140 | if got, want := sess.snapshotChunkCache.used, sess.snapshotChunkCache.limit; got > want { | 147 | if got, want := sess.snapshotChunkCache.used, sess.snapshotChunkCache.limit; got > want { | ||
| 141 | t.Fatalf("cache bytes = %d, limit = %d", got, want) | 148 | t.Fatalf("cache bytes = %d, limit = %d", got, want) | ||
| 142 | } | 149 | } | ||
M internal/usecase/daemon/snapshot_queue.go +17 -1
| @@ -1,6 +1,10 @@ | |||||
| 1 | package daemon | 1 | package daemon | ||
| 2 | 2 | ||||
| 3 | - | import "context" | 3 | + | import ( |
| 4 | + | "context" | |||
| 5 | + | ||||
| 6 | + | "github.com/bnema/vev/internal/domain" | |||
| 7 | + | ) | |||
| 4 | 8 | ||||
| 5 | func (d *Daemon) scheduleSnapshot(sess *session) bool { | 9 | func (d *Daemon) scheduleSnapshot(sess *session) bool { | ||
| 6 | return d.scheduleSnapshotWithFinalFallback(sess, false) | 10 | return d.scheduleSnapshotWithFinalFallback(sess, false) | ||
| @@ -54,6 +58,11 @@ func (d *Daemon) scheduleSnapshotWithFinalFallback(sess *session, final bool) bo | |||||
| 54 | sess.snapshotGeneration = mutationRevision | 58 | sess.snapshotGeneration = mutationRevision | ||
| 55 | } | 59 | } | ||
| 56 | publicationContext := sess.snapshotPublicationContext | 60 | publicationContext := sess.snapshotPublicationContext | ||
| 61 | + | var parentCheckpoint *domain.CheckpointRef | |||
| 62 | + | if sess.snapshotPublishedCheckpoint != nil { | |||
| 63 | + | parent := *sess.snapshotPublishedCheckpoint | |||
| 64 | + | parentCheckpoint = &parent | |||
| 65 | + | } | |||
| 57 | sess.snapshotPendingCaptures++ | 66 | sess.snapshotPendingCaptures++ | ||
| 58 | sess.snapshotPending = true | 67 | sess.snapshotPending = true | ||
| 59 | sess.signalSnapshotChangedLocked() | 68 | sess.signalSnapshotChangedLocked() | ||
| @@ -64,6 +73,7 @@ func (d *Daemon) scheduleSnapshotWithFinalFallback(sess *session, final bool) bo | |||||
| 64 | capture.mutationRevision = mutationRevision | 73 | capture.mutationRevision = mutationRevision | ||
| 65 | capture.attemptKind = kind | 74 | capture.attemptKind = kind | ||
| 66 | capture.publicationContext = publicationContext | 75 | capture.publicationContext = publicationContext | ||
| 76 | + | capture.parentCheckpoint = parentCheckpoint | |||
| 67 | } | 77 | } | ||
| 68 | if !ok { | 78 | if !ok { | ||
| 69 | d.finishSnapshotCapture(capture, false) | 79 | d.finishSnapshotCapture(capture, false) | ||
| @@ -97,10 +107,16 @@ func (d *Daemon) enqueueSnapshotCapture(capture *snapshotCapture) bool { | |||||
| 97 | if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | 107 | if d.snapshotWorkerClosing || d.snapshotWorkerCancel == nil || d.snapshotWorkerCtx == nil || d.snapshotWorkerCtx.Err() != nil { | ||
| 98 | return false | 108 | return false | ||
| 99 | } | 109 | } | ||
| 110 | + | capture.normalWorkerAdmitted = true | |||
| 100 | select { | 111 | select { | ||
| 101 | case d.snapshotJobs <- capture: | 112 | case d.snapshotJobs <- capture: | ||
| 113 | + | if d.snapshotAdmitted == nil { | |||
| 114 | + | d.snapshotAdmitted = make(map[*snapshotCapture]struct{}) | |||
| 115 | + | } | |||
| 116 | + | d.snapshotAdmitted[capture] = struct{}{} | |||
| 102 | return true | 117 | return true | ||
| 103 | default: | 118 | default: | ||
| 119 | + | capture.normalWorkerAdmitted = false | |||
| 104 | return false | 120 | return false | ||
| 105 | } | 121 | } | ||
| 106 | } | 122 | } | ||
M internal/usecase/daemon/snapshot_restore.go +77 -3
| @@ -2,23 +2,68 @@ package daemon | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | + | "crypto/rand" | |||
| 6 | + | "errors" | |||
| 5 | "fmt" | 7 | "fmt" | ||
| 6 | "math" | 8 | "math" | ||
| 7 | 9 | ||||
| 8 | "github.com/bnema/vev/internal/domain" | 10 | "github.com/bnema/vev/internal/domain" | ||
| 11 | + | "github.com/bnema/vev/internal/ports" | |||
| 9 | "github.com/bnema/vev/internal/usecase/layout" | 12 | "github.com/bnema/vev/internal/usecase/layout" | ||
| 10 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | 13 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 11 | "github.com/bnema/vev/pkg/vt" | 14 | "github.com/bnema/vev/pkg/vt" | ||
| 12 | ) | 15 | ) | ||
| 13 | 16 | ||||
| 14 | func (d *Daemon) closeRestoreDone() { | 17 | func (d *Daemon) closeRestoreDone() { | ||
| 18 | + | d.mu.Lock() | |||
| 19 | + | for name, entry := range d.stopped { | |||
| 20 | + | if entry.restoreDone == nil { | |||
| 21 | + | continue | |||
| 22 | + | } | |||
| 23 | + | select { | |||
| 24 | + | case <-entry.restoreDone: | |||
| 25 | + | continue | |||
| 26 | + | default: | |||
| 27 | + | } | |||
| 28 | + | entry.state = ports.SessionBroken | |||
| 29 | + | if entry.record.DegradedReason == "" { | |||
| 30 | + | entry.record.DegradedReason = "restore unavailable" | |||
| 31 | + | } | |||
| 32 | + | d.stopped[name] = entry | |||
| 33 | + | closeRuntimeRestoreDoneLocked(entry.restoreDone) | |||
| 34 | + | } | |||
| 35 | + | d.mu.Unlock() | |||
| 15 | d.restoreOnce.Do(func() { close(d.restoreDone) }) | 36 | d.restoreOnce.Do(func() { close(d.restoreDone) }) | ||
| 16 | } | 37 | } | ||
| 17 | 38 | ||||
| 39 | + | // startSnapshotRestoration launches catalogue-driven restoration as a durable | |||
| 40 | + | // writer. Restoration reconciles committed checkpoints, so lifecycle ownership | |||
| 41 | + | // must outlive it exactly as it outlives the snapshot and maintenance workers. The completion | |||
| 42 | + | // channel is registered before the goroutine starts, so shutdown can never | |||
| 43 | + | // begin waiting while this writer is still unregistered. | |||
| 44 | + | func (d *Daemon) startSnapshotRestoration() { | |||
| 45 | + | if d == nil { | |||
| 46 | + | return | |||
| 47 | + | } | |||
| 48 | + | d.snapshotWorkerMu.Lock() | |||
| 49 | + | if d.restoreWorkerDone != nil { | |||
| 50 | + | d.snapshotWorkerMu.Unlock() | |||
| 51 | + | return | |||
| 52 | + | } | |||
| 53 | + | done := make(chan struct{}) | |||
| 54 | + | d.restoreWorkerDone = done | |||
| 55 | + | d.snapshotWorkerMu.Unlock() | |||
| 56 | + | d.sessWg.Go(func() { | |||
| 57 | + | defer close(done) | |||
| 58 | + | d.restoreSnapshots(d.serveCtx) | |||
| 59 | + | }) | |||
| 60 | + | } | |||
| 61 | + | ||||
| 18 | func (d *Daemon) restoreSnapshots(ctx context.Context) { | 62 | func (d *Daemon) restoreSnapshots(ctx context.Context) { | ||
| 19 | if d == nil { | 63 | if d == nil { | ||
| 20 | return | 64 | return | ||
| 21 | } | 65 | } | ||
| 66 | + | defer d.logStartupRecoveryCounts(0) | |||
| 22 | defer d.closeRestoreDone() | 67 | defer d.closeRestoreDone() | ||
| 23 | if ns := d.noticeStore; ns != nil { | 68 | if ns := d.noticeStore; ns != nil { | ||
| 24 | claimed, err := ns.Claim() | 69 | claimed, err := ns.Claim() | ||
| @@ -64,7 +109,7 @@ func validateRestoreSessionSnapshot(snap snapcodec.Session) error { | |||||
| 64 | 109 | ||||
| 65 | // restoreSession owns the restored tabs until registration succeeds. Every | 110 | // restoreSession owns the restored tabs until registration succeeds. Every | ||
| 66 | // unsuccessful path closes their PTYs and cancels their shared session context. | 111 | // unsuccessful path closes their PTYs and cancels their shared session context. | ||
| 67 | - | func (d *Daemon) restoreSession(ctx context.Context, snap snapcodec.Session, repositoryGeneration uint64) error { | 112 | + | func (d *Daemon) restoreSession(ctx context.Context, snap snapcodec.Session, repositoryGeneration uint64, checkpoint domain.CheckpointRef) error { |
| 68 | if err := ctx.Err(); err != nil { | 113 | if err := ctx.Err(); err != nil { | ||
| 69 | return err | 114 | return err | ||
| 70 | } | 115 | } | ||
| @@ -101,6 +146,7 @@ func (d *Daemon) restoreSession(ctx context.Context, snap snapcodec.Session, rep | |||||
| 101 | // The loaded manifest is the repository head for this name. Future dirty | 146 | // The loaded manifest is the repository head for this name. Future dirty | ||
| 102 | // checkpoints must continue from it rather than reuse generation one. | 147 | // checkpoints must continue from it rather than reuse generation one. | ||
| 103 | sess.snapshotPublishedGeneration = repositoryGeneration | 148 | sess.snapshotPublishedGeneration = repositoryGeneration | ||
| 149 | + | sess.snapshotPublishedCheckpoint = &checkpoint | |||
| 104 | registered, err := d.persistAndRegisterRestoredSession(ctx, sess) | 150 | registered, err := d.persistAndRegisterRestoredSession(ctx, sess) | ||
| 105 | if err != nil || !registered { | 151 | if err != nil || !registered { | ||
| 106 | return err | 152 | return err | ||
| @@ -253,8 +299,32 @@ func (d *Daemon) newRestoredSession(snap snapcodec.Session, sctx context.Context | |||||
| 253 | // persistAndRegisterRestoredSession atomically transfers a successfully | 299 | // persistAndRegisterRestoredSession atomically transfers a successfully | ||
| 254 | // persisted session to the daemon. The caller retains cleanup on false/error. | 300 | // persisted session to the daemon. The caller retains cleanup on false/error. | ||
| 255 | func (d *Daemon) persistAndRegisterRestoredSession(ctx context.Context, sess *session) (bool, error) { | 301 | func (d *Daemon) persistAndRegisterRestoredSession(ctx context.Context, sess *session) (bool, error) { | ||
| 256 | - | if err := d.persist.Save(sess.persistRecordLocked(sess.createdAt)); err != nil { | 302 | + | if sess.incarnation == (domain.IncarnationID{}) { |
| 257 | - | return false, err | 303 | + | d.mu.Lock() |
| 304 | + | stopped := d.stopped[sess.name] | |||
| 305 | + | d.mu.Unlock() | |||
| 306 | + | sess.incarnation = stopped.incarnation | |||
| 307 | + | if sess.incarnation == (domain.IncarnationID{}) { | |||
| 308 | + | var err error | |||
| 309 | + | sess.incarnation, err = domain.NewIncarnationID(rand.Reader) | |||
| 310 | + | if err != nil { | |||
| 311 | + | return false, fmt.Errorf("snapshot: generate durable identity: %w", err) | |||
| 312 | + | } | |||
| 313 | + | } | |||
| 314 | + | } | |||
| 315 | + | // Restoration is catalogue-authorized. Missing or unreadable authority is a | |||
| 316 | + | // hard failure. | |||
| 317 | + | if d.persistEnabled { | |||
| 318 | + | record, ok, err := d.catalogueRecord(sess.name) | |||
| 319 | + | if err != nil { | |||
| 320 | + | return false, err | |||
| 321 | + | } | |||
| 322 | + | if !ok { | |||
| 323 | + | return false, errors.New("snapshot: restored session is absent from catalogue") | |||
| 324 | + | } | |||
| 325 | + | if record.IncarnationID != sess.incarnation { | |||
| 326 | + | return false, errors.New("snapshot: restored session incarnation differs from catalogue") | |||
| 327 | + | } | |||
| 258 | } | 328 | } | ||
| 259 | d.mu.Lock() | 329 | d.mu.Lock() | ||
| 260 | defer d.mu.Unlock() | 330 | defer d.mu.Unlock() | ||
| @@ -263,6 +333,10 @@ func (d *Daemon) persistAndRegisterRestoredSession(ctx context.Context, sess *se | |||||
| 263 | } | 333 | } | ||
| 264 | sess.id = domain.SessionID(fmt.Sprintf("sess-%d", d.nextID)) | 334 | sess.id = domain.SessionID(fmt.Sprintf("sess-%d", d.nextID)) | ||
| 265 | d.nextID++ | 335 | d.nextID++ | ||
| 336 | + | stopped := d.stopped[sess.name] | |||
| 337 | + | sess.mu.Lock() | |||
| 338 | + | sess.restoreDone = stopped.restoreDone | |||
| 339 | + | sess.mu.Unlock() | |||
| 266 | delete(d.stopped, sess.name) | 340 | delete(d.stopped, sess.name) | ||
| 267 | d.sessions[sess.id] = sess | 341 | d.sessions[sess.id] = sess | ||
| 268 | sess.snapDirty.Store(false) | 342 | sess.snapDirty.Store(false) | ||
R internal/usecase/daemon/snapshot_legacy_import.go → internal/usecase/daemon/snapshot_restore_fixture_test.go +10 -7 renamed, 54% similar
| @@ -3,12 +3,15 @@ package daemon | |||||
| 3 | import ( | 3 | import ( | ||
| 4 | "fmt" | 4 | "fmt" | ||
| 5 | 5 | ||||
| 6 | + | "github.com/bnema/vev/internal/domain" | |||
| 6 | "github.com/bnema/vev/internal/ports" | 7 | "github.com/bnema/vev/internal/ports" | ||
| 7 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | 8 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 8 | ) | 9 | ) | ||
| 9 | 10 | ||||
| 10 | - | func legacyPublication(snapshot snapcodec.Session) (ports.SnapshotPublication, error) { | 11 | + | func acceptancePublication(snapshot snapcodec.Session) (ports.SnapshotPublication, error) { |
| 11 | - | manifest := snapcodec.Manifest{Generation: 1, Name: snapshot.Name, CreatedAt: snapshot.CreatedAt, Active: snapshot.Active, Tabs: make([]snapcodec.ManifestTab, 0, len(snapshot.Tabs))} | 12 | + | // acceptancePublication converts a restore fixture to repository form. |
| 13 | + | incarnation := domain.IncarnationID{1} | |||
| 14 | + | manifest := snapcodec.Manifest{Generation: 1, IncarnationID: incarnation, Name: snapshot.Name, CreatedAt: snapshot.CreatedAt, Active: snapshot.Active, Tabs: make([]snapcodec.ManifestTab, 0, len(snapshot.Tabs))} | |||
| 12 | objects := make([]ports.SnapshotObject, 0) | 15 | objects := make([]ports.SnapshotObject, 0) | ||
| 13 | for _, tab := range snapshot.Tabs { | 16 | for _, tab := range snapshot.Tabs { | ||
| 14 | outTab := snapcodec.ManifestTab{StableID: tab.StableID, Cols: tab.Cols, Rows: tab.Rows, NextPaneID: tab.NextPaneID, Focus: tab.Focus, Tree: tab.Tree, Panes: make([]snapcodec.ManifestPane, 0, len(tab.Panes))} | 17 | outTab := snapcodec.ManifestTab{StableID: tab.StableID, Cols: tab.Cols, Rows: tab.Rows, NextPaneID: tab.NextPaneID, Focus: tab.Focus, Tree: tab.Tree, Panes: make([]snapcodec.ManifestPane, 0, len(tab.Panes))} | ||
| @@ -17,18 +20,18 @@ func legacyPublication(snapshot snapcodec.Session) (ports.SnapshotPublication, e | |||||
| 17 | for i, payload := range pane.SealedChunks { | 20 | for i, payload := range pane.SealedChunks { | ||
| 18 | object, err := snapcodec.MarshalObject(snapcodec.HistoryChunk, payload) | 21 | object, err := snapcodec.MarshalObject(snapcodec.HistoryChunk, payload) | ||
| 19 | if err != nil { | 22 | if err != nil { | ||
| 20 | - | return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q sealed chunk %d: marshal object: %w", tab.StableID, pane.ID, i, err) | 23 | + | return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q sealed chunk %d: marshal object: %w", tab.StableID, pane.ID, i, err) |
| 21 | } | 24 | } | ||
| 22 | outPane.Sealed = append(outPane.Sealed, objectRef(snapcodec.HistoryChunk, object)) | 25 | outPane.Sealed = append(outPane.Sealed, objectRef(snapcodec.HistoryChunk, object)) | ||
| 23 | objects = append(objects, object) | 26 | objects = append(objects, object) | ||
| 24 | } | 27 | } | ||
| 25 | tail, err := snapcodec.MarshalObject(snapcodec.HistoryTail, pane.Tail) | 28 | tail, err := snapcodec.MarshalObject(snapcodec.HistoryTail, pane.Tail) | ||
| 26 | if err != nil { | 29 | if err != nil { | ||
| 27 | - | return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q tail: marshal object: %w", tab.StableID, pane.ID, err) | 30 | + | return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q tail: marshal object: %w", tab.StableID, pane.ID, err) |
| 28 | } | 31 | } | ||
| 29 | visible, err := snapcodec.MarshalObject(snapcodec.Visible, pane.Visible) | 32 | visible, err := snapcodec.MarshalObject(snapcodec.Visible, pane.Visible) | ||
| 30 | if err != nil { | 33 | if err != nil { | ||
| 31 | - | return ports.SnapshotPublication{}, fmt.Errorf("snapshot legacy import: tab %q pane %q visible: marshal object: %w", tab.StableID, pane.ID, err) | 34 | + | return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: tab %q pane %q visible: marshal object: %w", tab.StableID, pane.ID, err) |
| 32 | } | 35 | } | ||
| 33 | outPane.Tail, outPane.Visible = objectRef(snapcodec.HistoryTail, tail), objectRef(snapcodec.Visible, visible) | 36 | outPane.Tail, outPane.Visible = objectRef(snapcodec.HistoryTail, tail), objectRef(snapcodec.Visible, visible) | ||
| 34 | objects = append(objects, tail, visible) | 37 | objects = append(objects, tail, visible) | ||
| @@ -38,7 +41,7 @@ func legacyPublication(snapshot snapcodec.Session) (ports.SnapshotPublication, e | |||||
| 38 | } | 41 | } | ||
| 39 | encoded, err := snapcodec.MarshalManifest(manifest) | 42 | encoded, err := snapcodec.MarshalManifest(manifest) | ||
| 40 | if err != nil { | 43 | if err != nil { | ||
| 41 | - | return ports.SnapshotPublication{}, fmt.Errorf("snapshot: marshal legacy import manifest: %w", err) | 44 | + | return ports.SnapshotPublication{}, fmt.Errorf("snapshot fixture conversion: marshal manifest: %w", err) |
| 42 | } | 45 | } | ||
| 43 | - | return ports.SnapshotPublication{Name: snapshot.Name, Generation: 1, Manifest: encoded, Objects: objects}, nil | 46 | + | return ports.SnapshotPublication{IncarnationID: incarnation, Name: snapshot.Name, Generation: 1, Manifest: encoded, Objects: objects}, nil |
| 44 | } | 47 | } | ||
M internal/usecase/daemon/snapshot_restore_incremental.go +228 -143
| @@ -1,55 +1,254 @@ | |||||
| 1 | package daemon | 1 | package daemon | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | - | "bytes" | |||
| 5 | "context" | 4 | "context" | ||
| 6 | "crypto/sha256" | 5 | "crypto/sha256" | ||
| 6 | + | "errors" | |||
| 7 | "fmt" | 7 | "fmt" | ||
| 8 | + | "net" | |||
| 9 | + | "sync" | |||
| 10 | + | "syscall" | |||
| 8 | 11 | ||||
| 9 | "github.com/bnema/vev/internal/domain" | 12 | "github.com/bnema/vev/internal/domain" | ||
| 10 | "github.com/bnema/vev/internal/ports" | 13 | "github.com/bnema/vev/internal/ports" | ||
| 11 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | 14 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 12 | ) | 15 | ) | ||
| 13 | 16 | ||||
| 14 | - | // restoreIncrementalSnapshots imports v3 only before listing repository | 17 | + | const catalogueRestoreConcurrency = 8 |
| 15 | - | // sessions, then restores each independently. A corrupt session therefore | 18 | + | |
| 16 | - | // cannot prevent an unrelated saved session from starting. | 19 | + | var errRetryableRestoreLoad = errors.New("snapshot: retryable checkpoint load") |
| 20 | + | ||||
| 21 | + | func initialSessionState(record domain.CatalogueRecord) (ports.SessionState, chan struct{}) { | |||
| 22 | + | done := make(chan struct{}) | |||
| 23 | + | state := ports.SessionStopped | |||
| 24 | + | if record.DegradedReason != "" { | |||
| 25 | + | state = ports.SessionBroken | |||
| 26 | + | close(done) | |||
| 27 | + | } else if record.Committed == nil { | |||
| 28 | + | close(done) | |||
| 29 | + | } | |||
| 30 | + | return state, done | |||
| 31 | + | } | |||
| 32 | + | ||||
| 33 | + | // restoreIncrementalSnapshots restores only records named by the authoritative | |||
| 34 | + | // catalogue. Snapshot repository discovery is deliberately not part of this | |||
| 35 | + | // path, so stale or oversized namespaces cannot affect unrelated sessions. | |||
| 17 | func (d *Daemon) restoreIncrementalSnapshots(ctx context.Context) { | 36 | func (d *Daemon) restoreIncrementalSnapshots(ctx context.Context) { | ||
| 18 | - | d.importLegacySnapshots(ctx) | 37 | + | if d.catalogue == nil { |
| 19 | - | names, err := d.snapshotRepository.List(ctx) | 38 | + | return |
| 39 | + | } | |||
| 40 | + | records, err := d.catalogue.Records() | |||
| 20 | if err != nil { | 41 | if err != nil { | ||
| 21 | - | d.log.Warn("listing session snapshots failed", "err", err) | 42 | + | d.log.Error("loading catalogue for snapshot restoration failed", "err", err) |
| 22 | - | d.NotifyGlobal(domain.NoticeError, domain.NoticeSnapshotRestore, "couldn't load saved sessions after restart", err) | |||
| 23 | return | 43 | return | ||
| 24 | } | 44 | } | ||
| 25 | - | for _, name := range names { | 45 | + | d.restoreCatalogue(ctx, records) |
| 26 | - | if err := ctx.Err(); err != nil { | 46 | + | } |
| 27 | - | return | 47 | + | |
| 48 | + | // restoreCatalogue gives every expected record an independent bounded job. | |||
| 49 | + | // A failed record is degraded without cancelling any sibling restoration. | |||
| 50 | + | func (d *Daemon) restoreCatalogue(ctx context.Context, records []domain.CatalogueRecord) { | |||
| 51 | + | jobs := make(chan domain.CatalogueRecord) | |||
| 52 | + | workers := min(catalogueRestoreConcurrency, len(records)) | |||
| 53 | + | var wg sync.WaitGroup | |||
| 54 | + | for range workers { | |||
| 55 | + | wg.Go(func() { | |||
| 56 | + | for record := range jobs { | |||
| 57 | + | done := d.ensureCatalogueRegistryEntry(record) | |||
| 58 | + | err := d.restoreRecord(ctx, record) | |||
| 59 | + | d.finishRecordRestore(record, err, done) | |||
| 60 | + | } | |||
| 61 | + | }) | |||
| 62 | + | } | |||
| 63 | + | for _, record := range records { | |||
| 64 | + | jobs <- record | |||
| 65 | + | } | |||
| 66 | + | close(jobs) | |||
| 67 | + | wg.Wait() | |||
| 68 | + | } | |||
| 69 | + | ||||
| 70 | + | func stoppedSessionFromRecord(record domain.CatalogueRecord, state ports.SessionState, done chan struct{}) stoppedSession { | |||
| 71 | + | return stoppedSession{ | |||
| 72 | + | name: record.Name, | |||
| 73 | + | cwd: record.Cwd, | |||
| 74 | + | createdAt: record.CreatedAt, | |||
| 75 | + | incarnation: record.IncarnationID, | |||
| 76 | + | lastUsedSeq: record.LastUsedSeq, | |||
| 77 | + | tabNames: append([]string(nil), record.TabNames...), | |||
| 78 | + | record: record, | |||
| 79 | + | state: state, | |||
| 80 | + | restoreDone: done, | |||
| 81 | + | } | |||
| 82 | + | } | |||
| 83 | + | ||||
| 84 | + | func (d *Daemon) ensureCatalogueRegistryEntry(record domain.CatalogueRecord) chan struct{} { | |||
| 85 | + | d.mu.Lock() | |||
| 86 | + | defer d.mu.Unlock() | |||
| 87 | + | if entry, ok := d.stopped[record.Name]; ok { | |||
| 88 | + | return entry.restoreDone | |||
| 89 | + | } | |||
| 90 | + | state, done := initialSessionState(record) | |||
| 91 | + | d.stopped[record.Name] = stoppedSessionFromRecord(record, state, done) | |||
| 92 | + | return done | |||
| 93 | + | } | |||
| 94 | + | ||||
| 95 | + | // closeRuntimeRestoreDoneLocked closes a per-record restoration barrier. | |||
| 96 | + | // Caller must hold d.mu so state transition and waiter release are atomic. | |||
| 97 | + | func closeRuntimeRestoreDoneLocked(done chan struct{}) { | |||
| 98 | + | if done == nil { | |||
| 99 | + | return | |||
| 100 | + | } | |||
| 101 | + | select { | |||
| 102 | + | case <-done: | |||
| 103 | + | default: | |||
| 104 | + | close(done) | |||
| 105 | + | } | |||
| 106 | + | } | |||
| 107 | + | ||||
| 108 | + | func (d *Daemon) setStoppedRecovery(record domain.CatalogueRecord, state ports.SessionState) { | |||
| 109 | + | d.mu.Lock() | |||
| 110 | + | defer d.mu.Unlock() | |||
| 111 | + | entry, ok := d.stopped[record.Name] | |||
| 112 | + | if !ok { | |||
| 113 | + | return | |||
| 114 | + | } | |||
| 115 | + | d.stopped[record.Name] = stoppedSessionFromRecord(record, state, entry.restoreDone) | |||
| 116 | + | } | |||
| 117 | + | ||||
| 118 | + | func (d *Daemon) finishRecordRestore(record domain.CatalogueRecord, restoreErr error, done chan struct{}) { | |||
| 119 | + | retryable := errors.Is(restoreErr, errRetryableRestoreLoad) | |||
| 120 | + | degraded := false | |||
| 121 | + | if restoreErr != nil { | |||
| 122 | + | reason := "checkpoint validation failed" | |||
| 123 | + | switch { | |||
| 124 | + | case retryable: | |||
| 125 | + | reason = "checkpoint load failed" | |||
| 126 | + | case errors.Is(restoreErr, context.Canceled) || errors.Is(restoreErr, context.DeadlineExceeded): | |||
| 127 | + | reason = "restore interrupted" | |||
| 28 | } | 128 | } | ||
| 29 | - | generation, err := d.snapshotRepository.Load(ctx, name) | 129 | + | if d.recovery == nil { |
| 30 | - | if err != nil { | 130 | + | d.log.Error("marking session degraded failed", "session", record.Name, "err", errors.New("recovery coordinator unavailable")) |
| 31 | - | d.snapshotRestoreFailure(name, err) | 131 | + | } else if current, marked, err := d.recovery.MarkBroken(context.Background(), record.Name, record.IncarnationID, reason); err != nil { |
| 32 | - | continue | 132 | + | d.log.Error("marking session degraded failed", "session", record.Name, "err", err) |
| 133 | + | } else if marked { | |||
| 134 | + | record = current | |||
| 135 | + | degraded = true | |||
| 33 | } | 136 | } | ||
| 34 | - | if generation.Fallback { | 137 | + | } |
| 35 | - | d.NotifyGlobal(domain.NoticeWarn, domain.NoticeSnapshotRestore, "restored an older saved checkpoint for session "+name, nil) | 138 | + | |
| 139 | + | d.mu.Lock() | |||
| 140 | + | if entry, ok := d.stopped[record.Name]; ok && degraded { | |||
| 141 | + | d.stopped[record.Name] = stoppedSessionFromRecord(record, ports.SessionBroken, entry.restoreDone) | |||
| 142 | + | } | |||
| 143 | + | closeRuntimeRestoreDoneLocked(done) | |||
| 144 | + | d.mu.Unlock() | |||
| 145 | + | if degraded { | |||
| 146 | + | reasonCode := "checkpoint-invalid" | |||
| 147 | + | switch { | |||
| 148 | + | case retryable: | |||
| 149 | + | reasonCode = "checkpoint-load-failed" | |||
| 150 | + | case errors.Is(restoreErr, context.Canceled) || errors.Is(restoreErr, context.DeadlineExceeded): | |||
| 151 | + | reasonCode = "restore-interrupted" | |||
| 36 | } | 152 | } | ||
| 37 | - | snapshot, err := sessionFromGeneration(generation) | 153 | + | d.logSessionDegraded(record, reasonCode) |
| 38 | - | if err != nil { | 154 | + | } |
| 39 | - | d.snapshotRestoreFailure(name, err) | 155 | + | } |
| 40 | - | continue | 156 | + | |
| 157 | + | func (d *Daemon) restoreRecord(ctx context.Context, record domain.CatalogueRecord) error { | |||
| 158 | + | if record.DegradedReason != "" { | |||
| 159 | + | d.setStoppedRecovery(record, ports.SessionBroken) | |||
| 160 | + | d.logSessionDegraded(record, "persisted-broken") | |||
| 161 | + | return nil | |||
| 162 | + | } | |||
| 163 | + | if record.Committed == nil { | |||
| 164 | + | d.setStoppedRecovery(record, ports.SessionStopped) | |||
| 165 | + | d.logSessionRestoreComplete(record, 0, false) | |||
| 166 | + | return nil | |||
| 167 | + | } | |||
| 168 | + | if d.snapshotRepository == nil { | |||
| 169 | + | return errors.New("snapshot: repository unavailable") | |||
| 170 | + | } | |||
| 171 | + | ||||
| 172 | + | if record.Committed == nil { | |||
| 173 | + | return errors.New("snapshot: healthy record has no checkpoint") | |||
| 174 | + | } | |||
| 175 | + | selected := *record.Committed | |||
| 176 | + | selectedGeneration, err := d.snapshotRepository.LoadCheckpoint(ctx, record.IncarnationID, record.Name, selected) | |||
| 177 | + | if err != nil { | |||
| 178 | + | if ctxErr := ctx.Err(); ctxErr != nil { | |||
| 179 | + | return ctxErr | |||
| 41 | } | 180 | } | ||
| 42 | - | if err := d.restoreSession(ctx, snapshot, generation.Generation); err != nil { | 181 | + | if isRetryableRestoreLoadError(err) { |
| 43 | - | if ctx.Err() == nil { | 182 | + | return fmt.Errorf("%w: %w", errRetryableRestoreLoad, err) |
| 44 | - | d.snapshotRestoreFailure(name, err) | |||
| 45 | - | } | |||
| 46 | } | 183 | } | ||
| 184 | + | d.logRejectedRestoreCandidate(record, selected, false, "load-failed", err) | |||
| 185 | + | return errors.New("snapshot: committed checkpoint did not validate") | |||
| 186 | + | } | |||
| 187 | + | selectedSnapshot, err := sessionFromGeneration(selectedGeneration) | |||
| 188 | + | if err != nil { | |||
| 189 | + | d.logRejectedRestoreCandidate(record, selected, false, "decode-failed", err) | |||
| 190 | + | return errors.New("snapshot: committed checkpoint did not validate") | |||
| 191 | + | } | |||
| 192 | + | if selectedGeneration.IncarnationID != record.IncarnationID { | |||
| 193 | + | d.logRejectedRestoreCandidate(record, selected, false, "incarnation-mismatch", nil) | |||
| 194 | + | return errors.New("snapshot: committed checkpoint did not validate") | |||
| 195 | + | } | |||
| 196 | + | if selectedGeneration.Name != record.Name { | |||
| 197 | + | d.logRejectedRestoreCandidate(record, selected, false, "name-mismatch", nil) | |||
| 198 | + | return errors.New("snapshot: committed checkpoint did not validate") | |||
| 47 | } | 199 | } | ||
| 200 | + | if selectedGeneration.Generation != selected.Generation { | |||
| 201 | + | d.logRejectedRestoreCandidate(record, selected, false, "generation-mismatch", nil) | |||
| 202 | + | return errors.New("snapshot: committed checkpoint did not validate") | |||
| 203 | + | } | |||
| 204 | + | if err := d.snapshotRepository.ReconcileCheckpoint(ctx, record.IncarnationID, selected); err != nil { | |||
| 205 | + | d.log.Warn("snapshot_checkpoint_reconciliation_pending", "session", record.Name, "incarnation", record.IncarnationID.String(), "generation", selected.Generation, "reason_code", "reconcile-failed") | |||
| 206 | + | } else { | |||
| 207 | + | d.log.Info("snapshot_checkpoint_reconciliation_complete", "session", record.Name, "incarnation", record.IncarnationID.String(), "generation", selected.Generation) | |||
| 208 | + | } | |||
| 209 | + | ||||
| 210 | + | if err := d.restoreSession(ctx, selectedSnapshot, selectedGeneration.Generation, selected); err != nil { | |||
| 211 | + | return err | |||
| 212 | + | } | |||
| 213 | + | d.setStoppedRecovery(record, ports.SessionStopped) | |||
| 214 | + | d.logSessionRestoreComplete(record, selected.Generation, false) | |||
| 215 | + | return nil | |||
| 216 | + | } | |||
| 217 | + | ||||
| 218 | + | func isRetryableRestoreLoadError(err error) bool { | |||
| 219 | + | if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { | |||
| 220 | + | return false | |||
| 221 | + | } | |||
| 222 | + | if errors.Is(err, syscall.ENOSPC) || | |||
| 223 | + | errors.Is(err, syscall.ENOMEM) || | |||
| 224 | + | errors.Is(err, syscall.EMFILE) || | |||
| 225 | + | errors.Is(err, syscall.ENFILE) || | |||
| 226 | + | errors.Is(err, syscall.EDQUOT) { | |||
| 227 | + | return true | |||
| 228 | + | } | |||
| 229 | + | if netErr, ok := errors.AsType[net.Error](err); ok && netErr.Timeout() { | |||
| 230 | + | return true | |||
| 231 | + | } | |||
| 232 | + | temporary, ok := errors.AsType[interface { | |||
| 233 | + | error | |||
| 234 | + | Temporary() bool | |||
| 235 | + | }](err) | |||
| 236 | + | return ok && temporary.Temporary() | |||
| 48 | } | 237 | } | ||
| 49 | 238 | ||||
| 50 | - | func (d *Daemon) snapshotRestoreFailure(name string, err error) { | 239 | + | func (d *Daemon) logRejectedRestoreCandidate(record domain.CatalogueRecord, candidate domain.CheckpointRef, fallback bool, reason string, err error) { |
| 51 | - | d.log.Warn("restoring session snapshot failed", "err", err, "session", name) | 240 | + | kind := "committed" |
| 52 | - | d.NotifyGlobal(domain.NoticeError, domain.NoticeSnapshotRestore, "couldn't restore session "+name+" after restart", err) | 241 | + | if fallback { |
| 242 | + | kind = "fallback" | |||
| 243 | + | } | |||
| 244 | + | d.log.Warn("snapshot_restore_candidate_rejected", | |||
| 245 | + | "session", record.Name, | |||
| 246 | + | "incarnation", record.IncarnationID.String(), | |||
| 247 | + | "generation", candidate.Generation, | |||
| 248 | + | "candidate", kind, | |||
| 249 | + | "reason_code", reason, | |||
| 250 | + | "err", err, | |||
| 251 | + | ) | |||
| 53 | } | 252 | } | ||
| 54 | 253 | ||||
| 55 | func sessionFromGeneration(generation ports.SnapshotGeneration) (snapcodec.Session, error) { | 254 | func sessionFromGeneration(generation ports.SnapshotGeneration) (snapcodec.Session, error) { | ||
| @@ -57,7 +256,7 @@ func sessionFromGeneration(generation ports.SnapshotGeneration) (snapcodec.Sessi | |||||
| 57 | if err != nil { | 256 | if err != nil { | ||
| 58 | return snapcodec.Session{}, err | 257 | return snapcodec.Session{}, err | ||
| 59 | } | 258 | } | ||
| 60 | - | if manifest.Name != generation.Name || manifest.Generation != generation.Generation { | 259 | + | if manifest.IncarnationID != generation.IncarnationID || manifest.Name != generation.Name || manifest.Generation != generation.Generation { |
| 61 | return snapcodec.Session{}, fmt.Errorf("snapshot: generation identity mismatch") | 260 | return snapcodec.Session{}, fmt.Errorf("snapshot: generation identity mismatch") | ||
| 62 | } | 261 | } | ||
| 63 | result := snapcodec.Session{Name: manifest.Name, CreatedAt: manifest.CreatedAt, Active: manifest.Active, Tabs: make([]snapcodec.Tab, 0, len(manifest.Tabs))} | 262 | result := snapcodec.Session{Name: manifest.Name, CreatedAt: manifest.CreatedAt, Active: manifest.Active, Tabs: make([]snapcodec.Tab, 0, len(manifest.Tabs))} | ||
| @@ -99,117 +298,3 @@ func generationObject(generation ports.SnapshotGeneration, ref snapcodec.ObjectR | |||||
| 99 | } | 298 | } | ||
| 100 | return append([]byte(nil), payload...), nil | 299 | return append([]byte(nil), payload...), nil | ||
| 101 | } | 300 | } | ||
| 102 | - | ||||
| 103 | - | // importLegacySnapshots is intentionally best effort per session. It deletes | |||
| 104 | - | // a legacy blob only after publication and a full repository reload verify the | |||
| 105 | - | // exact converted session, leaving failures retryable on the next startup. | |||
| 106 | - | func (d *Daemon) importLegacySnapshots(ctx context.Context) { | |||
| 107 | - | if d.legacySnapshots == nil || d.snapshotRepository == nil || ctx.Err() != nil { | |||
| 108 | - | return | |||
| 109 | - | } | |||
| 110 | - | existing, err := d.snapshotRepository.List(ctx) | |||
| 111 | - | if err != nil { | |||
| 112 | - | d.log.Warn("listing incremental snapshots before import failed", "err", err) | |||
| 113 | - | return | |||
| 114 | - | } | |||
| 115 | - | present := make(map[string]struct{}, len(existing)) | |||
| 116 | - | for _, name := range existing { | |||
| 117 | - | present[name] = struct{}{} | |||
| 118 | - | } | |||
| 119 | - | legacy, err := d.legacySnapshots.LoadLegacy(ctx) | |||
| 120 | - | if err != nil { | |||
| 121 | - | d.log.Warn("loading legacy snapshots failed", "err", err) | |||
| 122 | - | return | |||
| 123 | - | } | |||
| 124 | - | for _, blob := range legacy { | |||
| 125 | - | if ctx.Err() != nil { | |||
| 126 | - | return | |||
| 127 | - | } | |||
| 128 | - | if _, ok := present[blob.Name]; ok { | |||
| 129 | - | d.retryLegacyDelete(ctx, blob) | |||
| 130 | - | continue | |||
| 131 | - | } | |||
| 132 | - | snapshot, err := snapcodec.Unmarshal(blob.Data) | |||
| 133 | - | if err != nil || snapshot.Name != blob.Name { | |||
| 134 | - | d.log.Warn("importing legacy snapshot failed", "session", blob.Name, "err", err) | |||
| 135 | - | continue | |||
| 136 | - | } | |||
| 137 | - | publication, err := legacyPublication(snapshot) | |||
| 138 | - | if err == nil { | |||
| 139 | - | err = d.snapshotRepository.Publish(ctx, publication) | |||
| 140 | - | } | |||
| 141 | - | if err == nil { | |||
| 142 | - | var generation ports.SnapshotGeneration | |||
| 143 | - | generation, err = d.snapshotRepository.Load(ctx, snapshot.Name) | |||
| 144 | - | if err == nil { | |||
| 145 | - | err = verifyLegacyImportGeneration(generation, publication) | |||
| 146 | - | } | |||
| 147 | - | } | |||
| 148 | - | if err != nil { | |||
| 149 | - | d.log.Warn("importing legacy snapshot failed", "session", blob.Name, "err", err) | |||
| 150 | - | continue | |||
| 151 | - | } | |||
| 152 | - | d.rememberLegacyDelete(blob) | |||
| 153 | - | if err := d.legacySnapshots.DeleteVerifiedLegacy(ctx, blob); err != nil { | |||
| 154 | - | d.log.Warn("deleting imported legacy snapshot failed", "session", blob.Name, "err", err) | |||
| 155 | - | continue | |||
| 156 | - | } | |||
| 157 | - | d.forgetLegacyDelete(blob.Name) | |||
| 158 | - | present[blob.Name] = struct{}{} | |||
| 159 | - | } | |||
| 160 | - | } | |||
| 161 | - | ||||
| 162 | - | func verifyLegacyImportGeneration(generation ports.SnapshotGeneration, publication ports.SnapshotPublication) error { | |||
| 163 | - | if generation.Name != publication.Name || generation.Generation != publication.Generation || !bytes.Equal(generation.Manifest, publication.Manifest) { | |||
| 164 | - | return fmt.Errorf("snapshot: legacy import verification mismatch") | |||
| 165 | - | } | |||
| 166 | - | expected := make(map[ports.SnapshotDigest][]byte, len(publication.Objects)) | |||
| 167 | - | for _, object := range publication.Objects { | |||
| 168 | - | if sha256.Sum256(object.Data) != object.Digest { | |||
| 169 | - | return fmt.Errorf("snapshot: legacy import publication digest mismatch") | |||
| 170 | - | } | |||
| 171 | - | expected[object.Digest] = object.Data | |||
| 172 | - | } | |||
| 173 | - | if len(generation.Objects) != len(expected) { | |||
| 174 | - | return fmt.Errorf("snapshot: legacy import verification mismatch") | |||
| 175 | - | } | |||
| 176 | - | for digest, expectedData := range expected { | |||
| 177 | - | actual, ok := generation.Objects[digest] | |||
| 178 | - | if !ok || !bytes.Equal(actual, expectedData) || sha256.Sum256(actual) != digest { | |||
| 179 | - | return fmt.Errorf("snapshot: legacy import verification mismatch") | |||
| 180 | - | } | |||
| 181 | - | } | |||
| 182 | - | return nil | |||
| 183 | - | } | |||
| 184 | - | ||||
| 185 | - | func (d *Daemon) rememberLegacyDelete(blob ports.LegacySnapshot) { | |||
| 186 | - | d.legacyImportMu.Lock() | |||
| 187 | - | if d.legacyImportPending == nil { | |||
| 188 | - | d.legacyImportPending = make(map[string][]byte) | |||
| 189 | - | } | |||
| 190 | - | d.legacyImportPending[blob.Name] = append([]byte(nil), blob.Data...) | |||
| 191 | - | d.legacyImportMu.Unlock() | |||
| 192 | - | } | |||
| 193 | - | ||||
| 194 | - | func (d *Daemon) forgetLegacyDelete(name string) { | |||
| 195 | - | d.legacyImportMu.Lock() | |||
| 196 | - | delete(d.legacyImportPending, name) | |||
| 197 | - | d.legacyImportMu.Unlock() | |||
| 198 | - | } | |||
| 199 | - | ||||
| 200 | - | // retryLegacyDelete only acts on a blob which this daemon has already | |||
| 201 | - | // published and exactly reloaded. An unrelated pre-existing generation must | |||
| 202 | - | // continue to leave its legacy counterpart untouched. | |||
| 203 | - | func (d *Daemon) retryLegacyDelete(ctx context.Context, blob ports.LegacySnapshot) { | |||
| 204 | - | d.legacyImportMu.Lock() | |||
| 205 | - | pending, ok := d.legacyImportPending[blob.Name] | |||
| 206 | - | d.legacyImportMu.Unlock() | |||
| 207 | - | if !ok || !bytes.Equal(pending, blob.Data) { | |||
| 208 | - | return | |||
| 209 | - | } | |||
| 210 | - | if err := d.legacySnapshots.DeleteVerifiedLegacy(ctx, blob); err != nil { | |||
| 211 | - | d.log.Warn("deleting imported legacy snapshot failed", "session", blob.Name, "err", err) | |||
| 212 | - | return | |||
| 213 | - | } | |||
| 214 | - | d.forgetLegacyDelete(blob.Name) | |||
| 215 | - | } | |||
M internal/usecase/daemon/snapshot_restore_test.go +74 -21
| @@ -12,12 +12,14 @@ import ( | |||||
| 12 | "github.com/bnema/vev/internal/domain" | 12 | "github.com/bnema/vev/internal/domain" | ||
| 13 | "github.com/bnema/vev/internal/ports" | 13 | "github.com/bnema/vev/internal/ports" | ||
| 14 | "github.com/bnema/vev/internal/usecase/layout" | 14 | "github.com/bnema/vev/internal/usecase/layout" | ||
| 15 | + | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | |||
| 15 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | 16 | snapcodec "github.com/bnema/vev/internal/usecase/snapshot" | ||
| 16 | "github.com/bnema/vev/pkg/renderer" | 17 | "github.com/bnema/vev/pkg/renderer" | ||
| 17 | "github.com/bnema/vev/pkg/vt" | 18 | "github.com/bnema/vev/pkg/vt" | ||
| 18 | ) | 19 | ) | ||
| 19 | 20 | ||||
| 20 | type snapshotAcceptanceRepository struct { | 21 | type snapshotAcceptanceRepository struct { | ||
| 22 | + | noOpSnapshotRepository | |||
| 21 | mu sync.Mutex | 23 | mu sync.Mutex | ||
| 22 | names []string | 24 | names []string | ||
| 23 | generations map[string]ports.SnapshotGeneration | 25 | generations map[string]ports.SnapshotGeneration | ||
| @@ -42,7 +44,7 @@ func (r *snapshotAcceptanceRepository) Publish(ctx context.Context, p ports.Snap | |||||
| 42 | for _, object := range p.Objects { | 44 | for _, object := range p.Objects { | ||
| 43 | objects[object.Digest] = append([]byte(nil), object.Data...) | 45 | objects[object.Digest] = append([]byte(nil), object.Data...) | ||
| 44 | } | 46 | } | ||
| 45 | - | r.generations[p.Name] = ports.SnapshotGeneration{Name: p.Name, Generation: p.Generation, Manifest: append([]byte(nil), p.Manifest...), Objects: objects} | 47 | + | r.generations[p.Name] = ports.SnapshotGeneration{IncarnationID: p.IncarnationID, Name: p.Name, Generation: p.Generation, ParentCheckpoint: p.ParentCheckpoint, Manifest: append([]byte(nil), p.Manifest...), Objects: objects} |
| 46 | found := false | 48 | found := false | ||
| 47 | for _, name := range r.names { | 49 | for _, name := range r.names { | ||
| 48 | found = found || name == p.Name | 50 | found = found || name == p.Name | ||
| @@ -85,10 +87,16 @@ func (r *snapshotAcceptanceRepository) Load(ctx context.Context, name string) (p | |||||
| 85 | return cloneAcceptanceGeneration(generation), nil | 87 | return cloneAcceptanceGeneration(generation), nil | ||
| 86 | } | 88 | } | ||
| 87 | 89 | ||||
| 88 | - | func (*snapshotAcceptanceRepository) Delete(context.Context, string) error { return nil } | 90 | + | func (r *snapshotAcceptanceRepository) LoadCheckpoint(ctx context.Context, id domain.IncarnationID, name string, ref ports.CheckpointRef) (ports.SnapshotGeneration, error) { |
| 89 | - | func (*snapshotAcceptanceRepository) Tombstone(context.Context, string) error { return nil } | 91 | + | generation, err := r.Load(ctx, name) |
| 90 | - | func (*snapshotAcceptanceRepository) DeleteTombstone(context.Context, string) error { return nil } | 92 | + | if err != nil { |
| 91 | - | func (*snapshotAcceptanceRepository) Maintain(context.Context) error { return nil } | 93 | + | return ports.SnapshotGeneration{}, err |
| 94 | + | } | |||
| 95 | + | if generation.IncarnationID != id || generation.Generation != ref.Generation || snapcodec.ManifestDigest(generation.Manifest) != ref.ManifestDigest { | |||
| 96 | + | return ports.SnapshotGeneration{}, errors.New("checkpoint unavailable") | |||
| 97 | + | } | |||
| 98 | + | return generation, nil | |||
| 99 | + | } | |||
| 92 | 100 | ||||
| 93 | func TestSnapshotAcceptanceRepositoryLoadMutationDoesNotChangeStoredGeneration(t *testing.T) { | 101 | func TestSnapshotAcceptanceRepositoryLoadMutationDoesNotChangeStoredGeneration(t *testing.T) { | ||
| 94 | digest := ports.SnapshotDigest{1} | 102 | digest := ports.SnapshotDigest{1} | ||
| @@ -167,7 +175,7 @@ func restoreAcceptanceSession(t *testing.T, name string) snapcodec.Session { | |||||
| 167 | 175 | ||||
| 168 | func acceptanceGeneration(t *testing.T, snapshot snapcodec.Session, generation uint64) ports.SnapshotGeneration { | 176 | func acceptanceGeneration(t *testing.T, snapshot snapcodec.Session, generation uint64) ports.SnapshotGeneration { | ||
| 169 | t.Helper() | 177 | t.Helper() | ||
| 170 | - | publication, err := legacyPublication(snapshot) | 178 | + | publication, err := acceptancePublication(snapshot) |
| 171 | require.NoError(t, err) | 179 | require.NoError(t, err) | ||
| 172 | manifest, err := snapcodec.UnmarshalManifest(publication.Manifest) | 180 | manifest, err := snapcodec.UnmarshalManifest(publication.Manifest) | ||
| 173 | require.NoError(t, err) | 181 | require.NoError(t, err) | ||
| @@ -178,7 +186,7 @@ func acceptanceGeneration(t *testing.T, snapshot snapcodec.Session, generation u | |||||
| 178 | for _, object := range publication.Objects { | 186 | for _, object := range publication.Objects { | ||
| 179 | objects[object.Digest] = append([]byte(nil), object.Data...) | 187 | objects[object.Digest] = append([]byte(nil), object.Data...) | ||
| 180 | } | 188 | } | ||
| 181 | - | return ports.SnapshotGeneration{Name: snapshot.Name, Generation: generation, Manifest: encoded, Objects: objects} | 189 | + | return ports.SnapshotGeneration{IncarnationID: publication.IncarnationID, Name: snapshot.Name, Generation: generation, ParentCheckpoint: publication.ParentCheckpoint, Manifest: encoded, Objects: objects} |
| 182 | } | 190 | } | ||
| 183 | 191 | ||||
| 184 | func TestValidateRestoreSessionSnapshot(t *testing.T) { | 192 | func TestValidateRestoreSessionSnapshot(t *testing.T) { | ||
| @@ -210,9 +218,12 @@ func TestRestoreIncrementalGenerationAcceptance(t *testing.T) { | |||||
| 210 | repository := &snapshotAcceptanceRepository{names: []string{snapshot.Name}, generations: map[string]ports.SnapshotGeneration{snapshot.Name: generation}} | 218 | repository := &snapshotAcceptanceRepository{names: []string{snapshot.Name}, generations: map[string]ports.SnapshotGeneration{snapshot.Name: generation}} | ||
| 211 | pty, release := newBlockingPTY(t) | 219 | pty, release := newBlockingPTY(t) | ||
| 212 | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | 220 | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | ||
| 213 | - | store, _ := newMockStore(t) | 221 | + | checkpoint := domain.CheckpointRef{Generation: generation.Generation, ManifestDigest: snapcodec.ManifestDigest(generation.Manifest)} |
| 214 | - | WithStore(store)(d) | 222 | + | record := domain.CatalogueRecord{Name: snapshot.Name, IncarnationID: generation.IncarnationID, Cwd: "/snapshot/cwd", CreatedAt: int64(snapshot.CreatedAt), Committed: &checkpoint} |
| 215 | - | WithSnapshotRepository(repository, nil)(d) | 223 | + | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record}) |
| 224 | + | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) | |||
| 225 | + | WithSnapshotRepository(repository)(d) | |||
| 226 | + | WithRecoveryCoordinator(recoveryusecase.NewCoordinator(catalogue, repository, nil))(d) | |||
| 216 | t.Cleanup(func() { release(); d.sessWg.Wait() }) | 227 | t.Cleanup(func() { release(); d.sessWg.Wait() }) | ||
| 217 | 228 | ||||
| 218 | d.restoreIncrementalSnapshots(context.Background()) | 229 | d.restoreIncrementalSnapshots(context.Background()) | ||
| @@ -246,20 +257,63 @@ func TestRestoreIncrementalGenerationAcceptance(t *testing.T) { | |||||
| 246 | require.Equal(t, uint64(10), repository.publishes[0].Generation, "a restored session must continue the concrete repository generation stream") | 257 | require.Equal(t, uint64(10), repository.publishes[0].Generation, "a restored session must continue the concrete repository generation stream") | ||
| 247 | } | 258 | } | ||
| 248 | 259 | ||||
| 249 | - | func TestRestoreIncrementalFallbackAndInvalidObjectMappings(t *testing.T) { | 260 | + | func TestRestoredSessionMetadataUpdatePreservesCheckpointLineage(t *testing.T) { |
| 250 | - | snapshot := restoreAcceptanceSession(t, "fallback") | 261 | + | snapshot := restoreAcceptanceSession(t, "restored") |
| 251 | - | valid := acceptanceGeneration(t, snapshot, 3) | 262 | + | generation := acceptanceGeneration(t, snapshot, 9) |
| 252 | - | valid.Fallback = true | 263 | + | repository := &snapshotAcceptanceRepository{names: []string{snapshot.Name}, generations: map[string]ports.SnapshotGeneration{snapshot.Name: generation}} |
| 264 | + | committed := domain.CheckpointRef{Generation: generation.Generation, ManifestDigest: snapcodec.ManifestDigest(generation.Manifest)} | |||
| 265 | + | record := domain.CatalogueRecord{ | |||
| 266 | + | Name: snapshot.Name, IncarnationID: generation.IncarnationID, Cwd: "/snapshot/cwd", | |||
| 267 | + | CreatedAt: int64(snapshot.CreatedAt), UpdatedAt: 81, LastUsedSeq: 17, | |||
| 268 | + | TabNames: []string{"before"}, Committed: &committed, | |||
| 269 | + | } | |||
| 270 | + | catalogue := newDurableRecoveryCatalogue([]domain.CatalogueRecord{record}) | |||
| 271 | + | ||||
| 253 | pty, release := newBlockingPTY(t) | 272 | pty, release := newBlockingPTY(t) | ||
| 254 | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | 273 | d := newTestDaemon(t, newFactory(t, pty), stubClock{}) | ||
| 255 | - | store, _ := newMockStore(t) | 274 | + | coordinator := recoveryusecase.NewCoordinator(catalogue, repository, nil) |
| 256 | - | WithStore(store)(d) | 275 | + | WithCatalogue(catalogue, []domain.CatalogueRecord{record})(d) |
| 257 | - | repository := &snapshotAcceptanceRepository{names: []string{snapshot.Name}, generations: map[string]ports.SnapshotGeneration{snapshot.Name: valid}} | 276 | + | WithSnapshotRepository(repository)(d) |
| 258 | - | WithSnapshotRepository(repository, nil)(d) | 277 | + | WithRecoveryCoordinator(coordinator)(d) |
| 259 | t.Cleanup(func() { release(); d.sessWg.Wait() }) | 278 | t.Cleanup(func() { release(); d.sessWg.Wait() }) | ||
| 279 | + | ||||
| 260 | d.restoreIncrementalSnapshots(context.Background()) | 280 | d.restoreIncrementalSnapshots(context.Background()) | ||
| 261 | - | require.Len(t, d.notices.history(), 1) | 281 | + | d.mu.Lock() |
| 262 | - | require.Equal(t, domain.NoticeWarn, d.notices.history()[0].Severity) | 282 | + | restored := d.findByNameLocked(snapshot.Name) |
| 283 | + | d.mu.Unlock() | |||
| 284 | + | require.NotNil(t, restored) | |||
| 285 | + | require.NoError(t, d.renameTab(restored, restored.tabs[0], "after")) | |||
| 286 | + | ||||
| 287 | + | updates := catalogue.MetadataUpdates() | |||
| 288 | + | require.Len(t, updates, 1, "the authoritative Catalogue port must receive restored-session metadata updates") | |||
| 289 | + | require.Equal(t, record.IncarnationID, updates[0].IncarnationID) | |||
| 290 | + | updated, ok, err := catalogue.Record(record.Name) | |||
| 291 | + | require.NoError(t, err) | |||
| 292 | + | require.True(t, ok) | |||
| 293 | + | require.Equal(t, record.IncarnationID, updated.IncarnationID) | |||
| 294 | + | require.Equal(t, record.Committed, updated.Committed) | |||
| 295 | + | require.Equal(t, record.DegradedReason, updated.DegradedReason) | |||
| 296 | + | require.Equal(t, []string{"after"}, updated.TabNames) | |||
| 297 | + | ||||
| 298 | + | startSnapshotEncodeWorker(t, d) | |||
| 299 | + | markSnapshotDirty(restored) | |||
| 300 | + | require.True(t, d.scheduleSnapshot(restored)) | |||
| 301 | + | awaitSnapshotIdle(t, restored) | |||
| 302 | + | d.snapshotNoticeMu.Lock() | |||
| 303 | + | failure := d.snapshotActiveFailureSignature | |||
| 304 | + | d.snapshotNoticeMu.Unlock() | |||
| 305 | + | require.Empty(t, failure) | |||
| 306 | + | require.Len(t, repository.publishes, 1) | |||
| 307 | + | require.Equal(t, record.Committed, repository.publishes[0].ParentCheckpoint) | |||
| 308 | + | published, ok, err := catalogue.Record(record.Name) | |||
| 309 | + | require.NoError(t, err) | |||
| 310 | + | require.True(t, ok) | |||
| 311 | + | require.Equal(t, uint64(10), published.Committed.Generation) | |||
| 312 | + | } | |||
| 313 | + | ||||
| 314 | + | func TestRestoreIncrementalFallbackAndInvalidObjectMappings(t *testing.T) { | |||
| 315 | + | snapshot := restoreAcceptanceSession(t, "fallback") | |||
| 316 | + | valid := acceptanceGeneration(t, snapshot, 3) | |||
| 263 | 317 | ||||
| 264 | for _, mutate := range []struct { | 318 | for _, mutate := range []struct { | ||
| 265 | name string | 319 | name string | ||
| @@ -300,7 +354,6 @@ func TestRestoreIncrementalFallbackAndInvalidObjectMappings(t *testing.T) { | |||||
| 300 | } { | 354 | } { | ||
| 301 | t.Run(mutate.name, func(t *testing.T) { | 355 | t.Run(mutate.name, func(t *testing.T) { | ||
| 302 | generation := cloneAcceptanceGeneration(valid) | 356 | generation := cloneAcceptanceGeneration(valid) | ||
| 303 | - | generation.Fallback = false | |||
| 304 | mutate.fn(&generation) | 357 | mutate.fn(&generation) | ||
| 305 | _, err := sessionFromGeneration(generation) | 358 | _, err := sessionFromGeneration(generation) | ||
| 306 | require.Error(t, err) | 359 | require.Error(t, err) | ||
M internal/usecase/daemon/snapshot_scheduler_test.go +50 -196
| @@ -2,7 +2,6 @@ package daemon | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | - | "errors" | |||
| 6 | "io" | 5 | "io" | ||
| 7 | "sync" | 6 | "sync" | ||
| 8 | "sync/atomic" | 7 | "sync/atomic" | ||
| @@ -20,7 +19,7 @@ import ( | |||||
| 20 | func TestSnapshotCoordinatorQuarantineCancelsPublicationBeforeDelete(t *testing.T) { | 19 | func TestSnapshotCoordinatorQuarantineCancelsPublicationBeforeDelete(t *testing.T) { | ||
| 21 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 20 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 22 | repository := portsmocks.NewMockSnapshotRepository(t) | 21 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 23 | - | WithSnapshotRepository(repository, nil)(d) | 22 | + | WithSnapshotRepository(repository)(d) |
| 24 | startSnapshotEncodeWorker(t, d) | 23 | startSnapshotEncodeWorker(t, d) | ||
| 25 | 24 | ||||
| 26 | started := make(chan struct{}) | 25 | started := make(chan struct{}) | ||
| @@ -49,7 +48,7 @@ func TestSnapshotCoordinatorQuarantineCancelsPublicationBeforeDelete(t *testing. | |||||
| 49 | func TestSnapshotCoordinatorQuarantineJoinsInFlightPublication(t *testing.T) { | 48 | func TestSnapshotCoordinatorQuarantineJoinsInFlightPublication(t *testing.T) { | ||
| 50 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 49 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 51 | repository := portsmocks.NewMockSnapshotRepository(t) | 50 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 52 | - | WithSnapshotRepository(repository, nil)(d) | 51 | + | WithSnapshotRepository(repository)(d) |
| 53 | startSnapshotEncodeWorker(t, d) | 52 | startSnapshotEncodeWorker(t, d) | ||
| 54 | 53 | ||||
| 55 | started := make(chan struct{}) | 54 | started := make(chan struct{}) | ||
| @@ -82,166 +81,10 @@ func TestSnapshotCoordinatorQuarantineJoinsInFlightPublication(t *testing.T) { | |||||
| 82 | } | 81 | } | ||
| 83 | } | 82 | } | ||
| 84 | 83 | ||||
| 85 | - | func TestRenameKeepsSnapshotCoordinatorQuarantinedUntilNewIdentityCommits(t *testing.T) { | |||
| 86 | - | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 87 | - | store, state := newMockStore(t) | |||
| 88 | - | WithStore(store)(d) | |||
| 89 | - | repository := portsmocks.NewMockSnapshotRepository(t) | |||
| 90 | - | WithSnapshotRepository(repository, nil)(d) | |||
| 91 | - | startSnapshotEncodeWorker(t, d) | |||
| 92 | - | ||||
| 93 | - | deleteEntered := make(chan struct{}) | |||
| 94 | - | allowDelete := make(chan struct{}) | |||
| 95 | - | deleteCompleted := make(chan struct{}) | |||
| 96 | - | state.deleteDone = deleteCompleted | |||
| 97 | - | published := make(chan ports.SnapshotPublication, 2) | |||
| 98 | - | repository.EXPECT().Tombstone(mock.Anything, "work").Return(nil).Once() | |||
| 99 | - | repository.EXPECT().Delete(mock.Anything, "work").RunAndReturn(func(context.Context, string) error { | |||
| 100 | - | close(deleteEntered) | |||
| 101 | - | <-allowDelete | |||
| 102 | - | return nil | |||
| 103 | - | }).Once() | |||
| 104 | - | repository.EXPECT().DeleteTombstone(mock.Anything, "work").Return(nil).Once() | |||
| 105 | - | repository.EXPECT().Publish(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, publication ports.SnapshotPublication) error { | |||
| 106 | - | published <- publication | |||
| 107 | - | return nil | |||
| 108 | - | }).Once() | |||
| 109 | - | ||||
| 110 | - | pty, release := newBlockingPTY(t) | |||
| 111 | - | defer release() | |||
| 112 | - | d.ptys = newFactory(t, pty) | |||
| 113 | - | sess, err := d.createSessionLocked("work", false, "/work", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |||
| 114 | - | require.NoError(t, err) | |||
| 115 | - | markSnapshotDirty(sess) | |||
| 116 | - | ||||
| 117 | - | renameDone := make(chan error, 1) | |||
| 118 | - | go func() { renameDone <- d.renameSession(sess, "renamed") }() | |||
| 119 | - | select { | |||
| 120 | - | case <-deleteEntered: | |||
| 121 | - | case <-time.After(testWaitTimeout): | |||
| 122 | - | t.Fatal("rename did not begin deleting the old snapshot identity") | |||
| 123 | - | } | |||
| 124 | - | ||||
| 125 | - | // Hold the daemon identity commit after Delete has returned. Scheduler work | |||
| 126 | - | // during this window must not capture or publish the old name. | |||
| 127 | - | d.mu.Lock() | |||
| 128 | - | close(allowDelete) | |||
| 129 | - | select { | |||
| 130 | - | case <-deleteCompleted: | |||
| 131 | - | case <-time.After(testWaitTimeout): | |||
| 132 | - | t.Fatal("old snapshot identity was not deleted") | |||
| 133 | - | } | |||
| 134 | - | require.False(t, state.has("work")) | |||
| 135 | - | sess.snapshotMu.Lock() | |||
| 136 | - | quarantined := sess.snapshotQuarantined | |||
| 137 | - | sess.snapshotMu.Unlock() | |||
| 138 | - | require.True(t, quarantined, "old Delete returned before the coordinator resumed") | |||
| 139 | - | ||||
| 140 | - | schedulerCtx, cancelScheduler := context.WithCancel(context.Background()) | |||
| 141 | - | defer cancelScheduler() | |||
| 142 | - | go d.snapshotRepositorySaver(schedulerCtx) | |||
| 143 | - | select { | |||
| 144 | - | case d.snapshotWake <- struct{}{}: | |||
| 145 | - | default: | |||
| 146 | - | } | |||
| 147 | - | require.True(t, d.scheduleSnapshot(sess), "old work may be coalesced while the coordinator is quarantined") | |||
| 148 | - | d.mu.Unlock() | |||
| 149 | - | ||||
| 150 | - | require.NoError(t, <-renameDone) | |||
| 151 | - | select { | |||
| 152 | - | case publication := <-published: | |||
| 153 | - | require.Equal(t, "renamed", publication.Name) | |||
| 154 | - | require.Equal(t, uint64(1), publication.Generation) | |||
| 155 | - | case <-time.After(testWaitTimeout): | |||
| 156 | - | t.Fatal("new identity was not published") | |||
| 157 | - | } | |||
| 158 | - | } | |||
| 159 | - | ||||
| 160 | - | func TestRenameRollbackAfterOldDeleteLeavesOldCoordinatorStopped(t *testing.T) { | |||
| 161 | - | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 162 | - | store, state := newMockStore(t) | |||
| 163 | - | WithStore(store)(d) | |||
| 164 | - | repository := portsmocks.NewMockSnapshotRepository(t) | |||
| 165 | - | WithSnapshotRepository(repository, nil)(d) | |||
| 166 | - | startSnapshotEncodeWorker(t, d) | |||
| 167 | - | ||||
| 168 | - | deleteOldRecordErr := errors.New("old record delete failed") | |||
| 169 | - | state.mu.Lock() | |||
| 170 | - | state.deleteErr = func(key string) error { | |||
| 171 | - | if key == "work" { | |||
| 172 | - | return deleteOldRecordErr | |||
| 173 | - | } | |||
| 174 | - | return nil | |||
| 175 | - | } | |||
| 176 | - | state.mu.Unlock() | |||
| 177 | - | repository.EXPECT().Tombstone(mock.Anything, "work").Return(nil).Once() | |||
| 178 | - | repository.EXPECT().Delete(mock.Anything, "work").Return(nil).Once() | |||
| 179 | - | // No Publish expectation is installed: any publication is an unexpected mock | |||
| 180 | - | // call. The saver handshake below joins the scheduler before this test ends. | |||
| 181 | - | ||||
| 182 | - | pty, release := newBlockingPTY(t) | |||
| 183 | - | defer release() | |||
| 184 | - | d.ptys = newFactory(t, pty) | |||
| 185 | - | sess, err := d.createSessionLocked("work", false, "/work", domain.Size{Cols: 80, Rows: 24}, terminalEnv{}, d.baseEnv) | |||
| 186 | - | require.NoError(t, err) | |||
| 187 | - | markSnapshotDirty(sess) | |||
| 188 | - | ||||
| 189 | - | require.ErrorIs(t, d.renameSession(sess, "renamed"), deleteOldRecordErr) | |||
| 190 | - | sess.mu.Lock() | |||
| 191 | - | require.Equal(t, "work", sess.name) | |||
| 192 | - | require.False(t, sess.ephemeral) | |||
| 193 | - | require.False(t, sess.renameInProgress) | |||
| 194 | - | sess.mu.Unlock() | |||
| 195 | - | require.True(t, state.has("work")) | |||
| 196 | - | require.False(t, state.has("renamed")) | |||
| 197 | - | sess.snapshotMu.Lock() | |||
| 198 | - | quarantined := sess.snapshotQuarantined | |||
| 199 | - | sess.snapshotMu.Unlock() | |||
| 200 | - | require.True(t, quarantined) | |||
| 201 | - | ||||
| 202 | - | // Drain stale producer notifications, then use the timer-reset handshake to | |||
| 203 | - | // prove the saver consumed this wake and completed its scheduling pass. Stop | |||
| 204 | - | // and join it before asserting: no publisher can start after this point. | |||
| 205 | - | for { | |||
| 206 | - | select { | |||
| 207 | - | case <-d.snapshotWake: | |||
| 208 | - | default: | |||
| 209 | - | goto wakeDrained | |||
| 210 | - | } | |||
| 211 | - | } | |||
| 212 | - | ||||
| 213 | - | wakeDrained: | |||
| 214 | - | clock := newDeterministicSnapshotClock(time.Unix(100, 0)) | |||
| 215 | - | d.clock = clock | |||
| 216 | - | schedulerCtx, cancelScheduler := context.WithCancel(context.Background()) | |||
| 217 | - | schedulerDone := make(chan struct{}) | |||
| 218 | - | go func() { | |||
| 219 | - | d.snapshotRepositorySaver(schedulerCtx) | |||
| 220 | - | close(schedulerDone) | |||
| 221 | - | }() | |||
| 222 | - | timer := clock.timer() | |||
| 223 | - | require.Equal(t, 24*time.Hour, <-timer.resets) | |||
| 224 | - | select { | |||
| 225 | - | case d.snapshotWake <- struct{}{}: | |||
| 226 | - | default: | |||
| 227 | - | t.Fatal("snapshot wake unexpectedly remained full") | |||
| 228 | - | } | |||
| 229 | - | require.Equal(t, 24*time.Hour, <-timer.resets, "scheduler did not process the rollback wake") | |||
| 230 | - | cancelScheduler() | |||
| 231 | - | select { | |||
| 232 | - | case <-schedulerDone: | |||
| 233 | - | case <-time.After(testWaitTimeout): | |||
| 234 | - | t.Fatal("snapshot scheduler did not stop") | |||
| 235 | - | } | |||
| 236 | - | // The deterministic scheduler clock has no independently firing deadline; | |||
| 237 | - | // restore the daemon's normal test clock before its session cleanup runs. | |||
| 238 | - | d.clock = stubClock{} | |||
| 239 | - | } | |||
| 240 | - | ||||
| 241 | func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) { | 84 | func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) { | ||
| 242 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 85 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 243 | repository := portsmocks.NewMockSnapshotRepository(t) | 86 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 244 | - | WithSnapshotRepository(repository, nil)(d) | 87 | + | WithSnapshotRepository(repository)(d) |
| 245 | startSnapshotEncodeWorker(t, d) | 88 | startSnapshotEncodeWorker(t, d) | ||
| 246 | 89 | ||||
| 247 | started := make(chan struct{}) | 90 | started := make(chan struct{}) | ||
| @@ -280,7 +123,7 @@ func TestForcedSnapshotSchedulesSuccessorAfterRoutineInFlight(t *testing.T) { | |||||
| 280 | func TestForcedSnapshotSchedulesSuccessorForRoutineCaptureQueuedBehindWorker(t *testing.T) { | 123 | func TestForcedSnapshotSchedulesSuccessorForRoutineCaptureQueuedBehindWorker(t *testing.T) { | ||
| 281 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 124 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 282 | repository := portsmocks.NewMockSnapshotRepository(t) | 125 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 283 | - | WithSnapshotRepository(repository, nil)(d) | 126 | + | WithSnapshotRepository(repository)(d) |
| 284 | startSnapshotEncodeWorker(t, d) | 127 | startSnapshotEncodeWorker(t, d) | ||
| 285 | 128 | ||||
| 286 | blockerStarted := make(chan struct{}) | 129 | blockerStarted := make(chan struct{}) | ||
| @@ -321,7 +164,7 @@ func TestForcedSnapshotSchedulesSuccessorForRoutineCaptureQueuedBehindWorker(t * | |||||
| 321 | func TestMultipleForcedSnapshotsCoalesceToOneSuccessor(t *testing.T) { | 164 | func TestMultipleForcedSnapshotsCoalesceToOneSuccessor(t *testing.T) { | ||
| 322 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 165 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 323 | repository := portsmocks.NewMockSnapshotRepository(t) | 166 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 324 | - | WithSnapshotRepository(repository, nil)(d) | 167 | + | WithSnapshotRepository(repository)(d) |
| 325 | startSnapshotEncodeWorker(t, d) | 168 | startSnapshotEncodeWorker(t, d) | ||
| 326 | 169 | ||||
| 327 | started := make(chan struct{}) | 170 | started := make(chan struct{}) | ||
| @@ -364,7 +207,7 @@ func TestForcedSnapshotShutdownTimeoutRetainsRetryableStateAndNotice(t *testing. | |||||
| 364 | clock := newSnapshotDeadlineClock() | 207 | clock := newSnapshotDeadlineClock() | ||
| 365 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | 208 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | ||
| 366 | repository := portsmocks.NewMockSnapshotRepository(t) | 209 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 367 | - | WithSnapshotRepository(repository, nil)(d) | 210 | + | WithSnapshotRepository(repository)(d) |
| 368 | notices := portsmocks.NewMockNoticeStore(t) | 211 | notices := portsmocks.NewMockNoticeStore(t) | ||
| 369 | WithNoticeStore(notices)(d) | 212 | WithNoticeStore(notices)(d) | ||
| 370 | startSnapshotEncodeWorker(t, d) | 213 | startSnapshotEncodeWorker(t, d) | ||
| @@ -442,13 +285,12 @@ func TestForcedSnapshotShutdownTimeoutRetainsRetryableStateAndNotice(t *testing. | |||||
| 442 | } | 285 | } | ||
| 443 | } | 286 | } | ||
| 444 | 287 | ||||
| 445 | - | func TestServeShutdownDeadlineBoundsPaneExitTeardownOwner(t *testing.T) { | 288 | + | func TestServeShutdownDeadlineStillJoinsPaneExitWriter(t *testing.T) { |
| 446 | pty, releasePTY := newBlockingPTY(t) | 289 | pty, releasePTY := newBlockingPTY(t) | ||
| 447 | clock := newServeShutdownClock() | 290 | clock := newServeShutdownClock() | ||
| 448 | d := newTestDaemon(t, newFactory(t, pty), clock) | 291 | d := newTestDaemon(t, newFactory(t, pty), clock) | ||
| 449 | repository := portsmocks.NewMockSnapshotRepository(t) | 292 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 450 | - | WithSnapshotRepository(repository, nil)(d) | 293 | + | WithSnapshotRepository(repository)(d) |
| 451 | - | repository.EXPECT().List(mock.Anything).Return(nil, nil).Maybe() | |||
| 452 | 294 | ||||
| 453 | publicationStarted := make(chan struct{}) | 295 | publicationStarted := make(chan struct{}) | ||
| 454 | releasePublication := make(chan struct{}) | 296 | releasePublication := make(chan struct{}) | ||
| @@ -463,7 +305,7 @@ func TestServeShutdownDeadlineBoundsPaneExitTeardownOwner(t *testing.T) { | |||||
| 463 | 305 | ||||
| 464 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | 306 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | ||
| 465 | listener := serveSnapshotListener(t, tr) | 307 | listener := serveSnapshotListener(t, tr) | ||
| 466 | - | ctx, cancel := context.WithCancel(context.Background()) | 308 | + | ctx, cancel := context.WithCancel(t.Context()) |
| 467 | defer cancel() | 309 | defer cancel() | ||
| 468 | served := make(chan error, 1) | 310 | served := make(chan error, 1) | ||
| 469 | go func() { served <- d.Serve(ctx, listener) }() | 311 | go func() { served <- d.Serve(ctx, listener) }() | ||
| @@ -486,9 +328,9 @@ func TestServeShutdownDeadlineBoundsPaneExitTeardownOwner(t *testing.T) { | |||||
| 486 | // deadline. | 328 | // deadline. | ||
| 487 | ownerDeadlineTimer := clock.nextFinalTimer(t) | 329 | ownerDeadlineTimer := clock.nextFinalTimer(t) | ||
| 488 | 330 | ||||
| 489 | - | // Cleanup is the only place that releases the competing owner. It also fires | 331 | + | // Cleanup fires the pane-exit owner's independent budget and joins the real |
| 490 | - | // that owner's independent budget and joins the real pane reader, proving no | 332 | + | // pane reader. Serve itself must remain owned until the repository writer is |
| 491 | - | // teardown goroutine is leaked without making Serve depend on repository I/O. | 333 | + | // released below, even after its checkpoint deadline expires. |
| 492 | t.Cleanup(func() { | 334 | t.Cleanup(func() { | ||
| 493 | releaseSnapshot() | 335 | releaseSnapshot() | ||
| 494 | ownerDeadlineTimer.fire() | 336 | ownerDeadlineTimer.fire() | ||
| @@ -499,11 +341,17 @@ func TestServeShutdownDeadlineBoundsPaneExitTeardownOwner(t *testing.T) { | |||||
| 499 | shutdownDeadlineTimer := clock.nextFinalTimer(t) | 341 | shutdownDeadlineTimer := clock.nextFinalTimer(t) | ||
| 500 | shutdownDeadlineTimer.fire() | 342 | shutdownDeadlineTimer.fire() | ||
| 501 | 343 | ||||
| 344 | + | select { | |||
| 345 | + | case err := <-served: | |||
| 346 | + | t.Fatalf("Serve returned with a durable writer alive: %v", err) | |||
| 347 | + | default: | |||
| 348 | + | } | |||
| 349 | + | releaseSnapshot() | |||
| 502 | select { | 350 | select { | ||
| 503 | case err := <-served: | 351 | case err := <-served: | ||
| 504 | require.NoError(t, err) | 352 | require.NoError(t, err) | ||
| 505 | case <-time.After(testWaitTimeout): | 353 | case <-time.After(testWaitTimeout): | ||
| 506 | - | t.Fatal("Serve did not return after its shared deadline while the pane-exit owner remained blocked") | 354 | + | t.Fatal("Serve did not return after the durable writer exited") |
| 507 | } | 355 | } | ||
| 508 | 356 | ||||
| 509 | sess.snapshotMu.Lock() | 357 | sess.snapshotMu.Lock() | ||
| @@ -534,7 +382,7 @@ func awaitTeardownWaiters(t *testing.T, sess *session, want uint) { | |||||
| 534 | func TestConcurrentSessionTeardownPublishesOneFinalSnapshot(t *testing.T) { | 382 | func TestConcurrentSessionTeardownPublishesOneFinalSnapshot(t *testing.T) { | ||
| 535 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 383 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 536 | repository := portsmocks.NewMockSnapshotRepository(t) | 384 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 537 | - | WithSnapshotRepository(repository, nil)(d) | 385 | + | WithSnapshotRepository(repository)(d) |
| 538 | startSnapshotEncodeWorker(t, d) | 386 | startSnapshotEncodeWorker(t, d) | ||
| 539 | 387 | ||||
| 540 | publicationStarted := make(chan struct{}) | 388 | publicationStarted := make(chan struct{}) | ||
| @@ -608,8 +456,7 @@ func TestServeShutdownCheckpointsBeforeStoppingSnapshotWorker(t *testing.T) { | |||||
| 608 | defer releasePTY() | 456 | defer releasePTY() | ||
| 609 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | 457 | d := newTestDaemon(t, newFactory(t, p), stubClock{}) | ||
| 610 | repository := portsmocks.NewMockSnapshotRepository(t) | 458 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 611 | - | WithSnapshotRepository(repository, nil)(d) | 459 | + | WithSnapshotRepository(repository)(d) |
| 612 | - | repository.EXPECT().List(mock.Anything).Return(nil, nil).Maybe() | |||
| 613 | 460 | ||||
| 614 | published := make(chan ports.SnapshotPublication, 2) | 461 | published := make(chan ports.SnapshotPublication, 2) | ||
| 615 | allowTerminalPublication := make(chan struct{}) | 462 | allowTerminalPublication := make(chan struct{}) | ||
| @@ -623,7 +470,7 @@ func TestServeShutdownCheckpointsBeforeStoppingSnapshotWorker(t *testing.T) { | |||||
| 623 | 470 | ||||
| 624 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | 471 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | ||
| 625 | l := serveSnapshotListener(t, tr) | 472 | l := serveSnapshotListener(t, tr) | ||
| 626 | - | ctx, cancel := context.WithCancel(context.Background()) | 473 | + | ctx, cancel := context.WithCancel(t.Context()) |
| 627 | defer cancel() | 474 | defer cancel() | ||
| 628 | served := make(chan error, 1) | 475 | served := make(chan error, 1) | ||
| 629 | go func() { served <- d.Serve(ctx, l) }() | 476 | go func() { served <- d.Serve(ctx, l) }() | ||
| @@ -647,21 +494,21 @@ func TestServeShutdownCheckpointsBeforeStoppingSnapshotWorker(t *testing.T) { | |||||
| 647 | } | 494 | } | ||
| 648 | } | 495 | } | ||
| 649 | 496 | ||||
| 650 | - | func TestServeShutdownDeadlineDoesNotWaitTwiceForUncooperativeSnapshotRepository(t *testing.T) { | 497 | + | func TestServeShutdownDeadlineStillJoinsUncooperativeSnapshotRepository(t *testing.T) { |
| 651 | p, releasePTY := newBlockingPTY(t) | 498 | p, releasePTY := newBlockingPTY(t) | ||
| 652 | defer releasePTY() | 499 | defer releasePTY() | ||
| 653 | clock := newServeShutdownClock() | 500 | clock := newServeShutdownClock() | ||
| 654 | d := newTestDaemon(t, newFactory(t, p), clock) | 501 | d := newTestDaemon(t, newFactory(t, p), clock) | ||
| 655 | repository := portsmocks.NewMockSnapshotRepository(t) | 502 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 656 | - | WithSnapshotRepository(repository, nil)(d) | 503 | + | WithSnapshotRepository(repository)(d) |
| 657 | - | repository.EXPECT().List(mock.Anything).Return(nil, nil).Maybe() | |||
| 658 | noticeStore := portsmocks.NewMockNoticeStore(t) | 504 | noticeStore := portsmocks.NewMockNoticeStore(t) | ||
| 659 | WithNoticeStore(noticeStore)(d) | 505 | WithNoticeStore(noticeStore)(d) | ||
| 660 | noticeStore.EXPECT().Claim().Return(nil, nil).Maybe() | 506 | noticeStore.EXPECT().Claim().Return(nil, nil).Maybe() | ||
| 661 | noticeStore.EXPECT().Ack().Return(nil).Maybe() | 507 | noticeStore.EXPECT().Ack().Return(nil).Maybe() | ||
| 508 | + | noticePersisted := make(chan struct{}) | |||
| 662 | noticeStore.EXPECT().Append(mock.MatchedBy(func(n domain.Notification) bool { | 509 | noticeStore.EXPECT().Append(mock.MatchedBy(func(n domain.Notification) bool { | ||
| 663 | return n.Code == domain.NoticeSnapshotWrite && n.Severity == domain.NoticeError | 510 | return n.Code == domain.NoticeSnapshotWrite && n.Severity == domain.NoticeError | ||
| 664 | - | })).Return(nil).Once() | 511 | + | })).Run(func(domain.Notification) { close(noticePersisted) }).Return(nil).Once() |
| 665 | 512 | ||||
| 666 | firstPublicationCompleted := make(chan struct{}) | 513 | firstPublicationCompleted := make(chan struct{}) | ||
| 667 | terminalPublicationStarted := make(chan struct{}) | 514 | terminalPublicationStarted := make(chan struct{}) | ||
| @@ -680,7 +527,7 @@ func TestServeShutdownDeadlineDoesNotWaitTwiceForUncooperativeSnapshotRepository | |||||
| 680 | 527 | ||||
| 681 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | 528 | tr, sends, _ := newConn(t, mustHello(ports.IntentNew, "work", domain.Size{Cols: 80, Rows: 24})) | ||
| 682 | l := serveSnapshotListener(t, tr) | 529 | l := serveSnapshotListener(t, tr) | ||
| 683 | - | ctx, cancel := context.WithCancel(context.Background()) | 530 | + | ctx, cancel := context.WithCancel(t.Context()) |
| 684 | defer cancel() | 531 | defer cancel() | ||
| 685 | served := make(chan error, 1) | 532 | served := make(chan error, 1) | ||
| 686 | go func() { served <- d.Serve(ctx, l) }() | 533 | go func() { served <- d.Serve(ctx, l) }() | ||
| @@ -704,21 +551,29 @@ func TestServeShutdownDeadlineDoesNotWaitTwiceForUncooperativeSnapshotRepository | |||||
| 704 | workerDone := d.snapshotWorkerDone | 551 | workerDone := d.snapshotWorkerDone | ||
| 705 | d.snapshotWorkerMu.Unlock() | 552 | d.snapshotWorkerMu.Unlock() | ||
| 706 | require.NotNil(t, workerDone) | 553 | require.NotNil(t, workerDone) | ||
| 707 | - | defer func() { | |||
| 708 | - | close(releaseTerminalPublication) | |||
| 709 | - | select { | |||
| 710 | - | case <-workerDone: | |||
| 711 | - | case <-time.After(testWaitTimeout): | |||
| 712 | - | t.Error("snapshot worker leaked after its repository call was released") | |||
| 713 | - | } | |||
| 714 | - | }() | |||
| 715 | clock.nextFinalTimer(t).fire() | 554 | clock.nextFinalTimer(t).fire() | ||
| 555 | + | select { | |||
| 556 | + | case <-noticePersisted: | |||
| 557 | + | case <-time.After(testWaitTimeout): | |||
| 558 | + | t.Fatal("shutdown timeout notice was not persisted before the writer join") | |||
| 559 | + | } | |||
| 716 | 560 | ||||
| 561 | + | select { | |||
| 562 | + | case err := <-served: | |||
| 563 | + | t.Fatalf("Serve returned with an uncooperative durable writer alive: %v", err) | |||
| 564 | + | default: | |||
| 565 | + | } | |||
| 566 | + | close(releaseTerminalPublication) | |||
| 567 | + | select { | |||
| 568 | + | case <-workerDone: | |||
| 569 | + | case <-time.After(testWaitTimeout): | |||
| 570 | + | t.Fatal("snapshot worker leaked after its repository call was released") | |||
| 571 | + | } | |||
| 717 | select { | 572 | select { | ||
| 718 | case err := <-served: | 573 | case err := <-served: | ||
| 719 | require.NoError(t, err) | 574 | require.NoError(t, err) | ||
| 720 | case <-time.After(testWaitTimeout): | 575 | case <-time.After(testWaitTimeout): | ||
| 721 | - | t.Fatal("Serve waited beyond the terminal checkpoint deadline") | 576 | + | t.Fatal("Serve did not return after the uncooperative writer exited") |
| 722 | } | 577 | } | ||
| 723 | require.LessOrEqual(t, clock.finalBudget.Load(), int64(snapshotFinalFlushTimeout), "shutdown must consume at most one snapshot deadline") | 578 | require.LessOrEqual(t, clock.finalBudget.Load(), int64(snapshotFinalFlushTimeout), "shutdown must consume at most one snapshot deadline") | ||
| 724 | require.Equal(t, uint64(1), lastValid.Load(), "the last valid checkpoint remains available after the terminal attempt times out") | 579 | require.Equal(t, uint64(1), lastValid.Load(), "the last valid checkpoint remains available after the terminal attempt times out") | ||
| @@ -783,7 +638,7 @@ func TestRoutineSnapshotEligibilityStartsAtCompletionAndForcedDoesNotMoveIt(t *t | |||||
| 783 | clock := &snapshotSchedulerClock{now: time.Unix(100, 0)} | 638 | clock := &snapshotSchedulerClock{now: time.Unix(100, 0)} | ||
| 784 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | 639 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | ||
| 785 | repository := portsmocks.NewMockSnapshotRepository(t) | 640 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 786 | - | WithSnapshotRepository(repository, nil)(d) | 641 | + | WithSnapshotRepository(repository)(d) |
| 787 | startSnapshotEncodeWorker(t, d) | 642 | startSnapshotEncodeWorker(t, d) | ||
| 788 | 643 | ||||
| 789 | var publishes atomic.Int32 | 644 | var publishes atomic.Int32 | ||
| @@ -931,7 +786,7 @@ func TestSnapshotCompletionSchedulesRoutineRetriesFromCompletion(t *testing.T) { | |||||
| 931 | clock := newDeterministicSnapshotClock(now) | 786 | clock := newDeterministicSnapshotClock(now) | ||
| 932 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | 787 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | ||
| 933 | repository := portsmocks.NewMockSnapshotRepository(t) | 788 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 934 | - | WithSnapshotRepository(repository, nil)(d) | 789 | + | WithSnapshotRepository(repository)(d) |
| 935 | 790 | ||||
| 936 | for _, tt := range []struct { | 791 | for _, tt := range []struct { | ||
| 937 | name string | 792 | name string | ||
| @@ -964,12 +819,11 @@ func TestSnapshotSchedulerImmediateEligibilityAndStaleCapturesRemainDirty(t *tes | |||||
| 964 | now := time.Unix(100, 0) | 819 | now := time.Unix(100, 0) | ||
| 965 | clock := newDeterministicSnapshotClock(now) | 820 | clock := newDeterministicSnapshotClock(now) | ||
| 966 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | 821 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | ||
| 967 | - | WithSnapshotRepository(portsmocks.NewMockSnapshotRepository(t), nil)(d) | |||
| 968 | 822 | ||||
| 969 | t.Run("no prior attempt is immediately eligible", func(t *testing.T) { | 823 | t.Run("no prior attempt is immediately eligible", func(t *testing.T) { | ||
| 970 | published := make(chan ports.SnapshotPublication, 1) | 824 | published := make(chan ports.SnapshotPublication, 1) | ||
| 971 | repository := portsmocks.NewMockSnapshotRepository(t) | 825 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 972 | - | WithSnapshotRepository(repository, nil)(d) | 826 | + | WithSnapshotRepository(repository)(d) |
| 973 | startSnapshotEncodeWorker(t, d) | 827 | startSnapshotEncodeWorker(t, d) | ||
| 974 | repository.EXPECT().Publish(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, publication ports.SnapshotPublication) error { | 828 | repository.EXPECT().Publish(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, publication ports.SnapshotPublication) error { | ||
| 975 | published <- publication | 829 | published <- publication | ||
| @@ -1024,7 +878,7 @@ func TestSnapshotWorkerQueueIsBoundedAndForcedCapturesCoalesce(t *testing.T) { | |||||
| 1024 | clock := newDeterministicSnapshotClock(now) | 878 | clock := newDeterministicSnapshotClock(now) | ||
| 1025 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | 879 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | ||
| 1026 | repository := portsmocks.NewMockSnapshotRepository(t) | 880 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 1027 | - | WithSnapshotRepository(repository, nil)(d) | 881 | + | WithSnapshotRepository(repository)(d) |
| 1028 | startSnapshotEncodeWorker(t, d) | 882 | startSnapshotEncodeWorker(t, d) | ||
| 1029 | 883 | ||||
| 1030 | started := make(chan struct{}) | 884 | started := make(chan struct{}) | ||
| @@ -1082,7 +936,7 @@ func TestSnapshotRepositorySaverUsesEarliestDeadlineAndRecomputes(t *testing.T) | |||||
| 1082 | clock := newDeterministicSnapshotClock(now) | 936 | clock := newDeterministicSnapshotClock(now) | ||
| 1083 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | 937 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), clock) | ||
| 1084 | repository := portsmocks.NewMockSnapshotRepository(t) | 938 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 1085 | - | WithSnapshotRepository(repository, nil)(d) | 939 | + | WithSnapshotRepository(repository)(d) |
| 1086 | startSnapshotEncodeWorker(t, d) | 940 | startSnapshotEncodeWorker(t, d) | ||
| 1087 | 941 | ||||
| 1088 | published := make(chan ports.SnapshotPublication, 1) | 942 | published := make(chan ports.SnapshotPublication, 1) | ||
| @@ -1097,7 +951,7 @@ func TestSnapshotRepositorySaverUsesEarliestDeadlineAndRecomputes(t *testing.T) | |||||
| 1097 | later.snapshotNextEligibleAt = now.Add(10 * time.Minute) | 951 | later.snapshotNextEligibleAt = now.Add(10 * time.Minute) | ||
| 1098 | d.sessions[later.id] = later | 952 | d.sessions[later.id] = later | ||
| 1099 | 953 | ||||
| 1100 | - | ctx, cancel := context.WithCancel(context.Background()) | 954 | + | ctx, cancel := context.WithCancel(t.Context()) |
| 1101 | done := make(chan struct{}) | 955 | done := make(chan struct{}) | ||
| 1102 | go func() { | 956 | go func() { | ||
| 1103 | d.snapshotRepositorySaver(ctx) | 957 | d.snapshotRepositorySaver(ctx) | ||
M internal/usecase/daemon/snapshot_test.go +68 -1
| @@ -12,15 +12,59 @@ import ( | |||||
| 12 | "github.com/bnema/vev/internal/domain" | 12 | "github.com/bnema/vev/internal/domain" | ||
| 13 | "github.com/bnema/vev/internal/ports" | 13 | "github.com/bnema/vev/internal/ports" | ||
| 14 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | 14 | portsmocks "github.com/bnema/vev/internal/ports/mocks" | ||
| 15 | + | recoveryusecase "github.com/bnema/vev/internal/usecase/recovery" | |||
| 15 | "github.com/bnema/vev/pkg/renderer" | 16 | "github.com/bnema/vev/pkg/renderer" | ||
| 16 | ) | 17 | ) | ||
| 17 | 18 | ||||
| 19 | + | // TestWithSnapshotRepositoryRejectsTypedNil verifies typed nil repositories are disabled. | |||
| 20 | + | func TestWithSnapshotRepositoryRejectsTypedNil(t *testing.T) { | |||
| 21 | + | var repository *snapshotAcceptanceRepository | |||
| 22 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 23 | + | ||||
| 24 | + | WithSnapshotRepository(repository)(d) | |||
| 25 | + | ||||
| 26 | + | require.Nil(t, d.snapshotRepository) | |||
| 27 | + | require.False(t, d.snapsEnabled) | |||
| 28 | + | } | |||
| 29 | + | ||||
| 30 | + | func TestDurableWriterFailureNamesIncludesBufferedCapture(t *testing.T) { | |||
| 31 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 32 | + | capture := &snapshotCapture{name: "work", session: newSnapshotTestSession(t, "work", false, "/work")} | |||
| 33 | + | d.snapshotWorkerMu.Lock() | |||
| 34 | + | d.snapshotAdmitted[capture] = struct{}{} | |||
| 35 | + | d.snapshotJobs <- capture | |||
| 36 | + | d.snapshotWorkerMu.Unlock() | |||
| 37 | + | ||||
| 38 | + | require.Equal(t, []string{"work"}, d.durableWriterFailureNames()) | |||
| 39 | + | } | |||
| 40 | + | ||||
| 41 | + | func TestCheckpointCatalogueFailureKeepsCaptureRetryable(t *testing.T) { | |||
| 42 | + | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | |||
| 43 | + | catalogue := portsmocks.NewMockCatalogue(t) | |||
| 44 | + | repository := portsmocks.NewMockSnapshotRepository(t) | |||
| 45 | + | record := domain.CatalogueRecord{ | |||
| 46 | + | Name: "work", | |||
| 47 | + | IncarnationID: domain.IncarnationID{1}, | |||
| 48 | + | } | |||
| 49 | + | catalogue.EXPECT().Record("work").Return(record, true, nil).Once() | |||
| 50 | + | repository.EXPECT().Publish(mock.Anything, mock.Anything).Return(nil).Once() | |||
| 51 | + | catalogue.EXPECT().Replace("work", mock.Anything).Return(errors.New("catalogue unavailable")).Once() | |||
| 52 | + | WithSnapshotRepository(repository)(d) | |||
| 53 | + | WithRecoveryCoordinator(recoveryusecase.NewCoordinator(catalogue, repository, nil))(d) | |||
| 54 | + | startSnapshotEncodeWorker(t, d) | |||
| 55 | + | ||||
| 56 | + | sess := newSnapshotTestSession(t, "work", false, "/work") | |||
| 57 | + | require.True(t, d.captureSession(sess)) | |||
| 58 | + | awaitSnapshotIdle(t, sess) | |||
| 59 | + | require.True(t, sess.snapDirty.Load(), "failed catalogue commit must remain retryable") | |||
| 60 | + | } | |||
| 61 | + | ||||
| 18 | // TestSnapshotWorkerPublishesContentAddressedCapture verifies that all new | 62 | // TestSnapshotWorkerPublishesContentAddressedCapture verifies that all new | ||
| 19 | // checkpoints use the repository publication contract. | 63 | // checkpoints use the repository publication contract. | ||
| 20 | func TestSnapshotWorkerPublishesContentAddressedCapture(t *testing.T) { | 64 | func TestSnapshotWorkerPublishesContentAddressedCapture(t *testing.T) { | ||
| 21 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | 65 | d := newTestDaemon(t, portsmocks.NewMockPTYFactory(t), stubClock{}) | ||
| 22 | repository := portsmocks.NewMockSnapshotRepository(t) | 66 | repository := portsmocks.NewMockSnapshotRepository(t) | ||
| 23 | - | WithSnapshotRepository(repository, nil)(d) | 67 | + | WithSnapshotRepository(repository)(d) |
| 24 | startSnapshotEncodeWorker(t, d) | 68 | startSnapshotEncodeWorker(t, d) | ||
| 25 | 69 | ||||
| 26 | repository.EXPECT().Publish(mock.Anything, mock.MatchedBy(func(p ports.SnapshotPublication) bool { | 70 | repository.EXPECT().Publish(mock.Anything, mock.MatchedBy(func(p ports.SnapshotPublication) bool { | ||
| @@ -46,10 +90,33 @@ func newSnapshotTestSession(t *testing.T, name string, ephemeral bool, cwd strin | |||||
| 46 | p.screen.Write([]byte("hello")) | 90 | p.screen.Write([]byte("hello")) | ||
| 47 | appendHistoryRow(t, p.history, []renderer.Cell{{Rune: 'h'}, {Rune: 'i'}}) | 91 | appendHistoryRow(t, p.history, []renderer.Cell{{Rune: 'h'}, {Rune: 'i'}}) | ||
| 48 | sess := &session{id: domain.SessionID("sess-" + name), name: name, ephemeral: ephemeral, ctx: context.Background(), cancel: func() {}, tabs: []*tab{tb}, active: 0, cwd: cwd, createdAt: 42} | 92 | sess := &session{id: domain.SessionID("sess-" + name), name: name, ephemeral: ephemeral, ctx: context.Background(), cancel: func() {}, tabs: []*tab{tb}, active: 0, cwd: cwd, createdAt: 42} | ||
| 93 | + | if !ephemeral { | |||
| 94 | + | sess.incarnation = domain.IncarnationID{1} | |||
| 95 | + | } | |||
| 49 | sess.snapEligible.Store(!ephemeral && name != "") | 96 | sess.snapEligible.Store(!ephemeral && name != "") | ||
| 50 | return sess | 97 | return sess | ||
| 51 | } | 98 | } | ||
| 52 | 99 | ||||
| 100 | + | // noOpSnapshotRepository supplies the current durable repository contract to | |||
| 101 | + | // focused test sinks that only need to observe publication or deletion calls. | |||
| 102 | + | type noOpSnapshotRepository struct{} | |||
| 103 | + | ||||
| 104 | + | var _ ports.SnapshotRepository = noOpSnapshotRepository{} | |||
| 105 | + | ||||
| 106 | + | func (noOpSnapshotRepository) Publish(context.Context, ports.SnapshotPublication) error { return nil } | |||
| 107 | + | func (noOpSnapshotRepository) LoadCheckpoint(context.Context, domain.IncarnationID, string, ports.CheckpointRef) (ports.SnapshotGeneration, error) { | |||
| 108 | + | return ports.SnapshotGeneration{}, errors.New("unused") | |||
| 109 | + | } | |||
| 110 | + | func (noOpSnapshotRepository) ReconcileCheckpoint(context.Context, domain.IncarnationID, ports.CheckpointRef) error { | |||
| 111 | + | return nil | |||
| 112 | + | } | |||
| 113 | + | func (noOpSnapshotRepository) DeleteIncarnation(context.Context, domain.IncarnationID) error { | |||
| 114 | + | return nil | |||
| 115 | + | } | |||
| 116 | + | func (noOpSnapshotRepository) CollectGarbage(context.Context, map[domain.IncarnationID]domain.CheckpointRef) error { | |||
| 117 | + | return nil | |||
| 118 | + | } | |||
| 119 | + | ||||
| 53 | func awaitSnapshotIdle(t testing.TB, sess *session) { | 120 | func awaitSnapshotIdle(t testing.TB, sess *session) { | ||
| 54 | t.Helper() | 121 | t.Helper() | ||
| 55 | timer := time.NewTimer(testWaitTimeout) | 122 | timer := time.NewTimer(testWaitTimeout) | ||
M internal/usecase/daemon/snapshot_worker.go +106 -29
| @@ -2,6 +2,8 @@ package daemon | |||||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | + | "errors" | |||
| 6 | + | "sort" | |||
| 5 | 7 | ||||
| 6 | "github.com/bnema/vev/internal/domain" | 8 | "github.com/bnema/vev/internal/domain" | ||
| 7 | ) | 9 | ) | ||
| @@ -130,7 +132,19 @@ func (d *Daemon) publishSnapshotCapture(workerCtx context.Context, workerID uint | |||||
| 130 | } | 132 | } | ||
| 131 | if err == nil && workerCtx.Err() == nil { | 133 | if err == nil && workerCtx.Err() == nil { | ||
| 132 | if err = publicationContext.Err(); err == nil { | 134 | if err = publicationContext.Err(); err == nil { | ||
| 133 | - | err = d.snapshotRepository.Publish(publicationContext, publication) | 135 | + | if d.recovery != nil { |
| 136 | + | var record domain.CatalogueRecord | |||
| 137 | + | record, err = d.recovery.PublishCheckpoint(publicationContext, capture.name, publication) | |||
| 138 | + | if err == nil { | |||
| 139 | + | if record.Committed == nil { | |||
| 140 | + | err = errors.New("snapshot: checkpoint publication returned no committed checkpoint") | |||
| 141 | + | } else { | |||
| 142 | + | capture.checkpoint = *record.Committed | |||
| 143 | + | } | |||
| 144 | + | } | |||
| 145 | + | } else { | |||
| 146 | + | err = d.snapshotRepository.Publish(publicationContext, publication) | |||
| 147 | + | } | |||
| 134 | } | 148 | } | ||
| 135 | } | 149 | } | ||
| 136 | d.clearSnapshotWorkerInFlight(workerID, capture) | 150 | d.clearSnapshotWorkerInFlight(workerID, capture) | ||
| @@ -185,45 +199,108 @@ func (d *Daemon) stopSnapshotEncodeWorker() { | |||||
| 185 | if d == nil { | 199 | if d == nil { | ||
| 186 | return | 200 | return | ||
| 187 | } | 201 | } | ||
| 188 | - | deadline := newSnapshotShutdownDeadline(d.clock) | 202 | + | _ = d.StopDurableWriters(context.Background()) |
| 189 | - | defer deadline.stop() | 203 | + | d.WaitDurableWriters() |
| 190 | - | d.stopSnapshotEncodeWorkerWithDeadline(deadline) | |||
| 191 | } | 204 | } | ||
| 192 | 205 | ||||
| 193 | - | // stopSnapshotEncodeWorkerWithDeadline performs the one deadline-aware join | 206 | + | // StopDurableWriters stops maintenance scheduling, cancels cooperative |
| 194 | - | // for a Serve shutdown. Queues are never closed, and detached writers only | 207 | + | // maintenance calls, stops snapshot admission, and requests the snapshot |
| 195 | - | // retain immutable captures plus contexts/channels that remain live, so an | 208 | + | // worker's final drain. The context bounds checkpoint success only: |
| 196 | - | // uncooperative repository call cannot race teardown or block process exit. | 209 | + | // cancellation never detaches a writer from its owner. |
| 197 | - | func (d *Daemon) stopSnapshotEncodeWorkerWithDeadline(deadline *snapshotShutdownDeadline) { | 210 | + | func (d *Daemon) StopDurableWriters(ctx context.Context) []string { |
| 211 | + | if d == nil { | |||
| 212 | + | return nil | |||
| 213 | + | } | |||
| 214 | + | cancel, done := d.requestDurableWriterStop() | |||
| 215 | + | if done == nil { | |||
| 216 | + | return nil | |||
| 217 | + | } | |||
| 218 | + | if ctx == nil { | |||
| 219 | + | ctx = context.Background() | |||
| 220 | + | } | |||
| 221 | + | select { | |||
| 222 | + | case <-done: | |||
| 223 | + | return nil | |||
| 224 | + | case <-ctx.Done(): | |||
| 225 | + | // Snapshot identity before cancellation can let a cooperative worker clear | |||
| 226 | + | // its in-flight capture. The caller persists timeout notices before the | |||
| 227 | + | // unconditional ownership join. | |||
| 228 | + | names := d.durableWriterFailureNames() | |||
| 229 | + | cancel() | |||
| 230 | + | return names | |||
| 231 | + | } | |||
| 232 | + | } | |||
| 233 | + | ||||
| 234 | + | // WaitDurableWriters is the unconditional ownership barrier. It has no timeout | |||
| 235 | + | // or status branch: every writer goroutine must exit before Serve may return. | |||
| 236 | + | // Restoration is joined with the others because it mutates durable state too; | |||
| 237 | + | // it observes serveCtx cancellation, so waiting for it cannot outlive an | |||
| 238 | + | // uncooperative repository call any longer than the snapshot worker does. | |||
| 239 | + | func (d *Daemon) WaitDurableWriters() { | |||
| 198 | if d == nil { | 240 | if d == nil { | ||
| 199 | return | 241 | return | ||
| 200 | } | 242 | } | ||
| 201 | d.snapshotWorkerMu.Lock() | 243 | d.snapshotWorkerMu.Lock() | ||
| 244 | + | snapshotDone := d.snapshotWorkerDone | |||
| 245 | + | maintenanceDone := d.maintenanceWorkerDone | |||
| 246 | + | restoreDone := d.restoreWorkerDone | |||
| 247 | + | d.snapshotWorkerMu.Unlock() | |||
| 248 | + | if snapshotDone != nil { | |||
| 249 | + | <-snapshotDone | |||
| 250 | + | d.finishStoppedSnapshotWorker(false) | |||
| 251 | + | } | |||
| 252 | + | if maintenanceDone != nil { | |||
| 253 | + | <-maintenanceDone | |||
| 254 | + | } | |||
| 255 | + | // Restoration never blocks on the workers above (snapshot admission is | |||
| 256 | + | // non-blocking), so joining it last cannot deadlock. | |||
| 257 | + | if restoreDone != nil { | |||
| 258 | + | <-restoreDone | |||
| 259 | + | } | |||
| 260 | + | } | |||
| 261 | + | ||||
| 262 | + | func (d *Daemon) requestDurableWriterStop() (context.CancelFunc, <-chan struct{}) { | |||
| 263 | + | d.snapshotWorkerMu.Lock() | |||
| 264 | + | defer d.snapshotWorkerMu.Unlock() | |||
| 265 | + | if d.maintenanceWorkerCancel != nil { | |||
| 266 | + | d.maintenanceWorkerCancel() | |||
| 267 | + | } | |||
| 202 | cancel := d.snapshotWorkerCancel | 268 | cancel := d.snapshotWorkerCancel | ||
| 203 | - | if cancel == nil || d.snapshotWorkerClosing { | 269 | + | if cancel == nil { |
| 204 | - | d.snapshotWorkerMu.Unlock() | 270 | + | return nil, nil |
| 205 | - | return | |||
| 206 | } | 271 | } | ||
| 207 | - | // Stop accepting producers, then let the single worker drain its bounded | 272 | + | if !d.snapshotWorkerClosing { |
| 208 | - | // queue. A non-context-aware store may still block indefinitely, so the | 273 | + | d.snapshotWorkerClosing = true |
| 209 | - | // shared shutdown deadline bounds the one and only join. | 274 | + | close(d.snapshotWorkerFlush) |
| 210 | - | d.snapshotWorkerClosing = true | 275 | + | } |
| 211 | - | flush := d.snapshotWorkerFlush | 276 | + | return cancel, d.snapshotWorkerDone |
| 212 | - | done := d.snapshotWorkerDone | 277 | + | } |
| 213 | - | close(flush) | |||
| 214 | - | d.snapshotWorkerMu.Unlock() | |||
| 215 | 278 | ||||
| 216 | - | if deadline == nil { | 279 | + | // durableWriterFailureNames includes admitted buffered captures as well as the |
| 217 | - | deadline = newSnapshotShutdownDeadline(d.clock) | 280 | + | // active and final queues. snapshotAdmitted tracks normal captures from queue |
| 218 | - | defer deadline.stop() | 281 | + | // admission through completion, so worker dequeue cannot make one disappear. |
| 282 | + | func (d *Daemon) durableWriterFailureNames() []string { | |||
| 283 | + | d.snapshotWorkerMu.Lock() | |||
| 284 | + | defer d.snapshotWorkerMu.Unlock() | |||
| 285 | + | seen := make(map[string]struct{}) | |||
| 286 | + | add := func(capture *snapshotCapture) { | |||
| 287 | + | if capture != nil && capture.name != "" { | |||
| 288 | + | seen[capture.name] = struct{}{} | |||
| 289 | + | } | |||
| 219 | } | 290 | } | ||
| 220 | - | select { | 291 | + | add(d.snapshotWorkerInFlight) |
| 221 | - | case <-done: | 292 | + | for _, capture := range d.snapshotFinalJobs { |
| 222 | - | d.finishStoppedSnapshotWorker(false) | 293 | + | add(capture) |
| 223 | - | case <-deadline.Done(): | 294 | + | } |
| 224 | - | cancel() | 295 | + | for capture := range d.snapshotAdmitted { |
| 225 | - | d.finishStoppedSnapshotWorker(true) | 296 | + | add(capture) |
| 297 | + | } | |||
| 298 | + | names := make([]string, 0, len(seen)) | |||
| 299 | + | for name := range seen { | |||
| 300 | + | names = append(names, name) | |||
| 226 | } | 301 | } | ||
| 302 | + | sort.Strings(names) | |||
| 303 | + | return names | |||
| 227 | } | 304 | } | ||
| 228 | 305 | ||||
| 229 | func (d *Daemon) finishStoppedSnapshotWorker(abandoned bool) { | 306 | func (d *Daemon) finishStoppedSnapshotWorker(abandoned bool) { | ||
M internal/usecase/daemon/status_test.go +2 -2
| @@ -1066,9 +1066,9 @@ func TestBarStateForMRURestoredStoppedSessionsUsePersistedOrder(t *testing.T) { | |||||
| 1066 | cur := &session{id: "cur", name: "current", tabs: []*tab{{}}} | 1066 | cur := &session{id: "cur", name: "current", tabs: []*tab{{}}} | ||
| 1067 | d.sessions[cur.id] = cur | 1067 | d.sessions[cur.id] = cur | ||
| 1068 | 1068 | ||||
| 1069 | - | _, err := d.createSessionLocked("zeta", false, "/tmp/zeta", sz, terminalEnv{}, d.baseEnv) | 1069 | + | _, err := createSessionForTest(d, "zeta", false, "/tmp/zeta", sz, terminalEnv{}, d.baseEnv) |
| 1070 | require.NoError(t, err) | 1070 | require.NoError(t, err) | ||
| 1071 | - | _, err = d.createSessionLocked("alpha", false, "/tmp/alpha", sz, terminalEnv{}, d.baseEnv) | 1071 | + | _, err = createSessionForTest(d, "alpha", false, "/tmp/alpha", sz, terminalEnv{}, d.baseEnv) |
| 1072 | require.NoError(t, err) | 1072 | require.NoError(t, err) | ||
| 1073 | 1073 | ||||
| 1074 | state := d.barStateFor(cur, "") | 1074 | state := d.barStateFor(cur, "") | ||
A internal/usecase/recovery/broken.go +47 -0
| @@ -0,0 +1,47 @@ | |||||
| 1 | + | package recovery | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "errors" | |||
| 6 | + | "fmt" | |||
| 7 | + | ||||
| 8 | + | "github.com/bnema/vev/internal/domain" | |||
| 9 | + | ) | |||
| 10 | + | ||||
| 11 | + | // MarkBroken records a restoration failure only when name still identifies the | |||
| 12 | + | // incarnation that failed. It is fenced with Discard so a late restore cannot | |||
| 13 | + | // degrade the replacement incarnation. | |||
| 14 | + | func (c *Coordinator) MarkBroken(ctx context.Context, name string, expected domain.IncarnationID, reason string) (domain.CatalogueRecord, bool, error) { | |||
| 15 | + | if c == nil || c.catalogue == nil || c.locks == nil { | |||
| 16 | + | return domain.CatalogueRecord{}, false, errors.New("recovery: incomplete broken-session dependencies") | |||
| 17 | + | } | |||
| 18 | + | if err := ctx.Err(); err != nil { | |||
| 19 | + | return domain.CatalogueRecord{}, false, err | |||
| 20 | + | } | |||
| 21 | + | if reason == "" { | |||
| 22 | + | return domain.CatalogueRecord{}, false, errors.New("recovery: empty broken-session reason") | |||
| 23 | + | } | |||
| 24 | + | c.mutationMu.Lock() | |||
| 25 | + | defer c.mutationMu.Unlock() | |||
| 26 | + | unlock := c.locks.Lock([]string{name}) | |||
| 27 | + | defer unlock() | |||
| 28 | + | ||||
| 29 | + | record, ok, err := c.catalogue.Record(name) | |||
| 30 | + | if err != nil { | |||
| 31 | + | return domain.CatalogueRecord{}, false, err | |||
| 32 | + | } | |||
| 33 | + | if !ok || record.IncarnationID != expected { | |||
| 34 | + | return domain.CatalogueRecord{}, false, nil | |||
| 35 | + | } | |||
| 36 | + | if record.DegradedReason == reason { | |||
| 37 | + | return record, true, nil | |||
| 38 | + | } | |||
| 39 | + | record.DegradedReason = reason | |||
| 40 | + | if err := record.Validate(); err != nil { | |||
| 41 | + | return domain.CatalogueRecord{}, false, fmt.Errorf("recovery: invalid broken-session transition: %w", err) | |||
| 42 | + | } | |||
| 43 | + | if err := c.catalogue.Replace(name, record); err != nil { | |||
| 44 | + | return domain.CatalogueRecord{}, false, err | |||
| 45 | + | } | |||
| 46 | + | return record, true, nil | |||
| 47 | + | } | |||
A internal/usecase/recovery/checkpoint.go +74 -0
| @@ -0,0 +1,74 @@ | |||||
| 1 | + | package recovery | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "crypto/sha256" | |||
| 6 | + | "errors" | |||
| 7 | + | "fmt" | |||
| 8 | + | ||||
| 9 | + | "github.com/bnema/vev/internal/domain" | |||
| 10 | + | "github.com/bnema/vev/internal/ports" | |||
| 11 | + | ) | |||
| 12 | + | ||||
| 13 | + | var ( | |||
| 14 | + | ErrCheckpointRecordNotFound = errors.New("recovery: checkpoint catalogue record not found") | |||
| 15 | + | ErrCheckpointConflict = errors.New("recovery: checkpoint publication conflicts with catalogue") | |||
| 16 | + | ) | |||
| 17 | + | ||||
| 18 | + | // PublishCheckpoint publishes repository data before atomically making its | |||
| 19 | + | // reference authoritative in the catalogue. A catalogue failure deliberately | |||
| 20 | + | // leaves the durable publication as a same-incarnation forward orphan. | |||
| 21 | + | func (c *Coordinator) PublishCheckpoint(ctx context.Context, name string, publication ports.SnapshotPublication) (domain.CatalogueRecord, error) { | |||
| 22 | + | if c == nil || c.catalogue == nil || c.repository == nil || c.locks == nil { | |||
| 23 | + | return domain.CatalogueRecord{}, errors.New("recovery: incomplete checkpoint coordinator dependencies") | |||
| 24 | + | } | |||
| 25 | + | if err := ctx.Err(); err != nil { | |||
| 26 | + | return domain.CatalogueRecord{}, err | |||
| 27 | + | } | |||
| 28 | + | c.mutationMu.Lock() | |||
| 29 | + | defer c.mutationMu.Unlock() | |||
| 30 | + | unlock := c.locks.Lock([]string{name}) | |||
| 31 | + | defer unlock() | |||
| 32 | + | ||||
| 33 | + | record, ok, err := c.catalogue.Record(name) | |||
| 34 | + | if err != nil { | |||
| 35 | + | return domain.CatalogueRecord{}, err | |||
| 36 | + | } | |||
| 37 | + | if !ok { | |||
| 38 | + | return domain.CatalogueRecord{}, ErrCheckpointRecordNotFound | |||
| 39 | + | } | |||
| 40 | + | currentGeneration := uint64(0) | |||
| 41 | + | if record.Committed != nil { | |||
| 42 | + | currentGeneration = record.Committed.Generation | |||
| 43 | + | } | |||
| 44 | + | if publication.Name != name || publication.Name != record.Name || publication.IncarnationID != record.IncarnationID || | |||
| 45 | + | publication.Generation <= currentGeneration || len(publication.Manifest) == 0 || | |||
| 46 | + | !equalCheckpointRefs(publication.ParentCheckpoint, record.Committed) { | |||
| 47 | + | return domain.CatalogueRecord{}, ErrCheckpointConflict | |||
| 48 | + | } | |||
| 49 | + | if err := c.repository.Publish(ctx, publication); err != nil { | |||
| 50 | + | return domain.CatalogueRecord{}, err | |||
| 51 | + | } | |||
| 52 | + | ||||
| 53 | + | ref := domain.CheckpointRef{Generation: publication.Generation, ManifestDigest: sha256.Sum256(publication.Manifest)} | |||
| 54 | + | next := record | |||
| 55 | + | next.Committed = copyCheckpointRef(&ref) | |||
| 56 | + | next.DegradedReason = "" | |||
| 57 | + | if err := next.Validate(); err != nil { | |||
| 58 | + | return domain.CatalogueRecord{}, fmt.Errorf("recovery: invalid checkpoint transition: %w", err) | |||
| 59 | + | } | |||
| 60 | + | if err := c.catalogue.Replace(name, next); err != nil { | |||
| 61 | + | return domain.CatalogueRecord{}, err | |||
| 62 | + | } | |||
| 63 | + | return next, nil | |||
| 64 | + | } | |||
| 65 | + | ||||
| 66 | + | func equalCheckpointRefs(a, b *domain.CheckpointRef) bool { return a.Equal(b) } | |||
| 67 | + | ||||
| 68 | + | func copyCheckpointRef(ref *domain.CheckpointRef) *domain.CheckpointRef { | |||
| 69 | + | if ref == nil { | |||
| 70 | + | return nil | |||
| 71 | + | } | |||
| 72 | + | cloned := *ref | |||
| 73 | + | return &cloned | |||
| 74 | + | } | |||
A internal/usecase/recovery/checkpoint_test.go +183 -0
| @@ -0,0 +1,183 @@ | |||||
| 1 | + | package recovery | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "errors" | |||
| 6 | + | "fmt" | |||
| 7 | + | "sync" | |||
| 8 | + | "testing" | |||
| 9 | + | ||||
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 11 | + | "github.com/bnema/vev/internal/ports" | |||
| 12 | + | "github.com/stretchr/testify/require" | |||
| 13 | + | ) | |||
| 14 | + | ||||
| 15 | + | type checkpointCatalogue struct { | |||
| 16 | + | mu sync.Mutex | |||
| 17 | + | record domain.CatalogueRecord | |||
| 18 | + | replaceErr error | |||
| 19 | + | replaces int | |||
| 20 | + | events *[]string | |||
| 21 | + | } | |||
| 22 | + | ||||
| 23 | + | func (c *checkpointCatalogue) Records() ([]domain.CatalogueRecord, error) { | |||
| 24 | + | c.mu.Lock() | |||
| 25 | + | defer c.mu.Unlock() | |||
| 26 | + | return []domain.CatalogueRecord{c.record}, nil | |||
| 27 | + | } | |||
| 28 | + | func (c *checkpointCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) { | |||
| 29 | + | c.mu.Lock() | |||
| 30 | + | defer c.mu.Unlock() | |||
| 31 | + | return c.record, c.record.Name == name, nil | |||
| 32 | + | } | |||
| 33 | + | func (c *checkpointCatalogue) Create(domain.CatalogueRecord) error { return nil } | |||
| 34 | + | func (c *checkpointCatalogue) UpdateMetadata(domain.CatalogueMetadataUpdate) error { | |||
| 35 | + | return nil | |||
| 36 | + | } | |||
| 37 | + | func (c *checkpointCatalogue) Sync() error { return nil } | |||
| 38 | + | func (c *checkpointCatalogue) Replace(name string, record domain.CatalogueRecord) error { | |||
| 39 | + | c.mu.Lock() | |||
| 40 | + | defer c.mu.Unlock() | |||
| 41 | + | c.replaces++ | |||
| 42 | + | if c.events != nil { | |||
| 43 | + | *c.events = append(*c.events, "catalogue") | |||
| 44 | + | } | |||
| 45 | + | if c.replaceErr != nil { | |||
| 46 | + | return c.replaceErr | |||
| 47 | + | } | |||
| 48 | + | if name != c.record.Name || record.Name != name { | |||
| 49 | + | return errors.New("catalogue key mismatch") | |||
| 50 | + | } | |||
| 51 | + | c.record = record | |||
| 52 | + | return nil | |||
| 53 | + | } | |||
| 54 | + | func (c *checkpointCatalogue) Rename(string, domain.CatalogueRecord) error { return nil } | |||
| 55 | + | func (c *checkpointCatalogue) Delete(string) error { return nil } | |||
| 56 | + | func (c *checkpointCatalogue) Close() error { return nil } | |||
| 57 | + | ||||
| 58 | + | type checkpointRepository struct { | |||
| 59 | + | ports.SnapshotRepository | |||
| 60 | + | publications []ports.SnapshotPublication | |||
| 61 | + | generations map[uint64]ports.SnapshotGeneration | |||
| 62 | + | loadErrors map[uint64]error | |||
| 63 | + | loads []uint64 | |||
| 64 | + | repairs []domain.CheckpointRef | |||
| 65 | + | repairErr error | |||
| 66 | + | events *[]string | |||
| 67 | + | } | |||
| 68 | + | ||||
| 69 | + | func (r *checkpointRepository) Publish(_ context.Context, publication ports.SnapshotPublication) error { | |||
| 70 | + | if r.events != nil { | |||
| 71 | + | *r.events = append(*r.events, "repository") | |||
| 72 | + | } | |||
| 73 | + | r.publications = append(r.publications, publication) | |||
| 74 | + | return nil | |||
| 75 | + | } | |||
| 76 | + | ||||
| 77 | + | func (r *checkpointRepository) LoadCheckpoint(_ context.Context, _ domain.IncarnationID, _ string, ref ports.CheckpointRef) (ports.SnapshotGeneration, error) { | |||
| 78 | + | r.loads = append(r.loads, ref.Generation) | |||
| 79 | + | if err := r.loadErrors[ref.Generation]; err != nil { | |||
| 80 | + | return ports.SnapshotGeneration{}, err | |||
| 81 | + | } | |||
| 82 | + | generation, ok := r.generations[ref.Generation] | |||
| 83 | + | if !ok { | |||
| 84 | + | return ports.SnapshotGeneration{}, errors.New("checkpoint unavailable") | |||
| 85 | + | } | |||
| 86 | + | return generation, nil | |||
| 87 | + | } | |||
| 88 | + | ||||
| 89 | + | func (r *checkpointRepository) ReconcileCheckpoint(_ context.Context, _ domain.IncarnationID, ref ports.CheckpointRef) error { | |||
| 90 | + | r.repairs = append(r.repairs, ref) | |||
| 91 | + | return r.repairErr | |||
| 92 | + | } | |||
| 93 | + | ||||
| 94 | + | func checkpointRecord() domain.CatalogueRecord { | |||
| 95 | + | return domain.CatalogueRecord{ | |||
| 96 | + | Name: "work", | |||
| 97 | + | IncarnationID: domain.IncarnationID{1}, | |||
| 98 | + | } | |||
| 99 | + | } | |||
| 100 | + | ||||
| 101 | + | func checkpointPublication(record domain.CatalogueRecord, generation uint64) ports.SnapshotPublication { | |||
| 102 | + | return ports.SnapshotPublication{ | |||
| 103 | + | IncarnationID: record.IncarnationID, | |||
| 104 | + | Name: record.Name, | |||
| 105 | + | Generation: generation, | |||
| 106 | + | ParentCheckpoint: record.Committed, | |||
| 107 | + | Manifest: []byte{byte(generation), 0xa5}, | |||
| 108 | + | } | |||
| 109 | + | } | |||
| 110 | + | ||||
| 111 | + | func runCheckpointCommit(t *testing.T, count int) (domain.CatalogueRecord, *checkpointCatalogue, *checkpointRepository) { | |||
| 112 | + | t.Helper() | |||
| 113 | + | events := make([]string, 0, count*2) | |||
| 114 | + | catalogue := &checkpointCatalogue{record: checkpointRecord(), events: &events} | |||
| 115 | + | repository := &checkpointRepository{events: &events} | |||
| 116 | + | coordinator := NewCoordinator(catalogue, repository, nil) | |||
| 117 | + | for generation := uint64(1); generation <= uint64(count); generation++ { | |||
| 118 | + | var err error | |||
| 119 | + | catalogue.record, err = coordinator.PublishCheckpoint(context.Background(), "work", checkpointPublication(catalogue.record, generation)) | |||
| 120 | + | require.NoError(t, err) | |||
| 121 | + | } | |||
| 122 | + | for i := range count { | |||
| 123 | + | require.Equal(t, []string{"repository", "catalogue"}, events[i*2:i*2+2]) | |||
| 124 | + | } | |||
| 125 | + | return catalogue.record, catalogue, repository | |||
| 126 | + | } | |||
| 127 | + | ||||
| 128 | + | func TestCheckpointCommits(t *testing.T) { | |||
| 129 | + | for _, tc := range []struct { | |||
| 130 | + | name string | |||
| 131 | + | count int | |||
| 132 | + | wantCommitted uint64 | |||
| 133 | + | }{ | |||
| 134 | + | {name: "first", count: 1, wantCommitted: 1}, | |||
| 135 | + | {name: "second", count: 2, wantCommitted: 2}, | |||
| 136 | + | {name: "third", count: 3, wantCommitted: 3}, | |||
| 137 | + | {name: "fourth", count: 4, wantCommitted: 4}, | |||
| 138 | + | } { | |||
| 139 | + | t.Run(tc.name, func(t *testing.T) { | |||
| 140 | + | record, catalogue, _ := runCheckpointCommit(t, tc.count) | |||
| 141 | + | require.Equal(t, tc.wantCommitted, record.Committed.Generation) | |||
| 142 | + | require.Empty(t, record.DegradedReason) | |||
| 143 | + | require.Equal(t, tc.count, catalogue.replaces) | |||
| 144 | + | }) | |||
| 145 | + | } | |||
| 146 | + | } | |||
| 147 | + | ||||
| 148 | + | func TestCheckpointRejectsNonIncreasingGeneration(t *testing.T) { | |||
| 149 | + | prior := checkpointRecord() | |||
| 150 | + | priorRef := domain.CheckpointRef{Generation: 7, ManifestDigest: [32]byte{7}} | |||
| 151 | + | prior.Committed = &priorRef | |||
| 152 | + | ||||
| 153 | + | for _, generation := range []uint64{7, 6} { | |||
| 154 | + | t.Run(fmt.Sprintf("generation-%d", generation), func(t *testing.T) { | |||
| 155 | + | catalogue := &checkpointCatalogue{record: prior} | |||
| 156 | + | repository := &checkpointRepository{} | |||
| 157 | + | coordinator := NewCoordinator(catalogue, repository, nil) | |||
| 158 | + | ||||
| 159 | + | _, err := coordinator.PublishCheckpoint(context.Background(), prior.Name, checkpointPublication(prior, generation)) | |||
| 160 | + | ||||
| 161 | + | require.ErrorIs(t, err, ErrCheckpointConflict) | |||
| 162 | + | require.Empty(t, repository.publications) | |||
| 163 | + | require.Zero(t, catalogue.replaces) | |||
| 164 | + | }) | |||
| 165 | + | } | |||
| 166 | + | } | |||
| 167 | + | ||||
| 168 | + | func TestCheckpointCommitPublishOrphan(t *testing.T) { | |||
| 169 | + | prior := checkpointRecord() | |||
| 170 | + | priorRef := domain.CheckpointRef{Generation: 7, ManifestDigest: [32]byte{7}} | |||
| 171 | + | prior.Committed = &priorRef | |||
| 172 | + | cause := errors.New("catalogue sync failed") | |||
| 173 | + | catalogue := &checkpointCatalogue{record: prior, replaceErr: cause} | |||
| 174 | + | repository := &checkpointRepository{} | |||
| 175 | + | coordinator := NewCoordinator(catalogue, repository, nil) | |||
| 176 | + | publication := checkpointPublication(prior, 8) | |||
| 177 | + | ||||
| 178 | + | _, err := coordinator.PublishCheckpoint(context.Background(), prior.Name, publication) | |||
| 179 | + | require.ErrorIs(t, err, cause) | |||
| 180 | + | require.Equal(t, prior, catalogue.record, "failed catalogue commit must preserve the authoritative record") | |||
| 181 | + | require.Len(t, repository.publications, 1, "the durable publication remains a forward orphan") | |||
| 182 | + | require.Equal(t, publication, repository.publications[0]) | |||
| 183 | + | } | |||
A internal/usecase/recovery/coordinator.go +227 -0
| @@ -0,0 +1,227 @@ | |||||
| 1 | + | // Package recovery coordinates idempotent operations across the catalogue and | |||
| 2 | + | // snapshot repository without depending on filesystem adapters. | |||
| 3 | + | package recovery | |||
| 4 | + | ||||
| 5 | + | import ( | |||
| 6 | + | "context" | |||
| 7 | + | "errors" | |||
| 8 | + | "fmt" | |||
| 9 | + | "io" | |||
| 10 | + | "sort" | |||
| 11 | + | "sync" | |||
| 12 | + | ||||
| 13 | + | "github.com/bnema/vev/internal/domain" | |||
| 14 | + | "github.com/bnema/vev/internal/ports" | |||
| 15 | + | ) | |||
| 16 | + | ||||
| 17 | + | type KeyLocks struct { | |||
| 18 | + | mu sync.Mutex | |||
| 19 | + | refs map[string]*keyLockRef | |||
| 20 | + | } | |||
| 21 | + | ||||
| 22 | + | type keyLockRef struct { | |||
| 23 | + | mu sync.Mutex | |||
| 24 | + | refs int | |||
| 25 | + | } | |||
| 26 | + | ||||
| 27 | + | func NewKeyLocks() *KeyLocks { | |||
| 28 | + | return &KeyLocks{refs: make(map[string]*keyLockRef)} | |||
| 29 | + | } | |||
| 30 | + | ||||
| 31 | + | // Lock acquires each distinct stable key in lexical order. The returned | |||
| 32 | + | // function releases in reverse order and removes unused lock entries. | |||
| 33 | + | func (k *KeyLocks) Lock(names []string) func() { | |||
| 34 | + | if k == nil { | |||
| 35 | + | return func() {} | |||
| 36 | + | } | |||
| 37 | + | ordered := append([]string(nil), names...) | |||
| 38 | + | sort.Strings(ordered) | |||
| 39 | + | ordered = compactStrings(ordered) | |||
| 40 | + | ||||
| 41 | + | refs := make([]*keyLockRef, len(ordered)) | |||
| 42 | + | k.mu.Lock() | |||
| 43 | + | for i, name := range ordered { | |||
| 44 | + | ref := k.refs[name] | |||
| 45 | + | if ref == nil { | |||
| 46 | + | ref = &keyLockRef{} | |||
| 47 | + | k.refs[name] = ref | |||
| 48 | + | } | |||
| 49 | + | ref.refs++ | |||
| 50 | + | refs[i] = ref | |||
| 51 | + | } | |||
| 52 | + | k.mu.Unlock() | |||
| 53 | + | for _, ref := range refs { | |||
| 54 | + | ref.mu.Lock() | |||
| 55 | + | } | |||
| 56 | + | return func() { | |||
| 57 | + | for i := len(refs) - 1; i >= 0; i-- { | |||
| 58 | + | refs[i].mu.Unlock() | |||
| 59 | + | } | |||
| 60 | + | k.mu.Lock() | |||
| 61 | + | for i, name := range ordered { | |||
| 62 | + | refs[i].refs-- | |||
| 63 | + | if refs[i].refs == 0 { | |||
| 64 | + | delete(k.refs, name) | |||
| 65 | + | } | |||
| 66 | + | } | |||
| 67 | + | k.mu.Unlock() | |||
| 68 | + | } | |||
| 69 | + | } | |||
| 70 | + | ||||
| 71 | + | func compactStrings(values []string) []string { | |||
| 72 | + | if len(values) == 0 { | |||
| 73 | + | return values | |||
| 74 | + | } | |||
| 75 | + | out := values[:1] | |||
| 76 | + | for _, value := range values[1:] { | |||
| 77 | + | if value != out[len(out)-1] { | |||
| 78 | + | out = append(out, value) | |||
| 79 | + | } | |||
| 80 | + | } | |||
| 81 | + | return out | |||
| 82 | + | } | |||
| 83 | + | ||||
| 84 | + | type Coordinator struct { | |||
| 85 | + | catalogue ports.Catalogue | |||
| 86 | + | repository ports.SnapshotRepository | |||
| 87 | + | locks *KeyLocks | |||
| 88 | + | // mutationMu serializes the startup GC snapshot and collection with every | |||
| 89 | + | // catalogue or checkpoint mutation. Startup normally provides this fence by | |||
| 90 | + | // running before socket publication; retaining it here also makes the | |||
| 91 | + | // coordinator safe when used directly by a composition root. | |||
| 92 | + | mutationMu sync.Mutex | |||
| 93 | + | random io.Reader | |||
| 94 | + | } | |||
| 95 | + | ||||
| 96 | + | func NewCoordinator(catalogue ports.Catalogue, repository ports.SnapshotRepository, random io.Reader) *Coordinator { | |||
| 97 | + | return &Coordinator{catalogue: catalogue, repository: repository, locks: NewKeyLocks(), random: random} | |||
| 98 | + | } | |||
| 99 | + | ||||
| 100 | + | // Create commits fresh catalogue metadata before the caller creates or exposes | |||
| 101 | + | // any runtime session. | |||
| 102 | + | func (c *Coordinator) Create(ctx context.Context, record domain.CatalogueRecord) (domain.CatalogueRecord, error) { | |||
| 103 | + | if c == nil || c.catalogue == nil || c.locks == nil || c.random == nil { | |||
| 104 | + | return domain.CatalogueRecord{}, errors.New("recovery: incomplete create dependencies") | |||
| 105 | + | } | |||
| 106 | + | if err := ctx.Err(); err != nil { | |||
| 107 | + | return domain.CatalogueRecord{}, err | |||
| 108 | + | } | |||
| 109 | + | c.mutationMu.Lock() | |||
| 110 | + | defer c.mutationMu.Unlock() | |||
| 111 | + | unlock := c.locks.Lock([]string{record.Name}) | |||
| 112 | + | defer unlock() | |||
| 113 | + | if _, exists, err := c.catalogue.Record(record.Name); err != nil { | |||
| 114 | + | return domain.CatalogueRecord{}, err | |||
| 115 | + | } else if exists { | |||
| 116 | + | return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q already exists", record.Name) | |||
| 117 | + | } | |||
| 118 | + | id, err := domain.NewIncarnationID(c.random) | |||
| 119 | + | if err != nil { | |||
| 120 | + | return domain.CatalogueRecord{}, fmt.Errorf("recovery: generate incarnation: %w", err) | |||
| 121 | + | } | |||
| 122 | + | record.IncarnationID = id | |||
| 123 | + | record.Committed = nil | |||
| 124 | + | record.DegradedReason = "" | |||
| 125 | + | if err := record.Validate(); err != nil { | |||
| 126 | + | return domain.CatalogueRecord{}, fmt.Errorf("recovery: invalid fresh record: %w", err) | |||
| 127 | + | } | |||
| 128 | + | if err := c.catalogue.Create(record); err != nil { | |||
| 129 | + | return domain.CatalogueRecord{}, err | |||
| 130 | + | } | |||
| 131 | + | return record, nil | |||
| 132 | + | } | |||
| 133 | + | ||||
| 134 | + | // Rename atomically moves one catalogue key while retaining the immutable | |||
| 135 | + | // incarnation and all checkpoint references. | |||
| 136 | + | func (c *Coordinator) Rename(ctx context.Context, oldName, newName string) (domain.CatalogueRecord, error) { | |||
| 137 | + | if c == nil || c.catalogue == nil || c.locks == nil { | |||
| 138 | + | return domain.CatalogueRecord{}, errors.New("recovery: incomplete rename dependencies") | |||
| 139 | + | } | |||
| 140 | + | if err := ctx.Err(); err != nil { | |||
| 141 | + | return domain.CatalogueRecord{}, err | |||
| 142 | + | } | |||
| 143 | + | c.mutationMu.Lock() | |||
| 144 | + | defer c.mutationMu.Unlock() | |||
| 145 | + | unlock := c.locks.Lock([]string{oldName, newName}) | |||
| 146 | + | defer unlock() | |||
| 147 | + | record, ok, err := c.catalogue.Record(oldName) | |||
| 148 | + | if err != nil { | |||
| 149 | + | return domain.CatalogueRecord{}, err | |||
| 150 | + | } | |||
| 151 | + | if !ok { | |||
| 152 | + | return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q not found", oldName) | |||
| 153 | + | } | |||
| 154 | + | if existing, exists, err := c.catalogue.Record(newName); err != nil { | |||
| 155 | + | return domain.CatalogueRecord{}, err | |||
| 156 | + | } else if exists && (newName != oldName || existing.IncarnationID != record.IncarnationID) { | |||
| 157 | + | return domain.CatalogueRecord{}, fmt.Errorf("recovery: session %q already exists", newName) | |||
| 158 | + | } | |||
| 159 | + | record.Name = newName | |||
| 160 | + | if err := record.Validate(); err != nil { | |||
| 161 | + | return domain.CatalogueRecord{}, fmt.Errorf("recovery: invalid rename: %w", err) | |||
| 162 | + | } | |||
| 163 | + | if oldName != newName { | |||
| 164 | + | if err := c.catalogue.Rename(oldName, record); err != nil { | |||
| 165 | + | return domain.CatalogueRecord{}, err | |||
| 166 | + | } | |||
| 167 | + | } | |||
| 168 | + | return record, nil | |||
| 169 | + | } | |||
| 170 | + | ||||
| 171 | + | // Delete removes the catalogue record before deleting its incarnation. A crash | |||
| 172 | + | // between those steps leaves an orphan that startup garbage collection removes. | |||
| 173 | + | func (c *Coordinator) Delete(ctx context.Context, name string) error { | |||
| 174 | + | if c == nil || c.catalogue == nil || c.repository == nil || c.locks == nil { | |||
| 175 | + | return errors.New("recovery: incomplete delete dependencies") | |||
| 176 | + | } | |||
| 177 | + | if err := ctx.Err(); err != nil { | |||
| 178 | + | return err | |||
| 179 | + | } | |||
| 180 | + | c.mutationMu.Lock() | |||
| 181 | + | defer c.mutationMu.Unlock() | |||
| 182 | + | unlock := c.locks.Lock([]string{name}) | |||
| 183 | + | defer unlock() | |||
| 184 | + | record, ok, err := c.catalogue.Record(name) | |||
| 185 | + | if err != nil { | |||
| 186 | + | return err | |||
| 187 | + | } | |||
| 188 | + | if !ok { | |||
| 189 | + | return nil | |||
| 190 | + | } | |||
| 191 | + | if err := c.catalogue.Delete(record.Name); err != nil { | |||
| 192 | + | return err | |||
| 193 | + | } | |||
| 194 | + | return c.repository.DeleteIncarnation(ctx, record.IncarnationID) | |||
| 195 | + | } | |||
| 196 | + | ||||
| 197 | + | // CollectGarbage takes a catalogue snapshot and applies retention while all | |||
| 198 | + | // catalogue mutations and checkpoint publications are fenced. Its keep map is | |||
| 199 | + | // therefore never stale relative to an operation that creates an incarnation | |||
| 200 | + | // or commits a new generation. | |||
| 201 | + | func (c *Coordinator) CollectGarbage(ctx context.Context) (int, error) { | |||
| 202 | + | if c == nil || c.catalogue == nil || c.repository == nil { | |||
| 203 | + | return 0, errors.New("recovery: incomplete garbage collection dependencies") | |||
| 204 | + | } | |||
| 205 | + | if err := ctx.Err(); err != nil { | |||
| 206 | + | return 0, err | |||
| 207 | + | } | |||
| 208 | + | c.mutationMu.Lock() | |||
| 209 | + | defer c.mutationMu.Unlock() | |||
| 210 | + | ||||
| 211 | + | records, err := c.catalogue.Records() | |||
| 212 | + | if err != nil { | |||
| 213 | + | return 0, err | |||
| 214 | + | } | |||
| 215 | + | keep := make(map[domain.IncarnationID]domain.CheckpointRef, len(records)) | |||
| 216 | + | for _, record := range records { | |||
| 217 | + | if record.Committed != nil { | |||
| 218 | + | keep[record.IncarnationID] = *record.Committed | |||
| 219 | + | continue | |||
| 220 | + | } | |||
| 221 | + | keep[record.IncarnationID] = domain.CheckpointRef{} | |||
| 222 | + | } | |||
| 223 | + | if err := c.repository.CollectGarbage(ctx, keep); err != nil { | |||
| 224 | + | return len(keep), err | |||
| 225 | + | } | |||
| 226 | + | return len(keep), nil | |||
| 227 | + | } | |||
A internal/usecase/recovery/discard.go +62 -0
| @@ -0,0 +1,62 @@ | |||||
| 1 | + | package recovery | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "errors" | |||
| 6 | + | "fmt" | |||
| 7 | + | ||||
| 8 | + | "github.com/bnema/vev/internal/domain" | |||
| 9 | + | ) | |||
| 10 | + | ||||
| 11 | + | var ( | |||
| 12 | + | ErrRecoveryRecordNotFound = errors.New("recovery: session not found") | |||
| 13 | + | ErrSessionNotBroken = errors.New("recovery: session is not broken") | |||
| 14 | + | ) | |||
| 15 | + | ||||
| 16 | + | // Discard throws away a broken session's persisted state and replaces it with a | |||
| 17 | + | // fresh incarnation. It is retry-idempotent, not crash-resumable: a crash | |||
| 18 | + | // leaves either the old record (rerun discard) or a new record plus an orphan | |||
| 19 | + | // directory that startup garbage collection removes. | |||
| 20 | + | func (c *Coordinator) Discard(ctx context.Context, name string) error { | |||
| 21 | + | if c == nil || c.catalogue == nil || c.repository == nil || c.locks == nil || c.random == nil { | |||
| 22 | + | return errors.New("recovery: incomplete discard dependencies") | |||
| 23 | + | } | |||
| 24 | + | if err := ctx.Err(); err != nil { | |||
| 25 | + | return err | |||
| 26 | + | } | |||
| 27 | + | c.mutationMu.Lock() | |||
| 28 | + | defer c.mutationMu.Unlock() | |||
| 29 | + | unlock := c.locks.Lock([]string{name}) | |||
| 30 | + | defer unlock() | |||
| 31 | + | ||||
| 32 | + | record, ok, err := c.catalogue.Record(name) | |||
| 33 | + | if err != nil { | |||
| 34 | + | return err | |||
| 35 | + | } | |||
| 36 | + | if !ok { | |||
| 37 | + | return fmt.Errorf("discard %q: %w", name, ErrRecoveryRecordNotFound) | |||
| 38 | + | } | |||
| 39 | + | if record.Committed == nil { | |||
| 40 | + | return nil | |||
| 41 | + | } | |||
| 42 | + | if record.DegradedReason == "" { | |||
| 43 | + | return ErrSessionNotBroken | |||
| 44 | + | } | |||
| 45 | + | ||||
| 46 | + | old := record.IncarnationID | |||
| 47 | + | next, err := domain.NewIncarnationID(c.random) | |||
| 48 | + | if err != nil { | |||
| 49 | + | return fmt.Errorf("recovery: generate replacement incarnation: %w", err) | |||
| 50 | + | } | |||
| 51 | + | fresh := record | |||
| 52 | + | fresh.IncarnationID = next | |||
| 53 | + | fresh.Committed = nil | |||
| 54 | + | fresh.DegradedReason = "" | |||
| 55 | + | if err := fresh.Validate(); err != nil { | |||
| 56 | + | return fmt.Errorf("recovery: invalid discard replacement: %w", err) | |||
| 57 | + | } | |||
| 58 | + | if err := c.catalogue.Replace(name, fresh); err != nil { | |||
| 59 | + | return err | |||
| 60 | + | } | |||
| 61 | + | return c.repository.DeleteIncarnation(ctx, old) | |||
| 62 | + | } | |||
A internal/usecase/recovery/transactions_test.go +381 -0
| @@ -0,0 +1,381 @@ | |||||
| 1 | + | package recovery | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "bytes" | |||
| 5 | + | "context" | |||
| 6 | + | "errors" | |||
| 7 | + | "sync" | |||
| 8 | + | "testing" | |||
| 9 | + | ||||
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 11 | + | "github.com/bnema/vev/internal/ports" | |||
| 12 | + | "github.com/stretchr/testify/require" | |||
| 13 | + | ) | |||
| 14 | + | ||||
| 15 | + | type transactionCatalogue struct { | |||
| 16 | + | mu sync.Mutex | |||
| 17 | + | records map[string]domain.CatalogueRecord | |||
| 18 | + | events []string | |||
| 19 | + | replaceErrOnce error | |||
| 20 | + | onCreate func(domain.CatalogueRecord) | |||
| 21 | + | } | |||
| 22 | + | ||||
| 23 | + | func newTransactionCatalogue(records ...domain.CatalogueRecord) *transactionCatalogue { | |||
| 24 | + | catalogue := &transactionCatalogue{records: make(map[string]domain.CatalogueRecord)} | |||
| 25 | + | for _, record := range records { | |||
| 26 | + | catalogue.records[record.Name] = record | |||
| 27 | + | } | |||
| 28 | + | return catalogue | |||
| 29 | + | } | |||
| 30 | + | ||||
| 31 | + | func (c *transactionCatalogue) Records() ([]domain.CatalogueRecord, error) { | |||
| 32 | + | c.mu.Lock() | |||
| 33 | + | defer c.mu.Unlock() | |||
| 34 | + | out := make([]domain.CatalogueRecord, 0, len(c.records)) | |||
| 35 | + | for _, record := range c.records { | |||
| 36 | + | out = append(out, record) | |||
| 37 | + | } | |||
| 38 | + | return out, nil | |||
| 39 | + | } | |||
| 40 | + | ||||
| 41 | + | func (c *transactionCatalogue) Record(name string) (domain.CatalogueRecord, bool, error) { | |||
| 42 | + | c.mu.Lock() | |||
| 43 | + | defer c.mu.Unlock() | |||
| 44 | + | record, ok := c.records[name] | |||
| 45 | + | return record, ok, nil | |||
| 46 | + | } | |||
| 47 | + | ||||
| 48 | + | func (c *transactionCatalogue) Create(record domain.CatalogueRecord) error { | |||
| 49 | + | c.mu.Lock() | |||
| 50 | + | defer c.mu.Unlock() | |||
| 51 | + | if _, exists := c.records[record.Name]; exists { | |||
| 52 | + | return errors.New("record exists") | |||
| 53 | + | } | |||
| 54 | + | c.records[record.Name] = record | |||
| 55 | + | c.events = append(c.events, "create") | |||
| 56 | + | if c.onCreate != nil { | |||
| 57 | + | c.onCreate(record) | |||
| 58 | + | } | |||
| 59 | + | return nil | |||
| 60 | + | } | |||
| 61 | + | ||||
| 62 | + | func (c *transactionCatalogue) UpdateMetadata(domain.CatalogueMetadataUpdate) error { return nil } | |||
| 63 | + | func (c *transactionCatalogue) Sync() error { return nil } | |||
| 64 | + | ||||
| 65 | + | func (c *transactionCatalogue) Replace(name string, record domain.CatalogueRecord) error { | |||
| 66 | + | c.mu.Lock() | |||
| 67 | + | defer c.mu.Unlock() | |||
| 68 | + | if c.replaceErrOnce != nil { | |||
| 69 | + | err := c.replaceErrOnce | |||
| 70 | + | c.replaceErrOnce = nil | |||
| 71 | + | return err | |||
| 72 | + | } | |||
| 73 | + | if _, exists := c.records[name]; !exists { | |||
| 74 | + | return errors.New("record missing") | |||
| 75 | + | } | |||
| 76 | + | c.records[name] = record | |||
| 77 | + | c.events = append(c.events, "replace:"+name) | |||
| 78 | + | return nil | |||
| 79 | + | } | |||
| 80 | + | ||||
| 81 | + | func (c *transactionCatalogue) Rename(oldName string, record domain.CatalogueRecord) error { | |||
| 82 | + | c.mu.Lock() | |||
| 83 | + | defer c.mu.Unlock() | |||
| 84 | + | if _, exists := c.records[oldName]; !exists { | |||
| 85 | + | return errors.New("record missing") | |||
| 86 | + | } | |||
| 87 | + | if _, exists := c.records[record.Name]; exists && record.Name != oldName { | |||
| 88 | + | return errors.New("record exists") | |||
| 89 | + | } | |||
| 90 | + | delete(c.records, oldName) | |||
| 91 | + | c.records[record.Name] = record | |||
| 92 | + | c.events = append(c.events, "rename:"+oldName+":"+record.Name) | |||
| 93 | + | return nil | |||
| 94 | + | } | |||
| 95 | + | ||||
| 96 | + | func (c *transactionCatalogue) Delete(name string) error { | |||
| 97 | + | c.mu.Lock() | |||
| 98 | + | defer c.mu.Unlock() | |||
| 99 | + | delete(c.records, name) | |||
| 100 | + | c.events = append(c.events, "delete:"+name) | |||
| 101 | + | return nil | |||
| 102 | + | } | |||
| 103 | + | ||||
| 104 | + | func (c *transactionCatalogue) Close() error { return nil } | |||
| 105 | + | ||||
| 106 | + | type transactionRepository struct { | |||
| 107 | + | ports.SnapshotRepository | |||
| 108 | + | mu sync.Mutex | |||
| 109 | + | deleteErrOnce error | |||
| 110 | + | deletedIDs []domain.IncarnationID | |||
| 111 | + | deleteEntered chan struct{} | |||
| 112 | + | releaseDelete chan struct{} | |||
| 113 | + | } | |||
| 114 | + | ||||
| 115 | + | type garbageCollectionFenceRepository struct { | |||
| 116 | + | ports.SnapshotRepository | |||
| 117 | + | mu sync.Mutex | |||
| 118 | + | events []string | |||
| 119 | + | collectionEntered chan struct{} | |||
| 120 | + | releaseCollection chan struct{} | |||
| 121 | + | collectionFinished bool | |||
| 122 | + | publishBeforeGC bool | |||
| 123 | + | incarnations map[domain.IncarnationID]struct{} | |||
| 124 | + | generations map[domain.IncarnationID]map[uint64]struct{} | |||
| 125 | + | } | |||
| 126 | + | ||||
| 127 | + | func (r *garbageCollectionFenceRepository) CollectGarbage(_ context.Context, keep map[domain.IncarnationID]domain.CheckpointRef) error { | |||
| 128 | + | close(r.collectionEntered) | |||
| 129 | + | <-r.releaseCollection | |||
| 130 | + | r.mu.Lock() | |||
| 131 | + | defer r.mu.Unlock() | |||
| 132 | + | for id := range r.incarnations { | |||
| 133 | + | if _, known := keep[id]; !known { | |||
| 134 | + | delete(r.incarnations, id) | |||
| 135 | + | } | |||
| 136 | + | } | |||
| 137 | + | for id, generations := range r.generations { | |||
| 138 | + | committed, known := keep[id] | |||
| 139 | + | if !known { | |||
| 140 | + | delete(r.generations, id) | |||
| 141 | + | continue | |||
| 142 | + | } | |||
| 143 | + | for generation := range generations { | |||
| 144 | + | if generation != committed.Generation && generation+1 != committed.Generation { | |||
| 145 | + | delete(generations, generation) | |||
| 146 | + | } | |||
| 147 | + | } | |||
| 148 | + | } | |||
| 149 | + | r.collectionFinished = true | |||
| 150 | + | r.events = append(r.events, "collect") | |||
| 151 | + | return nil | |||
| 152 | + | } | |||
| 153 | + | ||||
| 154 | + | func (r *garbageCollectionFenceRepository) Publish(_ context.Context, publication ports.SnapshotPublication) error { | |||
| 155 | + | r.mu.Lock() | |||
| 156 | + | defer r.mu.Unlock() | |||
| 157 | + | if !r.collectionFinished { | |||
| 158 | + | r.publishBeforeGC = true | |||
| 159 | + | } | |||
| 160 | + | if r.generations == nil { | |||
| 161 | + | r.generations = make(map[domain.IncarnationID]map[uint64]struct{}) | |||
| 162 | + | } | |||
| 163 | + | if r.generations[publication.IncarnationID] == nil { | |||
| 164 | + | r.generations[publication.IncarnationID] = make(map[uint64]struct{}) | |||
| 165 | + | } | |||
| 166 | + | r.generations[publication.IncarnationID][publication.Generation] = struct{}{} | |||
| 167 | + | r.events = append(r.events, "publish") | |||
| 168 | + | return nil | |||
| 169 | + | } | |||
| 170 | + | ||||
| 171 | + | func (r *garbageCollectionFenceRepository) addIncarnation(record domain.CatalogueRecord) { | |||
| 172 | + | r.mu.Lock() | |||
| 173 | + | defer r.mu.Unlock() | |||
| 174 | + | if r.incarnations == nil { | |||
| 175 | + | r.incarnations = make(map[domain.IncarnationID]struct{}) | |||
| 176 | + | } | |||
| 177 | + | r.incarnations[record.IncarnationID] = struct{}{} | |||
| 178 | + | } | |||
| 179 | + | ||||
| 180 | + | func (r *garbageCollectionFenceRepository) hasIncarnation(id domain.IncarnationID) bool { | |||
| 181 | + | r.mu.Lock() | |||
| 182 | + | defer r.mu.Unlock() | |||
| 183 | + | _, ok := r.incarnations[id] | |||
| 184 | + | return ok | |||
| 185 | + | } | |||
| 186 | + | ||||
| 187 | + | func (r *garbageCollectionFenceRepository) hasGeneration(id domain.IncarnationID, generation uint64) bool { | |||
| 188 | + | r.mu.Lock() | |||
| 189 | + | defer r.mu.Unlock() | |||
| 190 | + | _, ok := r.generations[id][generation] | |||
| 191 | + | return ok | |||
| 192 | + | } | |||
| 193 | + | ||||
| 194 | + | func (r *garbageCollectionFenceRepository) eventLog() ([]string, bool) { | |||
| 195 | + | r.mu.Lock() | |||
| 196 | + | defer r.mu.Unlock() | |||
| 197 | + | return append([]string(nil), r.events...), r.publishBeforeGC | |||
| 198 | + | } | |||
| 199 | + | ||||
| 200 | + | func (r *transactionRepository) DeleteIncarnation(_ context.Context, id domain.IncarnationID) error { | |||
| 201 | + | if r.deleteEntered != nil { | |||
| 202 | + | close(r.deleteEntered) | |||
| 203 | + | <-r.releaseDelete | |||
| 204 | + | } | |||
| 205 | + | r.mu.Lock() | |||
| 206 | + | defer r.mu.Unlock() | |||
| 207 | + | if r.deleteErrOnce != nil { | |||
| 208 | + | err := r.deleteErrOnce | |||
| 209 | + | r.deleteErrOnce = nil | |||
| 210 | + | return err | |||
| 211 | + | } | |||
| 212 | + | r.deletedIDs = append(r.deletedIDs, id) | |||
| 213 | + | return nil | |||
| 214 | + | } | |||
| 215 | + | ||||
| 216 | + | func degradedTransactionRecord() domain.CatalogueRecord { | |||
| 217 | + | return domain.CatalogueRecord{ | |||
| 218 | + | Name: "broken", IncarnationID: domain.IncarnationID{1}, Committed: &domain.CheckpointRef{Generation: 3, ManifestDigest: [32]byte{3}}, | |||
| 219 | + | DegradedReason: "checkpoint unreadable", | |||
| 220 | + | } | |||
| 221 | + | } | |||
| 222 | + | ||||
| 223 | + | func TestStartupGarbageCollectionFencesNewIncarnationsAndCheckpointPublications(t *testing.T) { | |||
| 224 | + | t.Parallel() | |||
| 225 | + | ||||
| 226 | + | t.Run("a stale keep snapshot cannot remove a newly created incarnation", func(t *testing.T) { | |||
| 227 | + | t.Parallel() | |||
| 228 | + | catalogue := newTransactionCatalogue() | |||
| 229 | + | repository := &garbageCollectionFenceRepository{ | |||
| 230 | + | collectionEntered: make(chan struct{}), | |||
| 231 | + | releaseCollection: make(chan struct{}), | |||
| 232 | + | } | |||
| 233 | + | catalogue.onCreate = repository.addIncarnation | |||
| 234 | + | coordinator := NewCoordinator(catalogue, repository, bytes.NewReader(bytes.Repeat([]byte{2}, 16))) | |||
| 235 | + | ||||
| 236 | + | collected := make(chan error, 1) | |||
| 237 | + | go func() { | |||
| 238 | + | _, err := coordinator.CollectGarbage(context.Background()) | |||
| 239 | + | collected <- err | |||
| 240 | + | }() | |||
| 241 | + | <-repository.collectionEntered | |||
| 242 | + | ||||
| 243 | + | created := make(chan error, 1) | |||
| 244 | + | go func() { | |||
| 245 | + | _, err := coordinator.Create(context.Background(), domain.CatalogueRecord{Name: "new"}) | |||
| 246 | + | created <- err | |||
| 247 | + | }() | |||
| 248 | + | close(repository.releaseCollection) | |||
| 249 | + | ||||
| 250 | + | require.NoError(t, <-collected) | |||
| 251 | + | require.NoError(t, <-created) | |||
| 252 | + | got, ok, err := catalogue.Record("new") | |||
| 253 | + | require.NoError(t, err) | |||
| 254 | + | require.True(t, ok) | |||
| 255 | + | require.NotZero(t, got.IncarnationID) | |||
| 256 | + | require.True(t, repository.hasIncarnation(got.IncarnationID), "GC's stale keep snapshot must finish before the new incarnation is published") | |||
| 257 | + | events, _ := repository.eventLog() | |||
| 258 | + | require.Equal(t, []string{"collect"}, events) | |||
| 259 | + | }) | |||
| 260 | + | ||||
| 261 | + | t.Run("a stale keep snapshot cannot prune a newly committed generation", func(t *testing.T) { | |||
| 262 | + | t.Parallel() | |||
| 263 | + | committed := domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{1}} | |||
| 264 | + | record := domain.CatalogueRecord{Name: "work", IncarnationID: domain.IncarnationID{1}, Committed: &committed} | |||
| 265 | + | catalogue := newTransactionCatalogue(record) | |||
| 266 | + | repository := &garbageCollectionFenceRepository{ | |||
| 267 | + | collectionEntered: make(chan struct{}), | |||
| 268 | + | releaseCollection: make(chan struct{}), | |||
| 269 | + | generations: map[domain.IncarnationID]map[uint64]struct{}{ | |||
| 270 | + | record.IncarnationID: {1: {}}, | |||
| 271 | + | }, | |||
| 272 | + | } | |||
| 273 | + | coordinator := NewCoordinator(catalogue, repository, bytes.NewReader(nil)) | |||
| 274 | + | ||||
| 275 | + | collected := make(chan error, 1) | |||
| 276 | + | go func() { | |||
| 277 | + | _, err := coordinator.CollectGarbage(context.Background()) | |||
| 278 | + | collected <- err | |||
| 279 | + | }() | |||
| 280 | + | <-repository.collectionEntered | |||
| 281 | + | ||||
| 282 | + | published := make(chan error, 1) | |||
| 283 | + | go func() { | |||
| 284 | + | _, err := coordinator.PublishCheckpoint(context.Background(), "work", ports.SnapshotPublication{ | |||
| 285 | + | Name: "work", | |||
| 286 | + | IncarnationID: record.IncarnationID, | |||
| 287 | + | Generation: 2, | |||
| 288 | + | ParentCheckpoint: &committed, | |||
| 289 | + | Manifest: []byte("generation two"), | |||
| 290 | + | }) | |||
| 291 | + | published <- err | |||
| 292 | + | }() | |||
| 293 | + | close(repository.releaseCollection) | |||
| 294 | + | ||||
| 295 | + | require.NoError(t, <-collected) | |||
| 296 | + | require.NoError(t, <-published) | |||
| 297 | + | events, publishedBeforeGC := repository.eventLog() | |||
| 298 | + | require.Equal(t, []string{"collect", "publish"}, events) | |||
| 299 | + | require.False(t, publishedBeforeGC) | |||
| 300 | + | require.True(t, repository.hasGeneration(record.IncarnationID, 2), "GC's stale keep snapshot must finish before generation two is published") | |||
| 301 | + | }) | |||
| 302 | + | } | |||
| 303 | + | ||||
| 304 | + | func TestMarkBrokenCannotOverwriteDiscardReplacement(t *testing.T) { | |||
| 305 | + | t.Parallel() | |||
| 306 | + | old := degradedTransactionRecord() | |||
| 307 | + | catalogue := newTransactionCatalogue(old) | |||
| 308 | + | repository := &transactionRepository{deleteEntered: make(chan struct{}), releaseDelete: make(chan struct{})} | |||
| 309 | + | coordinator := NewCoordinator(catalogue, repository, bytes.NewReader(bytes.Repeat([]byte{2}, 16))) | |||
| 310 | + | ||||
| 311 | + | discarded := make(chan error, 1) | |||
| 312 | + | go func() { discarded <- coordinator.Discard(t.Context(), old.Name) }() | |||
| 313 | + | <-repository.deleteEntered // Replace committed the fresh incarnation while Discard retains the mutation fence. | |||
| 314 | + | ||||
| 315 | + | marked := make(chan struct { | |||
| 316 | + | marked bool | |||
| 317 | + | err error | |||
| 318 | + | }, 1) | |||
| 319 | + | go func() { | |||
| 320 | + | _, markedSession, err := coordinator.MarkBroken(t.Context(), old.Name, old.IncarnationID, "late restore failure") | |||
| 321 | + | marked <- struct { | |||
| 322 | + | marked bool | |||
| 323 | + | err error | |||
| 324 | + | }{markedSession, err} | |||
| 325 | + | }() | |||
| 326 | + | ||||
| 327 | + | close(repository.releaseDelete) | |||
| 328 | + | require.NoError(t, <-discarded) | |||
| 329 | + | result := <-marked | |||
| 330 | + | require.NoError(t, result.err) | |||
| 331 | + | require.False(t, result.marked, "a late restore must not degrade the replacement incarnation") | |||
| 332 | + | ||||
| 333 | + | got, ok, err := catalogue.Record(old.Name) | |||
| 334 | + | require.NoError(t, err) | |||
| 335 | + | require.True(t, ok) | |||
| 336 | + | require.NotEqual(t, old.IncarnationID, got.IncarnationID) | |||
| 337 | + | require.Nil(t, got.Committed) | |||
| 338 | + | require.Empty(t, got.DegradedReason) | |||
| 339 | + | } | |||
| 340 | + | ||||
| 341 | + | func TestDiscardIsRetryIdempotent(t *testing.T) { | |||
| 342 | + | t.Parallel() | |||
| 343 | + | failure := errors.New("injected discard failure") | |||
| 344 | + | tests := []struct { | |||
| 345 | + | name string | |||
| 346 | + | failBefore bool | |||
| 347 | + | }{ | |||
| 348 | + | {name: "retry from old catalogue record", failBefore: true}, | |||
| 349 | + | {name: "retry from fresh catalogue record"}, | |||
| 350 | + | } | |||
| 351 | + | for _, tt := range tests { | |||
| 352 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 353 | + | t.Parallel() | |||
| 354 | + | old := degradedTransactionRecord() | |||
| 355 | + | catalogue := newTransactionCatalogue(old) | |||
| 356 | + | repository := &transactionRepository{} | |||
| 357 | + | if tt.failBefore { | |||
| 358 | + | catalogue.replaceErrOnce = failure | |||
| 359 | + | } else { | |||
| 360 | + | repository.deleteErrOnce = failure | |||
| 361 | + | } | |||
| 362 | + | ids := append(bytes.Repeat([]byte{2}, 16), bytes.Repeat([]byte{3}, 16)...) | |||
| 363 | + | coordinator := NewCoordinator(catalogue, repository, bytes.NewReader(ids)) | |||
| 364 | + | ||||
| 365 | + | err := coordinator.Discard(context.Background(), old.Name) | |||
| 366 | + | require.ErrorIs(t, err, failure) | |||
| 367 | + | require.NoError(t, coordinator.Discard(context.Background(), old.Name)) | |||
| 368 | + | ||||
| 369 | + | got, ok, err := catalogue.Record(old.Name) | |||
| 370 | + | require.NoError(t, err) | |||
| 371 | + | require.True(t, ok) | |||
| 372 | + | require.Nil(t, got.Committed) | |||
| 373 | + | require.NotEqual(t, old.IncarnationID, got.IncarnationID) | |||
| 374 | + | if tt.failBefore { | |||
| 375 | + | require.Contains(t, repository.deletedIDs, old.IncarnationID) | |||
| 376 | + | } else { | |||
| 377 | + | require.Empty(t, repository.deletedIDs, "startup garbage collection owns the orphaned old directory") | |||
| 378 | + | } | |||
| 379 | + | }) | |||
| 380 | + | } | |||
| 381 | + | } | |||
M internal/usecase/snapshot/codec.go +9 -0
| @@ -185,6 +185,15 @@ func writeProcess(w *payloadWriter, p *Process) error { | |||||
| 185 | 185 | ||||
| 186 | type payloadReader struct{ b []byte } | 186 | type payloadReader struct{ b []byte } | ||
| 187 | 187 | ||||
| 188 | + | func (r *payloadReader) getBytes(n int) ([]byte, error) { | |||
| 189 | + | if n < 0 || len(r.b) < n { | |||
| 190 | + | return nil, ErrShortPayload | |||
| 191 | + | } | |||
| 192 | + | b := r.b[:n] | |||
| 193 | + | r.b = r.b[n:] | |||
| 194 | + | return b, nil | |||
| 195 | + | } | |||
| 196 | + | ||||
| 188 | func (r *payloadReader) getUint8() (uint8, error) { | 197 | func (r *payloadReader) getUint8() (uint8, error) { | ||
| 189 | if len(r.b) < 1 { | 198 | if len(r.b) < 1 { | ||
| 190 | return 0, ErrShortPayload | 199 | return 0, ErrShortPayload | ||
M internal/usecase/snapshot/codec_test.go +29 -0
| @@ -14,6 +14,35 @@ import ( | |||||
| 14 | "github.com/bnema/vev/pkg/vt" | 14 | "github.com/bnema/vev/pkg/vt" | ||
| 15 | ) | 15 | ) | ||
| 16 | 16 | ||||
| 17 | + | func TestPayloadReaderGetBytes(t *testing.T) { | |||
| 18 | + | for _, tc := range []struct { | |||
| 19 | + | name string | |||
| 20 | + | input []byte | |||
| 21 | + | n int | |||
| 22 | + | want []byte | |||
| 23 | + | remaining []byte | |||
| 24 | + | wantErr error | |||
| 25 | + | }{ | |||
| 26 | + | {name: "reads and advances", input: []byte{1, 2, 3}, n: 2, want: []byte{1, 2}, remaining: []byte{3}}, | |||
| 27 | + | {name: "reads empty", input: []byte{1}, n: 0, want: []byte{}, remaining: []byte{1}}, | |||
| 28 | + | {name: "short payload does not advance", input: []byte{1}, n: 2, remaining: []byte{1}, wantErr: ErrShortPayload}, | |||
| 29 | + | } { | |||
| 30 | + | t.Run(tc.name, func(t *testing.T) { | |||
| 31 | + | r := payloadReader{b: append([]byte(nil), tc.input...)} | |||
| 32 | + | got, err := r.getBytes(tc.n) | |||
| 33 | + | if !errors.Is(err, tc.wantErr) { | |||
| 34 | + | t.Fatalf("getBytes(%d) error = %v, want %v", tc.n, err, tc.wantErr) | |||
| 35 | + | } | |||
| 36 | + | if !bytes.Equal(got, tc.want) { | |||
| 37 | + | t.Fatalf("getBytes(%d) = %v, want %v", tc.n, got, tc.want) | |||
| 38 | + | } | |||
| 39 | + | if !bytes.Equal(r.b, tc.remaining) { | |||
| 40 | + | t.Fatalf("remaining bytes = %v, want %v", r.b, tc.remaining) | |||
| 41 | + | } | |||
| 42 | + | }) | |||
| 43 | + | } | |||
| 44 | + | } | |||
| 45 | + | ||||
| 17 | func TestV3SnapshotRoundTripPreservesExactTerminalData(t *testing.T) { | 46 | func TestV3SnapshotRoundTripPreservesExactTerminalData(t *testing.T) { | ||
| 18 | indexed := renderer.DefaultStyle() | 47 | indexed := renderer.DefaultStyle() | ||
| 19 | indexed.Bold, indexed.Inverse = true, true | 48 | indexed.Bold, indexed.Inverse = true, true | ||
M internal/usecase/snapshot/manifest.go +65 -9
| @@ -7,6 +7,7 @@ import ( | |||||
| 7 | "hash/crc32" | 7 | "hash/crc32" | ||
| 8 | "math" | 8 | "math" | ||
| 9 | 9 | ||||
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 10 | "github.com/bnema/vev/internal/ports" | 11 | "github.com/bnema/vev/internal/ports" | ||
| 11 | "github.com/bnema/vev/internal/usecase/layout" | 12 | "github.com/bnema/vev/internal/usecase/layout" | ||
| 12 | ) | 13 | ) | ||
| @@ -15,7 +16,7 @@ const ( | |||||
| 15 | manifestMagic = "VEVM" | 16 | manifestMagic = "VEVM" | ||
| 16 | headMagic = "VEVH" | 17 | headMagic = "VEVH" | ||
| 17 | objectMagic = "VEVO" | 18 | objectMagic = "VEVO" | ||
| 18 | - | ManifestVersion = uint16(1) | 19 | + | ManifestVersion = uint16(2) |
| 19 | manifestHeaderSize = 16 | 20 | manifestHeaderSize = 16 | ||
| 20 | objectEnvelopeBodyPrefixSize = 1 + 4 // object kind and payload length | 21 | objectEnvelopeBodyPrefixSize = 1 + 4 // object kind and payload length | ||
| 21 | minObjectEnvelopeSize = manifestHeaderSize + objectEnvelopeBodyPrefixSize + 1 | 22 | minObjectEnvelopeSize = manifestHeaderSize + objectEnvelopeBodyPrefixSize + 1 | ||
| @@ -96,11 +97,13 @@ func UnmarshalHead(encoded []byte) (Head, error) { | |||||
| 96 | 97 | ||||
| 97 | // Manifest is the complete VEVM payload carried directly in a publication. | 98 | // Manifest is the complete VEVM payload carried directly in a publication. | ||
| 98 | type Manifest struct { | 99 | type Manifest struct { | ||
| 99 | - | Generation uint64 | 100 | + | Generation uint64 |
| 100 | - | Name string | 101 | + | IncarnationID domain.IncarnationID |
| 101 | - | CreatedAt uint64 | 102 | + | ParentCheckpoint *domain.CheckpointRef |
| 102 | - | Active uint16 | 103 | + | Name string |
| 103 | - | Tabs []ManifestTab | 104 | + | CreatedAt uint64 |
| 105 | + | Active uint16 | |||
| 106 | + | Tabs []ManifestTab | |||
| 104 | } | 107 | } | ||
| 105 | 108 | ||||
| 106 | type ManifestTab struct { | 109 | type ManifestTab struct { | ||
| @@ -129,6 +132,14 @@ func MarshalManifest(m Manifest) ([]byte, error) { | |||||
| 129 | } | 132 | } | ||
| 130 | var w payloadWriter | 133 | var w payloadWriter | ||
| 131 | w.putUint64(m.Generation) | 134 | w.putUint64(m.Generation) | ||
| 135 | + | w.b = append(w.b, m.IncarnationID[:]...) | |||
| 136 | + | if m.ParentCheckpoint == nil { | |||
| 137 | + | w.putUint8(0) | |||
| 138 | + | } else { | |||
| 139 | + | w.putUint8(1) | |||
| 140 | + | w.putUint64(m.ParentCheckpoint.Generation) | |||
| 141 | + | w.b = append(w.b, m.ParentCheckpoint.ManifestDigest[:]...) | |||
| 142 | + | } | |||
| 132 | if err := w.putString(m.Name); err != nil { | 143 | if err := w.putString(m.Name); err != nil { | ||
| 133 | return nil, err | 144 | return nil, err | ||
| 134 | } | 145 | } | ||
| @@ -168,6 +179,33 @@ func decodeManifest(body []byte) (Manifest, error) { | |||||
| 168 | if err != nil { | 179 | if err != nil { | ||
| 169 | return Manifest{}, err | 180 | return Manifest{}, err | ||
| 170 | } | 181 | } | ||
| 182 | + | incarnationBytes, err := r.getBytes(len(domain.IncarnationID{})) | |||
| 183 | + | if err != nil { | |||
| 184 | + | return Manifest{}, err | |||
| 185 | + | } | |||
| 186 | + | var incarnationID domain.IncarnationID | |||
| 187 | + | copy(incarnationID[:], incarnationBytes) | |||
| 188 | + | parentPresent, err := r.getUint8() | |||
| 189 | + | if err != nil { | |||
| 190 | + | return Manifest{}, err | |||
| 191 | + | } | |||
| 192 | + | var parent *domain.CheckpointRef | |||
| 193 | + | switch parentPresent { | |||
| 194 | + | case 0: | |||
| 195 | + | case 1: | |||
| 196 | + | parent = &domain.CheckpointRef{} | |||
| 197 | + | parent.Generation, err = r.getUint64() | |||
| 198 | + | if err != nil { | |||
| 199 | + | return Manifest{}, err | |||
| 200 | + | } | |||
| 201 | + | digest, err := r.getBytes(len(parent.ManifestDigest)) | |||
| 202 | + | if err != nil { | |||
| 203 | + | return Manifest{}, err | |||
| 204 | + | } | |||
| 205 | + | copy(parent.ManifestDigest[:], digest) | |||
| 206 | + | default: | |||
| 207 | + | return Manifest{}, fmt.Errorf("%w: parent checkpoint presence", ErrInvalidData) | |||
| 208 | + | } | |||
| 171 | name, err := r.getString() | 209 | name, err := r.getString() | ||
| 172 | if err != nil { | 210 | if err != nil { | ||
| 173 | return Manifest{}, err | 211 | return Manifest{}, err | ||
| @@ -184,7 +222,7 @@ func decodeManifest(body []byte) (Manifest, error) { | |||||
| 184 | if err != nil { | 222 | if err != nil { | ||
| 185 | return Manifest{}, err | 223 | return Manifest{}, err | ||
| 186 | } | 224 | } | ||
| 187 | - | m := Manifest{Generation: generation, Name: name, CreatedAt: createdAt, Active: active, Tabs: make([]ManifestTab, n)} | 225 | + | m := Manifest{Generation: generation, IncarnationID: incarnationID, ParentCheckpoint: parent, Name: name, CreatedAt: createdAt, Active: active, Tabs: make([]ManifestTab, n)} |
| 188 | for i := range m.Tabs { | 226 | for i := range m.Tabs { | ||
| 189 | if m.Tabs[i], err = readManifestTab(&r); err != nil { | 227 | if m.Tabs[i], err = readManifestTab(&r); err != nil { | ||
| 190 | return Manifest{}, err | 228 | return Manifest{}, err | ||
| @@ -257,13 +295,20 @@ func marshalManifestEnvelope(magic string, body []byte) ([]byte, error) { | |||||
| 257 | } | 295 | } | ||
| 258 | out := make([]byte, manifestHeaderSize+len(body)) | 296 | out := make([]byte, manifestHeaderSize+len(body)) | ||
| 259 | copy(out[:4], magic) | 297 | copy(out[:4], magic) | ||
| 260 | - | binary.BigEndian.PutUint16(out[4:6], ManifestVersion) | 298 | + | binary.BigEndian.PutUint16(out[4:6], envelopeVersion(magic)) |
| 261 | binary.BigEndian.PutUint32(out[8:12], uint32(len(body))) | 299 | binary.BigEndian.PutUint32(out[8:12], uint32(len(body))) | ||
| 262 | binary.BigEndian.PutUint32(out[12:16], crc32.ChecksumIEEE(body)) | 300 | binary.BigEndian.PutUint32(out[12:16], crc32.ChecksumIEEE(body)) | ||
| 263 | copy(out[16:], body) | 301 | copy(out[16:], body) | ||
| 264 | return out, nil | 302 | return out, nil | ||
| 265 | } | 303 | } | ||
| 266 | 304 | ||||
| 305 | + | func envelopeVersion(magic string) uint16 { | |||
| 306 | + | if magic == manifestMagic { | |||
| 307 | + | return ManifestVersion | |||
| 308 | + | } | |||
| 309 | + | return 1 | |||
| 310 | + | } | |||
| 311 | + | ||||
| 267 | func unmarshalManifestEnvelope(encoded []byte, expectedMagic string) ([]byte, error) { | 312 | func unmarshalManifestEnvelope(encoded []byte, expectedMagic string) ([]byte, error) { | ||
| 268 | if len(encoded) < manifestHeaderSize { | 313 | if len(encoded) < manifestHeaderSize { | ||
| 269 | return nil, ErrShortPayload | 314 | return nil, ErrShortPayload | ||
| @@ -271,7 +316,7 @@ func unmarshalManifestEnvelope(encoded []byte, expectedMagic string) ([]byte, er | |||||
| 271 | if string(encoded[:4]) != expectedMagic { | 316 | if string(encoded[:4]) != expectedMagic { | ||
| 272 | return nil, ErrBadMagic | 317 | return nil, ErrBadMagic | ||
| 273 | } | 318 | } | ||
| 274 | - | if binary.BigEndian.Uint16(encoded[4:6]) != ManifestVersion { | 319 | + | if binary.BigEndian.Uint16(encoded[4:6]) != envelopeVersion(expectedMagic) { |
| 275 | return nil, ErrBadVersion | 320 | return nil, ErrBadVersion | ||
| 276 | } | 321 | } | ||
| 277 | if binary.BigEndian.Uint16(encoded[6:8]) != 0 { | 322 | if binary.BigEndian.Uint16(encoded[6:8]) != 0 { | ||
| @@ -542,6 +587,17 @@ func parseObjectRef(r *payloadReader) (ObjectKind, []byte, uint32, error) { | |||||
| 542 | } | 587 | } | ||
| 543 | 588 | ||||
| 544 | func validateManifest(m Manifest) error { | 589 | func validateManifest(m Manifest) error { | ||
| 590 | + | if m.Generation == 0 { | |||
| 591 | + | return fmt.Errorf("%w: manifest generation", ErrInvalidData) | |||
| 592 | + | } | |||
| 593 | + | if m.IncarnationID == (domain.IncarnationID{}) { | |||
| 594 | + | return fmt.Errorf("%w: manifest incarnation", ErrInvalidData) | |||
| 595 | + | } | |||
| 596 | + | if m.ParentCheckpoint != nil { | |||
| 597 | + | if m.ParentCheckpoint.Generation == 0 || m.ParentCheckpoint.Generation >= m.Generation || isZeroDigestBytes(m.ParentCheckpoint.ManifestDigest[:]) { | |||
| 598 | + | return fmt.Errorf("%w: parent checkpoint", ErrInvalidData) | |||
| 599 | + | } | |||
| 600 | + | } | |||
| 545 | if m.Name == "" { | 601 | if m.Name == "" { | ||
| 546 | return fmt.Errorf("%w: manifest name", ErrInvalidData) | 602 | return fmt.Errorf("%w: manifest name", ErrInvalidData) | ||
| 547 | } | 603 | } | ||
M internal/usecase/snapshot/manifest_preflight.go +36 -1
| @@ -3,6 +3,7 @@ package snapshot | |||||
| 3 | import ( | 3 | import ( | ||
| 4 | "fmt" | 4 | "fmt" | ||
| 5 | 5 | ||||
| 6 | + | "github.com/bnema/vev/internal/domain" | |||
| 6 | "github.com/bnema/vev/internal/usecase/layout" | 7 | "github.com/bnema/vev/internal/usecase/layout" | ||
| 7 | ) | 8 | ) | ||
| 8 | 9 | ||||
| @@ -11,9 +12,43 @@ import ( | |||||
| 11 | func preflightManifest(body []byte) error { | 12 | func preflightManifest(body []byte) error { | ||
| 12 | r := payloadReader{b: body} | 13 | r := payloadReader{b: body} | ||
| 13 | budget := manifestPreflightBudget{} | 14 | budget := manifestPreflightBudget{} | ||
| 14 | - | if _, err := r.getUint64(); err != nil { | 15 | + | generation, err := r.getUint64() |
| 16 | + | if err != nil { | |||
| 17 | + | return err | |||
| 18 | + | } | |||
| 19 | + | if generation == 0 { | |||
| 20 | + | return fmt.Errorf("%w: manifest generation", ErrInvalidData) | |||
| 21 | + | } | |||
| 22 | + | incarnationBytes, err := r.getBytes(len(domain.IncarnationID{})) | |||
| 23 | + | if err != nil { | |||
| 24 | + | return err | |||
| 25 | + | } | |||
| 26 | + | var incarnationID domain.IncarnationID | |||
| 27 | + | copy(incarnationID[:], incarnationBytes) | |||
| 28 | + | if incarnationID == (domain.IncarnationID{}) { | |||
| 29 | + | return fmt.Errorf("%w: manifest incarnation", ErrInvalidData) | |||
| 30 | + | } | |||
| 31 | + | parentPresent, err := r.getUint8() | |||
| 32 | + | if err != nil { | |||
| 15 | return err | 33 | return err | ||
| 16 | } | 34 | } | ||
| 35 | + | switch parentPresent { | |||
| 36 | + | case 0: | |||
| 37 | + | case 1: | |||
| 38 | + | parentGeneration, err := r.getUint64() | |||
| 39 | + | if err != nil { | |||
| 40 | + | return err | |||
| 41 | + | } | |||
| 42 | + | digest, err := r.getBytes(len(SnapshotDigest{})) | |||
| 43 | + | if err != nil { | |||
| 44 | + | return err | |||
| 45 | + | } | |||
| 46 | + | if parentGeneration == 0 || parentGeneration >= generation || isZeroDigestBytes(digest) { | |||
| 47 | + | return fmt.Errorf("%w: parent checkpoint", ErrInvalidData) | |||
| 48 | + | } | |||
| 49 | + | default: | |||
| 50 | + | return fmt.Errorf("%w: parent checkpoint presence", ErrInvalidData) | |||
| 51 | + | } | |||
| 17 | name, err := preflightManifestStringBytes(&r, &budget) | 52 | name, err := preflightManifestStringBytes(&r, &budget) | ||
| 18 | if err != nil { | 53 | if err != nil { | ||
| 19 | return err | 54 | return err | ||
M internal/usecase/snapshot/manifest_preflight_test.go +68 -4
| @@ -6,6 +6,7 @@ import ( | |||||
| 6 | "hash/crc32" | 6 | "hash/crc32" | ||
| 7 | "testing" | 7 | "testing" | ||
| 8 | 8 | ||||
| 9 | + | "github.com/bnema/vev/internal/domain" | |||
| 9 | "github.com/bnema/vev/internal/usecase/layout" | 10 | "github.com/bnema/vev/internal/usecase/layout" | ||
| 10 | ) | 11 | ) | ||
| 11 | 12 | ||||
| @@ -39,6 +40,7 @@ func TestManifestEnvelopePreflightParity(t *testing.T) { | |||||
| 39 | if err != nil { | 40 | if err != nil { | ||
| 40 | t.Fatal(err) | 41 | t.Fatal(err) | ||
| 41 | } | 42 | } | ||
| 43 | + | nameOffset, nodeKindOffset := manifestFieldOffsets(t, encoded) | |||
| 42 | mutations := []struct { | 44 | mutations := []struct { | ||
| 43 | name string | 45 | name string | ||
| 44 | edit func([]byte) []byte | 46 | edit func([]byte) []byte | ||
| @@ -50,9 +52,21 @@ func TestManifestEnvelopePreflightParity(t *testing.T) { | |||||
| 50 | {"body length", func(b []byte) []byte { binary.BigEndian.PutUint32(b[8:12], uint32(len(b))); return b }}, | 52 | {"body length", func(b []byte) []byte { binary.BigEndian.PutUint32(b[8:12], uint32(len(b))); return b }}, | ||
| 51 | {"crc", func(b []byte) []byte { b[12] ^= 1; return b }}, | 53 | {"crc", func(b []byte) []byte { b[12] ^= 1; return b }}, | ||
| 52 | {"trailing", func(b []byte) []byte { return append(b, 0) }}, | 54 | {"trailing", func(b []byte) []byte { return append(b, 0) }}, | ||
| 53 | - | {"body name", func(b []byte) []byte { b[26] ^= 1; binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])); return b }}, | 55 | + | {"body name", func(b []byte) []byte { |
| 54 | - | {"body generation", func(b []byte) []byte { b[16] ^= 1; binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])); return b }}, | 56 | + | b[nameOffset] = 0 |
| 55 | - | {"body node kind", func(b []byte) []byte { b[65] = 99; binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])); return b }}, | 57 | + | binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])) |
| 58 | + | return b | |||
| 59 | + | }}, | |||
| 60 | + | {"body generation", func(b []byte) []byte { | |||
| 61 | + | clear(b[manifestHeaderSize : manifestHeaderSize+8]) | |||
| 62 | + | binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])) | |||
| 63 | + | return b | |||
| 64 | + | }}, | |||
| 65 | + | {"body node kind", func(b []byte) []byte { | |||
| 66 | + | b[nodeKindOffset] = 99 | |||
| 67 | + | binary.BigEndian.PutUint32(b[12:16], checksum(b[16:])) | |||
| 68 | + | return b | |||
| 69 | + | }}, | |||
| 56 | } | 70 | } | ||
| 57 | for _, tc := range mutations { | 71 | for _, tc := range mutations { | ||
| 58 | t.Run(tc.name, func(t *testing.T) { | 72 | t.Run(tc.name, func(t *testing.T) { | ||
| @@ -67,6 +81,56 @@ func TestManifestEnvelopePreflightParity(t *testing.T) { | |||||
| 67 | } | 81 | } | ||
| 68 | } | 82 | } | ||
| 69 | 83 | ||||
| 84 | + | func manifestFieldOffsets(t *testing.T, encoded []byte) (nameOffset, nodeKindOffset int) { | |||
| 85 | + | t.Helper() | |||
| 86 | + | r := payloadReader{b: encoded[manifestHeaderSize:]} | |||
| 87 | + | must := func(err error) { | |||
| 88 | + | if err != nil { | |||
| 89 | + | t.Fatal(err) | |||
| 90 | + | } | |||
| 91 | + | } | |||
| 92 | + | _, err := r.getUint64() | |||
| 93 | + | must(err) | |||
| 94 | + | if len(r.b) < len(domain.IncarnationID{}) { | |||
| 95 | + | t.Fatalf("manifest body has %d incarnation bytes, want at least %d", len(r.b), len(domain.IncarnationID{})) | |||
| 96 | + | } | |||
| 97 | + | r.b = r.b[len(domain.IncarnationID{}):] | |||
| 98 | + | _, err = r.getUint8() | |||
| 99 | + | must(err) | |||
| 100 | + | nameLengthOffset := len(encoded) - len(r.b) | |||
| 101 | + | name, err := r.getString() | |||
| 102 | + | must(err) | |||
| 103 | + | nameOffset = nameLengthOffset + 2 | |||
| 104 | + | if len(name) == 0 { | |||
| 105 | + | t.Fatal("test manifest has empty name") | |||
| 106 | + | } | |||
| 107 | + | _, err = r.getUint64() | |||
| 108 | + | must(err) | |||
| 109 | + | _, err = r.getUint16() | |||
| 110 | + | must(err) | |||
| 111 | + | _, err = r.getUint16() | |||
| 112 | + | must(err) | |||
| 113 | + | _, err = r.getString() | |||
| 114 | + | must(err) | |||
| 115 | + | _, err = r.getUint16() | |||
| 116 | + | must(err) | |||
| 117 | + | _, err = r.getUint16() | |||
| 118 | + | must(err) | |||
| 119 | + | _, err = r.getUint64() | |||
| 120 | + | must(err) | |||
| 121 | + | _, err = r.getString() | |||
| 122 | + | must(err) | |||
| 123 | + | nodeKindOffset = len(encoded) - len(r.b) | |||
| 124 | + | const wantName = "named" | |||
| 125 | + | if nameOffset < 0 || nameOffset+len(wantName) > len(encoded) || string(encoded[nameOffset:nameOffset+len(wantName)]) != wantName { | |||
| 126 | + | t.Fatalf("name offset %d does not point to %q", nameOffset, wantName) | |||
| 127 | + | } | |||
| 128 | + | if nodeKindOffset < 0 || nodeKindOffset >= len(encoded) || encoded[nodeKindOffset] != manifestNodeLeaf { | |||
| 129 | + | t.Fatalf("node kind offset %d does not point to leaf kind", nodeKindOffset) | |||
| 130 | + | } | |||
| 131 | + | return nameOffset, nodeKindOffset | |||
| 132 | + | } | |||
| 133 | + | ||||
| 70 | func TestManifestObjectRefKinds(t *testing.T) { | 134 | func TestManifestObjectRefKinds(t *testing.T) { | ||
| 71 | for _, tc := range []struct { | 135 | for _, tc := range []struct { | ||
| 72 | name string | 136 | name string | ||
| @@ -163,7 +227,7 @@ func checksum(body []byte) uint32 { | |||||
| 163 | } | 227 | } | ||
| 164 | 228 | ||||
| 165 | func testManifest(tree *layout.Tree, process *Process) Manifest { | 229 | func testManifest(tree *layout.Tree, process *Process) Manifest { | ||
| 166 | - | return Manifest{Name: "named", Tabs: []ManifestTab{{ | 230 | + | return Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "named", Tabs: []ManifestTab{{ |
| 167 | StableID: "tab", Cols: 80, Rows: 24, Focus: "one", Tree: tree, | 231 | StableID: "tab", Cols: 80, Rows: 24, Focus: "one", Tree: tree, | ||
| 168 | Panes: []ManifestPane{ | 232 | Panes: []ManifestPane{ | ||
| 169 | {ID: "one", StableID: "one", Cwd: "/one", Sealed: []ObjectRef{{Kind: HistoryChunk, Digest: SnapshotDigest{1}, Size: minObjectEnvelopeSize}}, Tail: ObjectRef{Kind: HistoryTail, Digest: SnapshotDigest{2}, Size: minObjectEnvelopeSize}, Visible: ObjectRef{Kind: Visible, Digest: SnapshotDigest{3}, Size: minObjectEnvelopeSize}, Process: process}, | 233 | {ID: "one", StableID: "one", Cwd: "/one", Sealed: []ObjectRef{{Kind: HistoryChunk, Digest: SnapshotDigest{1}, Size: minObjectEnvelopeSize}}, Tail: ObjectRef{Kind: HistoryTail, Digest: SnapshotDigest{2}, Size: minObjectEnvelopeSize}, Visible: ObjectRef{Kind: Visible, Digest: SnapshotDigest{3}, Size: minObjectEnvelopeSize}, Process: process}, | ||
M internal/usecase/snapshot/manifest_test.go +49 -4
| @@ -3,14 +3,59 @@ package snapshot | |||||
| 3 | import ( | 3 | import ( | ||
| 4 | "bytes" | 4 | "bytes" | ||
| 5 | "crypto/sha256" | 5 | "crypto/sha256" | ||
| 6 | + | "encoding/binary" | |||
| 6 | "errors" | 7 | "errors" | ||
| 7 | "math" | 8 | "math" | ||
| 8 | "testing" | 9 | "testing" | ||
| 9 | 10 | ||||
| 11 | + | "github.com/bnema/vev/internal/domain" | |||
| 10 | "github.com/bnema/vev/internal/ports" | 12 | "github.com/bnema/vev/internal/ports" | ||
| 11 | "github.com/bnema/vev/internal/usecase/layout" | 13 | "github.com/bnema/vev/internal/usecase/layout" | ||
| 12 | ) | 14 | ) | ||
| 13 | 15 | ||||
| 16 | + | func TestManifestIncarnation(t *testing.T) { | |||
| 17 | + | manifest := manifestWithTailSize(minObjectEnvelopeSize) | |||
| 18 | + | manifest.Generation = 2 | |||
| 19 | + | manifest.IncarnationID = domain.IncarnationID{1} | |||
| 20 | + | manifest.ParentCheckpoint = &domain.CheckpointRef{Generation: 1, ManifestDigest: [32]byte{2}} | |||
| 21 | + | ||||
| 22 | + | encoded, err := MarshalManifest(manifest) | |||
| 23 | + | if err != nil { | |||
| 24 | + | t.Fatal(err) | |||
| 25 | + | } | |||
| 26 | + | decoded, err := UnmarshalManifest(encoded) | |||
| 27 | + | if err != nil { | |||
| 28 | + | t.Fatal(err) | |||
| 29 | + | } | |||
| 30 | + | if decoded.IncarnationID != manifest.IncarnationID || decoded.ParentCheckpoint == nil || *decoded.ParentCheckpoint != *manifest.ParentCheckpoint { | |||
| 31 | + | t.Fatalf("round trip = %#v", decoded) | |||
| 32 | + | } | |||
| 33 | + | reencoded, err := MarshalManifest(decoded) | |||
| 34 | + | if err != nil || !bytes.Equal(encoded, reencoded) { | |||
| 35 | + | t.Fatalf("manifest bytes are not canonical: %v", err) | |||
| 36 | + | } | |||
| 37 | + | if got := binary.BigEndian.Uint16(encoded[4:6]); got != ManifestVersion { | |||
| 38 | + | t.Fatalf("manifest version = %d", got) | |||
| 39 | + | } | |||
| 40 | + | for n := range len(encoded) { | |||
| 41 | + | if _, err := UnmarshalManifest(encoded[:n]); err == nil { | |||
| 42 | + | t.Fatalf("prefix %d accepted", n) | |||
| 43 | + | } | |||
| 44 | + | } | |||
| 45 | + | if _, err := UnmarshalManifest(append(encoded, 0)); err == nil { | |||
| 46 | + | t.Fatal("trailing data accepted") | |||
| 47 | + | } | |||
| 48 | + | versionOne := append([]byte(nil), encoded...) | |||
| 49 | + | binary.BigEndian.PutUint16(versionOne[4:6], 1) | |||
| 50 | + | if _, err := UnmarshalManifest(versionOne); !errors.Is(err, ErrBadVersion) { | |||
| 51 | + | t.Fatalf("manifest v1 error = %v", err) | |||
| 52 | + | } | |||
| 53 | + | manifest.IncarnationID = domain.IncarnationID{} | |||
| 54 | + | if _, err := MarshalManifest(manifest); err == nil { | |||
| 55 | + | t.Fatal("zero incarnation accepted") | |||
| 56 | + | } | |||
| 57 | + | } | |||
| 58 | + | ||||
| 14 | func TestManifestCodecUsesGenerationAndOrderedTypedReferences(t *testing.T) { | 59 | func TestManifestCodecUsesGenerationAndOrderedTypedReferences(t *testing.T) { | ||
| 15 | chunk, err := MarshalObject(HistoryChunk, []byte("history")) | 60 | chunk, err := MarshalObject(HistoryChunk, []byte("history")) | ||
| 16 | if err != nil { | 61 | if err != nil { | ||
| @@ -24,7 +69,7 @@ func TestManifestCodecUsesGenerationAndOrderedTypedReferences(t *testing.T) { | |||||
| 24 | if err != nil { | 69 | if err != nil { | ||
| 25 | t.Fatal(err) | 70 | t.Fatal(err) | ||
| 26 | } | 71 | } | ||
| 27 | - | manifest := Manifest{Generation: 7, Name: "named", CreatedAt: 9, Active: 0, Tabs: []ManifestTab{{ | 72 | + | manifest := Manifest{Generation: 7, IncarnationID: domain.IncarnationID{1}, Name: "named", CreatedAt: 9, Active: 0, Tabs: []ManifestTab{{ |
| 28 | StableID: "tab", Cols: 80, Rows: 24, NextPaneID: 2, Focus: "pane", Tree: layout.NewTree("pane"), | 73 | StableID: "tab", Cols: 80, Rows: 24, NextPaneID: 2, Focus: "pane", Tree: layout.NewTree("pane"), | ||
| 29 | Panes: []ManifestPane{{ID: "pane", StableID: "pane-stable", Cwd: "/tmp", | 74 | Panes: []ManifestPane{{ID: "pane", StableID: "pane-stable", Cwd: "/tmp", | ||
| 30 | Sealed: []ObjectRef{{Kind: HistoryChunk, Digest: chunk.Digest, Size: uint32(len(chunk.Data))}}, | 75 | Sealed: []ObjectRef{{Kind: HistoryChunk, Digest: chunk.Digest, Size: uint32(len(chunk.Data))}}, | ||
| @@ -97,7 +142,7 @@ func TestManifestDimensionBoundaries(t *testing.T) { | |||||
| 97 | {"hostile product", math.MaxUint16, math.MaxUint16, true}, | 142 | {"hostile product", math.MaxUint16, math.MaxUint16, true}, | ||
| 98 | } { | 143 | } { | ||
| 99 | t.Run(tc.name, func(t *testing.T) { | 144 | t.Run(tc.name, func(t *testing.T) { | ||
| 100 | - | m := Manifest{Name: "safe", Tabs: []ManifestTab{{Cols: tc.cols, Rows: tc.rows, Panes: []ManifestPane{{ID: "p", Tail: ref, Visible: visible}}}}} | 145 | + | m := Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "safe", Tabs: []ManifestTab{{Cols: tc.cols, Rows: tc.rows, Panes: []ManifestPane{{ID: "p", Tail: ref, Visible: visible}}}}} |
| 101 | _, err := MarshalManifest(m) | 146 | _, err := MarshalManifest(m) | ||
| 102 | if (err != nil) != tc.wantErr { | 147 | if (err != nil) != tc.wantErr { | ||
| 103 | t.Fatalf("MarshalManifest() error = %v, want error=%v", err, tc.wantErr) | 148 | t.Fatalf("MarshalManifest() error = %v, want error=%v", err, tc.wantErr) | ||
| @@ -107,7 +152,7 @@ func TestManifestDimensionBoundaries(t *testing.T) { | |||||
| 107 | } | 152 | } | ||
| 108 | 153 | ||||
| 109 | func TestManifestRejectsImpossibleObjectEnvelopeSize(t *testing.T) { | 154 | func TestManifestRejectsImpossibleObjectEnvelopeSize(t *testing.T) { | ||
| 110 | - | manifest := Manifest{Name: "named", Tabs: []ManifestTab{{Cols: 1, Rows: 1, Panes: []ManifestPane{{ | 155 | + | manifest := Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "named", Tabs: []ManifestTab{{Cols: 1, Rows: 1, Panes: []ManifestPane{{ |
| 111 | ID: "pane", | 156 | ID: "pane", | ||
| 112 | Tail: ObjectRef{Kind: HistoryTail, Digest: SnapshotDigest{1}, Size: 1}, | 157 | Tail: ObjectRef{Kind: HistoryTail, Digest: SnapshotDigest{1}, Size: 1}, | ||
| 113 | Visible: ObjectRef{Kind: Visible, Digest: SnapshotDigest{2}, Size: minObjectEnvelopeSize}, | 158 | Visible: ObjectRef{Kind: Visible, Digest: SnapshotDigest{2}, Size: minObjectEnvelopeSize}, | ||
| @@ -171,7 +216,7 @@ func TestObjectRefEnvelopeSizeBoundaries(t *testing.T) { | |||||
| 171 | } | 216 | } | ||
| 172 | 217 | ||||
| 173 | func manifestWithTailSize(size uint32) Manifest { | 218 | func manifestWithTailSize(size uint32) Manifest { | ||
| 174 | - | return Manifest{Name: "named", Tabs: []ManifestTab{{ | 219 | + | return Manifest{Generation: 1, IncarnationID: domain.IncarnationID{1}, Name: "named", Tabs: []ManifestTab{{ |
| 175 | Cols: 1, | 220 | Cols: 1, | ||
| 176 | Rows: 1, | 221 | Rows: 1, | ||
| 177 | Panes: []ManifestPane{{ | 222 | Panes: []ManifestPane{{ | ||
M internal/usecase/snapshot/weights_test.go +3 -3
| @@ -86,10 +86,10 @@ func TestSnapshotEnvelopeCompatibility(t *testing.T) { | |||||
| 86 | const ( | 86 | const ( | ||
| 87 | headGolden = "56455648000100000000002898ebcac200000000000000070100000000000000000000000000000000000000000000000000000000000000" | 87 | headGolden = "56455648000100000000002898ebcac200000000000000070100000000000000000000000000000000000000000000000000000000000000" | ||
| 88 | objectGolden = "5645564f00010000000000111cb8f09f030000000c63616e6f6e6963616c2d7674" | 88 | objectGolden = "5645564f00010000000000111cb8f09f030000000c63616e6f6e6963616c2d7674" | ||
| 89 | - | manifestGolden = "5645564d0001000000000122f8f8f0af000000000000000000056e616d6564000000000000000000000001000374616200500018000000000000000000036f6e65010000020000036f6e6500000374776f000200036f6e6500036f6e6500042f6f6e65000101010000000000000000000000000000000000000000000000000000000000000000000016020200000000000000000000000000000000000000000000000000000000000000000000160303000000000000000000000000000000000000000000000000000000000000000000001600000374776f000374776f00042f74776f0000020400000000000000000000000000000000000000000000000000000000000000000000160305000000000000000000000000000000000000000000000000000000000000000000001600" | 89 | + | manifestGolden = "5645564d0002000000000133d118cc030000000000000001010000000000000000000000000000000000056e616d6564000000000000000000000001000374616200500018000000000000000000036f6e65010000020000036f6e6500000374776f000200036f6e6500036f6e6500042f6f6e65000101010000000000000000000000000000000000000000000000000000000000000000000016020200000000000000000000000000000000000000000000000000000000000000000000160303000000000000000000000000000000000000000000000000000000000000000000001600000374776f000374776f00042f74776f0000020400000000000000000000000000000000000000000000000000000000000000000000160305000000000000000000000000000000000000000000000000000000000000000000001600" |
| 90 | ) | 90 | ) | ||
| 91 | - | if ManifestVersion != 1 { | 91 | + | if ManifestVersion != 2 { |
| 92 | - | t.Fatalf("ManifestVersion = %d, want 1", ManifestVersion) | 92 | + | t.Fatalf("ManifestVersion = %d, want 2", ManifestVersion) |
| 93 | } | 93 | } | ||
| 94 | 94 | ||||
| 95 | head, err := MarshalHead(Head{Generation: 7, ManifestDigest: SnapshotDigest{1}}) | 95 | head, err := MarshalHead(Head{Generation: 7, ManifestDigest: SnapshotDigest{1}}) | ||
M pkg/kv/codec.go +84 -55
| @@ -5,77 +5,106 @@ import ( | |||||
| 5 | "errors" | 5 | "errors" | ||
| 6 | "hash/crc32" | 6 | "hash/crc32" | ||
| 7 | "math" | 7 | "math" | ||
| 8 | + | "sort" | |||
| 8 | ) | 9 | ) | ||
| 9 | 10 | ||||
| 10 | - | const ( | 11 | + | var ErrCorrupt = errors.New("kv: corrupt store file") |
| 11 | - | opSet byte = 0 | |||
| 12 | - | opDel byte = 1 | |||
| 13 | - | ||||
| 14 | - | headerLen = 8 | |||
| 15 | - | payloadPrefixLen = 3 | |||
| 16 | - | compactThreshold = 64 | |||
| 17 | - | compactWasteRatio = 0.5 | |||
| 18 | - | maxKeyLen = 1<<16 - 1 | |||
| 19 | - | ) | |||
| 20 | 12 | ||||
| 21 | - | var errBadRecord = errors.New("bad record") | 13 | + | var fileMagic = [4]byte{'V', 'E', 'V', 'K'} |
| 22 | 14 | ||||
| 23 | - | type record struct { | 15 | + | const ( |
| 24 | - | op byte | 16 | + | fileVersion uint16 = 3 |
| 25 | - | key []byte | 17 | + | filePrefixLen = 4 + 2 + 4 |
| 26 | - | value []byte | 18 | + | entryPrefixLen = 4 + 4 |
| 27 | - | } | 19 | + | crcLen = 4 |
| 20 | + | ) | |||
| 28 | 21 | ||||
| 29 | - | func encodeRecord(op byte, key, value []byte) ([]byte, error) { | 22 | + | // encodeFile writes magic | version | count | (keyLen key valLen val)* | crc32. |
| 30 | - | if len(key) > maxKeyLen { | 23 | + | func encodeFile(data map[string][]byte) []byte { |
| 31 | - | return nil, errors.New("key too large") | 24 | + | keys := make([]string, 0, len(data)) |
| 32 | - | } | 25 | + | for key := range data { |
| 33 | - | if op != opSet && op != opDel { | 26 | + | keys = append(keys, key) |
| 34 | - | return nil, errBadRecord | |||
| 35 | } | 27 | } | ||
| 28 | + | sort.Strings(keys) | |||
| 36 | 29 | ||||
| 37 | - | payloadLen := payloadPrefixLen + len(key) | 30 | + | size := filePrefixLen + crcLen |
| 38 | - | if op == opSet { | 31 | + | for _, key := range keys { |
| 39 | - | payloadLen += len(value) | 32 | + | size += entryPrefixLen + len(key) + len(data[key]) |
| 40 | } | 33 | } | ||
| 41 | - | if uint64(payloadLen) > math.MaxUint32 { | 34 | + | buf := make([]byte, 0, size) |
| 42 | - | return nil, errors.New("payload too large") | 35 | + | buf = append(buf, fileMagic[:]...) |
| 43 | - | } | 36 | + | buf = binary.BigEndian.AppendUint16(buf, fileVersion) |
| 44 | - | 37 | + | buf = binary.BigEndian.AppendUint32(buf, uint32(len(keys))) | |
| 45 | - | buf := make([]byte, headerLen+payloadLen) | 38 | + | for _, key := range keys { |
| 46 | - | payload := buf[headerLen:] | 39 | + | value := data[key] |
| 47 | - | payload[0] = op | 40 | + | buf = binary.BigEndian.AppendUint32(buf, uint32(len(key))) |
| 48 | - | binary.BigEndian.PutUint16(payload[1:3], uint16(len(key))) | 41 | + | buf = append(buf, key...) |
| 49 | - | copy(payload[3:], key) | 42 | + | buf = binary.BigEndian.AppendUint32(buf, uint32(len(value))) |
| 50 | - | if op == opSet { | 43 | + | buf = append(buf, value...) |
| 51 | - | copy(payload[3+len(key):], value) | |||
| 52 | } | 44 | } | ||
| 45 | + | return appendCRC(buf) | |||
| 46 | + | } | |||
| 53 | 47 | ||||
| 54 | - | binary.BigEndian.PutUint32(buf[0:4], uint32(payloadLen)) | 48 | + | func appendCRC(payload []byte) []byte { |
| 55 | - | binary.BigEndian.PutUint32(buf[4:8], crc32.ChecksumIEEE(payload)) | 49 | + | return binary.BigEndian.AppendUint32(payload, crc32.ChecksumIEEE(payload)) |
| 56 | - | return buf, nil | |||
| 57 | } | 50 | } | ||
| 58 | 51 | ||||
| 59 | - | func decodePayload(payload []byte) (record, error) { | 52 | + | // decodeFile validates the single trailing CRC and rejects every malformed |
| 60 | - | if len(payload) < payloadPrefixLen { | 53 | + | // existing file. Callers must not replace or truncate a file that fails here. |
| 61 | - | return record{}, errBadRecord | 54 | + | func decodeFile(raw []byte) (map[string][]byte, error) { |
| 55 | + | if len(raw) < filePrefixLen+crcLen { | |||
| 56 | + | return nil, ErrCorrupt | |||
| 62 | } | 57 | } | ||
| 63 | - | op := payload[0] | 58 | + | payload := raw[:len(raw)-crcLen] |
| 64 | - | if op != opSet && op != opDel { | 59 | + | wantCRC := binary.BigEndian.Uint32(raw[len(raw)-crcLen:]) |
| 65 | - | return record{}, errBadRecord | 60 | + | if crc32.ChecksumIEEE(payload) != wantCRC { |
| 61 | + | return nil, ErrCorrupt | |||
| 66 | } | 62 | } | ||
| 67 | - | keyLen := int(binary.BigEndian.Uint16(payload[1:3])) | 63 | + | if string(payload[:4]) != string(fileMagic[:]) || binary.BigEndian.Uint16(payload[4:6]) != fileVersion { |
| 68 | - | if len(payload) < payloadPrefixLen+keyLen { | 64 | + | return nil, ErrCorrupt |
| 69 | - | return record{}, errBadRecord | |||
| 70 | } | 65 | } | ||
| 71 | - | if op == opDel && len(payload) != payloadPrefixLen+keyLen { | 66 | + | |
| 72 | - | return record{}, errBadRecord | 67 | + | count := binary.BigEndian.Uint32(payload[6:10]) |
| 68 | + | off := filePrefixLen | |||
| 69 | + | data := make(map[string][]byte) | |||
| 70 | + | for range count { | |||
| 71 | + | keyLen, next, ok := decodeLength(payload, off) | |||
| 72 | + | if !ok { | |||
| 73 | + | return nil, ErrCorrupt | |||
| 74 | + | } | |||
| 75 | + | off = next | |||
| 76 | + | if keyLen > uint64(len(payload)-off) { | |||
| 77 | + | return nil, ErrCorrupt | |||
| 78 | + | } | |||
| 79 | + | key := string(payload[off : off+int(keyLen)]) | |||
| 80 | + | off += int(keyLen) | |||
| 81 | + | valueLen, next, ok := decodeLength(payload, off) | |||
| 82 | + | if !ok { | |||
| 83 | + | return nil, ErrCorrupt | |||
| 84 | + | } | |||
| 85 | + | off = next | |||
| 86 | + | if valueLen > uint64(len(payload)-off) { | |||
| 87 | + | return nil, ErrCorrupt | |||
| 88 | + | } | |||
| 89 | + | if _, duplicate := data[key]; duplicate { | |||
| 90 | + | return nil, ErrCorrupt | |||
| 91 | + | } | |||
| 92 | + | data[key] = append([]byte(nil), payload[off:off+int(valueLen)]...) | |||
| 93 | + | off += int(valueLen) | |||
| 73 | } | 94 | } | ||
| 95 | + | if off != len(payload) { | |||
| 96 | + | return nil, ErrCorrupt | |||
| 97 | + | } | |||
| 98 | + | return data, nil | |||
| 99 | + | } | |||
| 74 | 100 | ||||
| 75 | - | key := append([]byte(nil), payload[payloadPrefixLen:payloadPrefixLen+keyLen]...) | 101 | + | func decodeLength(payload []byte, off int) (uint64, int, bool) { |
| 76 | - | var val []byte | 102 | + | if off < 0 || len(payload)-off < 4 { |
| 77 | - | if op == opSet { | 103 | + | return 0, off, false |
| 78 | - | val = append([]byte(nil), payload[payloadPrefixLen+keyLen:]...) | 104 | + | } |
| 105 | + | length := uint64(binary.BigEndian.Uint32(payload[off : off+4])) | |||
| 106 | + | if length > uint64(math.MaxInt) { | |||
| 107 | + | return 0, off, false | |||
| 79 | } | 108 | } | ||
| 80 | - | return record{op: op, key: key, value: val}, nil | 109 | + | return length, off + 4, true |
| 81 | } | 110 | } | ||
M pkg/kv/kv.go +114 -285
| @@ -1,10 +1,7 @@ | |||||
| 1 | package kv | 1 | package kv | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | - | "encoding/binary" | |||
| 5 | "errors" | 4 | "errors" | ||
| 6 | - | "fmt" | |||
| 7 | - | "hash/crc32" | |||
| 8 | "io" | 5 | "io" | ||
| 9 | "os" | 6 | "os" | ||
| 10 | "path/filepath" | 7 | "path/filepath" | ||
| @@ -15,369 +12,201 @@ import ( | |||||
| 15 | "github.com/bnema/vev/pkg/safedir" | 12 | "github.com/bnema/vev/pkg/safedir" | ||
| 16 | ) | 13 | ) | ||
| 17 | 14 | ||||
| 18 | - | // Store is an append-only WAL-backed key/value store. | 15 | + | // Store is an in-memory key/value map persisted by whole-file atomic rewrite. |
| 16 | + | // Set and Delete mutate memory only; Sync is the durability barrier. | |||
| 19 | type Store struct { | 17 | type Store struct { | ||
| 20 | - | mu sync.Mutex | 18 | + | mu sync.Mutex |
| 21 | - | path string | 19 | + | path string |
| 22 | - | file *os.File | 20 | + | lockFile *os.File |
| 23 | - | lockFile *os.File | 21 | + | data map[string][]byte |
| 24 | - | data map[string][]byte | 22 | + | dirty bool |
| 25 | - | entrySize map[string]int64 | 23 | + | closed bool |
| 26 | - | total int64 | |||
| 27 | - | live int64 | |||
| 28 | - | closed bool | |||
| 29 | } | 24 | } | ||
| 30 | 25 | ||||
| 31 | - | // Open opens or creates a store at path, replays valid records, truncates any | 26 | + | // Open reads path, or starts empty when it does not exist. Stray .tmp files |
| 32 | - | // corrupt or torn tail, and compacts if the file contains enough obsolete data. | 27 | + | // are ignored. Any invalid existing main file fails closed and is untouched. |
| 33 | func Open(path string) (*Store, error) { | 28 | func Open(path string) (*Store, error) { | ||
| 34 | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { | 29 | if err := safedir.EnsurePrivate(filepath.Dir(path)); err != nil { | ||
| 35 | return nil, err | 30 | return nil, err | ||
| 36 | } | 31 | } | ||
| 37 | - | ||||
| 38 | lockFile, err := acquireLock(path) | 32 | lockFile, err := acquireLock(path) | ||
| 39 | if err != nil { | 33 | if err != nil { | ||
| 40 | return nil, err | 34 | return nil, err | ||
| 41 | } | 35 | } | ||
| 42 | - | 36 | + | fail := func(err error) (*Store, error) { | |
| 43 | - | f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) | 37 | + | return nil, errors.Join(err, releaseLock(lockFile)) |
| 44 | - | if err != nil { | |||
| 45 | - | _ = releaseLock(lockFile) | |||
| 46 | - | return nil, err | |||
| 47 | - | } | |||
| 48 | - | ||||
| 49 | - | data, sizes, total, live, lastGood, err := replayFile(f) | |||
| 50 | - | if err != nil { | |||
| 51 | - | _ = f.Close() | |||
| 52 | - | _ = releaseLock(lockFile) | |||
| 53 | - | return nil, err | |||
| 54 | - | } | |||
| 55 | - | if err := f.Truncate(lastGood); err != nil { | |||
| 56 | - | _ = f.Close() | |||
| 57 | - | _ = releaseLock(lockFile) | |||
| 58 | - | return nil, err | |||
| 59 | - | } | |||
| 60 | - | if _, err := f.Seek(0, io.SeekEnd); err != nil { | |||
| 61 | - | _ = f.Close() | |||
| 62 | - | _ = releaseLock(lockFile) | |||
| 63 | - | return nil, err | |||
| 64 | - | } | |||
| 65 | - | ||||
| 66 | - | s := &Store{path: path, file: f, lockFile: lockFile, data: data, entrySize: sizes, total: total, live: live} | |||
| 67 | - | if s.shouldCompact() { | |||
| 68 | - | if err := s.compactLocked(); err != nil { | |||
| 69 | - | _ = f.Close() | |||
| 70 | - | _ = releaseLock(lockFile) | |||
| 71 | - | return nil, err | |||
| 72 | - | } | |||
| 73 | } | 38 | } | ||
| 74 | - | return s, nil | |||
| 75 | - | } | |||
| 76 | 39 | ||||
| 77 | - | // Replay reads path without mutating it and returns the state from valid records. | 40 | + | raw, err := os.ReadFile(path) |
| 78 | - | func Replay(path string) (map[string][]byte, error) { | |||
| 79 | - | f, err := os.Open(path) | |||
| 80 | if errors.Is(err, os.ErrNotExist) { | 41 | if errors.Is(err, os.ErrNotExist) { | ||
| 81 | - | return map[string][]byte{}, nil | 42 | + | store := &Store{path: path, lockFile: lockFile, data: make(map[string][]byte), dirty: true} |
| 43 | + | if err := store.syncLocked(); err != nil { | |||
| 44 | + | return fail(err) | |||
| 45 | + | } | |||
| 46 | + | return store, nil | |||
| 82 | } | 47 | } | ||
| 83 | if err != nil { | 48 | if err != nil { | ||
| 84 | - | return nil, err | 49 | + | return fail(err) |
| 85 | } | 50 | } | ||
| 86 | - | defer func() { _ = f.Close() }() | 51 | + | data, err := decodeFile(raw) |
| 87 | - | ||||
| 88 | - | data, _, _, _, _, err := replayFile(f) | |||
| 89 | if err != nil { | 52 | if err != nil { | ||
| 90 | - | return nil, err | 53 | + | return fail(err) |
| 91 | } | 54 | } | ||
| 92 | - | return data, nil | 55 | + | return &Store{path: path, lockFile: lockFile, data: data}, nil |
| 93 | } | 56 | } | ||
| 94 | 57 | ||||
| 95 | - | // Get returns a copy of the value for key. | |||
| 96 | func (s *Store) Get(key []byte) ([]byte, bool) { | 58 | func (s *Store) Get(key []byte) ([]byte, bool) { | ||
| 97 | s.mu.Lock() | 59 | s.mu.Lock() | ||
| 98 | defer s.mu.Unlock() | 60 | defer s.mu.Unlock() | ||
| 99 | - | 61 | + | value, ok := s.data[string(key)] | |
| 100 | - | v, ok := s.data[string(key)] | 62 | + | return append([]byte(nil), value...), ok |
| 101 | - | if !ok { | |||
| 102 | - | return nil, false | |||
| 103 | - | } | |||
| 104 | - | return append([]byte(nil), v...), true | |||
| 105 | } | 63 | } | ||
| 106 | 64 | ||||
| 107 | - | // Set appends a SET record. It does not fsync. | 65 | + | func (s *Store) Set(key, value []byte) error { |
| 108 | - | func (s *Store) Set(key, val []byte) error { | 66 | + | s.mu.Lock() |
| 109 | - | return s.appendRecord(opSet, key, val) | 67 | + | defer s.mu.Unlock() |
| 68 | + | if s.closed { | |||
| 69 | + | return os.ErrClosed | |||
| 70 | + | } | |||
| 71 | + | s.data[string(key)] = append([]byte(nil), value...) | |||
| 72 | + | s.dirty = true | |||
| 73 | + | return nil | |||
| 110 | } | 74 | } | ||
| 111 | 75 | ||||
| 112 | - | // Delete appends a DEL tombstone. It does not fsync. | |||
| 113 | func (s *Store) Delete(key []byte) error { | 76 | func (s *Store) Delete(key []byte) error { | ||
| 114 | - | return s.appendRecord(opDel, key, nil) | 77 | + | s.mu.Lock() |
| 78 | + | defer s.mu.Unlock() | |||
| 79 | + | if s.closed { | |||
| 80 | + | return os.ErrClosed | |||
| 81 | + | } | |||
| 82 | + | if _, exists := s.data[string(key)]; exists { | |||
| 83 | + | delete(s.data, string(key)) | |||
| 84 | + | s.dirty = true | |||
| 85 | + | } | |||
| 86 | + | return nil | |||
| 115 | } | 87 | } | ||
| 116 | 88 | ||||
| 117 | - | // Range calls fn for each key/value pair, using copies. Iteration stops when fn returns false. | |||
| 118 | func (s *Store) Range(fn func(k, v []byte) bool) { | 89 | func (s *Store) Range(fn func(k, v []byte) bool) { | ||
| 119 | s.mu.Lock() | 90 | s.mu.Lock() | ||
| 120 | keys := make([]string, 0, len(s.data)) | 91 | keys := make([]string, 0, len(s.data)) | ||
| 121 | - | for k := range s.data { | 92 | + | for key := range s.data { |
| 122 | - | keys = append(keys, k) | 93 | + | keys = append(keys, key) |
| 123 | } | 94 | } | ||
| 124 | sort.Strings(keys) | 95 | sort.Strings(keys) | ||
| 125 | - | pairs := make([]struct{ k, v []byte }, 0, len(keys)) | 96 | + | values := make([][]byte, len(keys)) |
| 126 | - | for _, k := range keys { | 97 | + | for i, key := range keys { |
| 127 | - | pairs = append(pairs, struct{ k, v []byte }{[]byte(k), append([]byte(nil), s.data[k]...)}) | 98 | + | values[i] = append([]byte(nil), s.data[key]...) |
| 128 | } | 99 | } | ||
| 129 | s.mu.Unlock() | 100 | s.mu.Unlock() | ||
| 130 | - | 101 | + | for i, key := range keys { | |
| 131 | - | for _, p := range pairs { | 102 | + | if !fn([]byte(key), values[i]) { |
| 132 | - | if !fn(p.k, p.v) { | |||
| 133 | return | 103 | return | ||
| 134 | } | 104 | } | ||
| 135 | } | 105 | } | ||
| 136 | } | 106 | } | ||
| 137 | 107 | ||||
| 138 | - | // Sync fsyncs the WAL file. | 108 | + | // Sync writes path.tmp, fsyncs it, renames it over path, then fsyncs the |
| 109 | + | // directory. The main pathname therefore contains either the previous whole | |||
| 110 | + | // map or the next whole map, never a subset of mutations. | |||
| 139 | func (s *Store) Sync() error { | 111 | func (s *Store) Sync() error { | ||
| 140 | s.mu.Lock() | 112 | s.mu.Lock() | ||
| 141 | defer s.mu.Unlock() | 113 | defer s.mu.Unlock() | ||
| 142 | if s.closed { | 114 | if s.closed { | ||
| 143 | return os.ErrClosed | 115 | return os.ErrClosed | ||
| 144 | } | 116 | } | ||
| 145 | - | return s.file.Sync() | 117 | + | return s.syncLocked() |
| 146 | } | 118 | } | ||
| 147 | 119 | ||||
| 148 | - | // Close fsyncs and closes the store. | 120 | + | func (s *Store) syncLocked() error { |
| 149 | - | func (s *Store) Close() error { | 121 | + | if !s.dirty { |
| 150 | - | s.mu.Lock() | |||
| 151 | - | defer s.mu.Unlock() | |||
| 152 | - | if s.closed { | |||
| 153 | return nil | 122 | return nil | ||
| 154 | } | 123 | } | ||
| 155 | - | errSync := s.file.Sync() | 124 | + | data := encodeFile(s.data) |
| 156 | - | errClose := s.file.Close() | 125 | + | tmp := s.path + ".tmp" |
| 157 | - | errUnlock := releaseLock(s.lockFile) | 126 | + | file, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) |
| 158 | - | s.closed = true | |||
| 159 | - | if errSync != nil { | |||
| 160 | - | return errSync | |||
| 161 | - | } | |||
| 162 | - | if errClose != nil { | |||
| 163 | - | return errClose | |||
| 164 | - | } | |||
| 165 | - | return errUnlock | |||
| 166 | - | } | |||
| 167 | - | ||||
| 168 | - | func acquireLock(path string) (*os.File, error) { | |||
| 169 | - | lf, err := os.OpenFile(path+".lock", os.O_RDWR|os.O_CREATE, 0o600) | |||
| 170 | if err != nil { | 127 | if err != nil { | ||
| 171 | - | return nil, err | 128 | + | return err |
| 172 | - | } | |||
| 173 | - | if err := syscall.Flock(int(lf.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { | |||
| 174 | - | _ = lf.Close() | |||
| 175 | - | return nil, err | |||
| 176 | } | 129 | } | ||
| 177 | - | return lf, nil | 130 | + | if err := writeAll(file, data); err != nil { |
| 178 | - | } | 131 | + | return errors.Join(err, file.Close()) |
| 179 | - | ||||
| 180 | - | func releaseLock(f *os.File) error { | |||
| 181 | - | if f == nil { | |||
| 182 | - | return nil | |||
| 183 | } | 132 | } | ||
| 184 | - | errUnlock := syscall.Flock(int(f.Fd()), syscall.LOCK_UN) | 133 | + | if err := file.Sync(); err != nil { |
| 185 | - | errClose := f.Close() | 134 | + | return errors.Join(err, file.Close()) |
| 186 | - | if errUnlock != nil { | |||
| 187 | - | return errUnlock | |||
| 188 | } | 135 | } | ||
| 189 | - | return errClose | 136 | + | if err := file.Close(); err != nil { |
| 190 | - | } | |||
| 191 | - | ||||
| 192 | - | func (s *Store) appendRecord(op byte, key, val []byte) error { | |||
| 193 | - | buf, err := encodeRecord(op, key, val) | |||
| 194 | - | if err != nil { | |||
| 195 | return err | 137 | return err | ||
| 196 | } | 138 | } | ||
| 197 | - | 139 | + | if err := os.Rename(tmp, s.path); err != nil { | |
| 198 | - | s.mu.Lock() | |||
| 199 | - | defer s.mu.Unlock() | |||
| 200 | - | if s.closed { | |||
| 201 | - | return os.ErrClosed | |||
| 202 | - | } | |||
| 203 | - | pos, err := s.file.Seek(0, io.SeekCurrent) | |||
| 204 | - | if err != nil { | |||
| 205 | return err | 140 | return err | ||
| 206 | } | 141 | } | ||
| 207 | - | n, err := s.file.Write(buf) | 142 | + | if err := syncDir(filepath.Dir(s.path)); err != nil { |
| 208 | - | if err != nil || n != len(buf) { | 143 | + | return err |
| 209 | - | writeErr := err | |||
| 210 | - | if writeErr == nil { | |||
| 211 | - | writeErr = io.ErrShortWrite | |||
| 212 | - | } | |||
| 213 | - | if tErr := s.file.Truncate(pos); tErr != nil { | |||
| 214 | - | s.closed = true | |||
| 215 | - | return fmt.Errorf("write failed (%w) and recovery truncate failed: %w", writeErr, tErr) | |||
| 216 | - | } | |||
| 217 | - | if _, sErr := s.file.Seek(pos, io.SeekStart); sErr != nil { | |||
| 218 | - | s.closed = true | |||
| 219 | - | return fmt.Errorf("write failed (%w) and recovery seek failed: %w", writeErr, sErr) | |||
| 220 | - | } | |||
| 221 | - | return writeErr | |||
| 222 | - | } | |||
| 223 | - | ||||
| 224 | - | k := string(key) | |||
| 225 | - | if old, ok := s.entrySize[k]; ok { | |||
| 226 | - | s.live -= old | |||
| 227 | - | } | |||
| 228 | - | s.total += int64(len(buf)) | |||
| 229 | - | if op == opSet { | |||
| 230 | - | s.data[k] = append([]byte(nil), val...) | |||
| 231 | - | s.entrySize[k] = int64(len(buf)) | |||
| 232 | - | s.live += int64(len(buf)) | |||
| 233 | - | } else { | |||
| 234 | - | delete(s.data, k) | |||
| 235 | - | delete(s.entrySize, k) | |||
| 236 | - | } | |||
| 237 | - | if s.shouldCompact() { | |||
| 238 | - | return s.compactLocked() | |||
| 239 | } | 144 | } | ||
| 145 | + | s.dirty = false | |||
| 240 | return nil | 146 | return nil | ||
| 241 | } | 147 | } | ||
| 242 | 148 | ||||
| 243 | - | func (s *Store) shouldCompact() bool { | 149 | + | // Close syncs buffered mutations before releasing the process lock. |
| 244 | - | return s.total > compactThreshold && float64(s.total-s.live)/float64(s.total) > compactWasteRatio | 150 | + | func (s *Store) Close() error { |
| 245 | - | } | 151 | + | s.mu.Lock() |
| 246 | - | 152 | + | defer s.mu.Unlock() | |
| 247 | - | func (s *Store) compactLocked() error { | 153 | + | if s.closed { |
| 248 | - | tmp := s.path + ".compact" | 154 | + | return nil |
| 249 | - | tf, err := os.OpenFile(tmp, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600) | |||
| 250 | - | if err != nil { | |||
| 251 | - | return err | |||
| 252 | } | 155 | } | ||
| 156 | + | err := s.syncLocked() | |||
| 157 | + | s.closed = true | |||
| 158 | + | return errors.Join(err, releaseLock(s.lockFile)) | |||
| 159 | + | } | |||
| 253 | 160 | ||||
| 254 | - | keys := make([]string, 0, len(s.data)) | 161 | + | // CloseWithoutSync releases the process lock without making buffered mutations |
| 255 | - | for k := range s.data { | 162 | + | // durable. It is used when a higher-level transaction rejected those mutations. |
| 256 | - | keys = append(keys, k) | 163 | + | func (s *Store) CloseWithoutSync() error { |
| 164 | + | s.mu.Lock() | |||
| 165 | + | defer s.mu.Unlock() | |||
| 166 | + | if s.closed { | |||
| 167 | + | return nil | |||
| 257 | } | 168 | } | ||
| 258 | - | sort.Strings(keys) | 169 | + | s.closed = true |
| 170 | + | return releaseLock(s.lockFile) | |||
| 171 | + | } | |||
| 259 | 172 | ||||
| 260 | - | newSizes := make(map[string]int64, len(keys)) | 173 | + | func writeAll(file *os.File, data []byte) error { |
| 261 | - | var newTotal, newLive int64 | 174 | + | for len(data) > 0 { |
| 262 | - | for _, k := range keys { | 175 | + | n, err := file.Write(data) |
| 263 | - | buf, err := encodeRecord(opSet, []byte(k), s.data[k]) | |||
| 264 | if err != nil { | 176 | if err != nil { | ||
| 265 | - | _ = tf.Close() | |||
| 266 | - | _ = os.Remove(tmp) | |||
| 267 | return err | 177 | return err | ||
| 268 | } | 178 | } | ||
| 269 | - | if _, err := tf.Write(buf); err != nil { | 179 | + | if n == 0 { |
| 270 | - | _ = tf.Close() | 180 | + | return io.ErrShortWrite |
| 271 | - | _ = os.Remove(tmp) | |||
| 272 | - | return err | |||
| 273 | } | 181 | } | ||
| 274 | - | sz := int64(len(buf)) | 182 | + | data = data[n:] |
| 275 | - | newSizes[k] = sz | |||
| 276 | - | newTotal += sz | |||
| 277 | - | newLive += sz | |||
| 278 | - | } | |||
| 279 | - | if err := tf.Sync(); err != nil { | |||
| 280 | - | _ = tf.Close() | |||
| 281 | - | _ = os.Remove(tmp) | |||
| 282 | - | return err | |||
| 283 | } | 183 | } | ||
| 284 | - | if err := tf.Close(); err != nil { | |||
| 285 | - | _ = os.Remove(tmp) | |||
| 286 | - | return err | |||
| 287 | - | } | |||
| 288 | - | bak := s.path + ".bak" | |||
| 289 | - | _ = os.Remove(bak) | |||
| 290 | - | if err := os.Rename(s.path, bak); err != nil { | |||
| 291 | - | _ = os.Remove(tmp) | |||
| 292 | - | return err | |||
| 293 | - | } | |||
| 294 | - | if err := os.Rename(tmp, s.path); err != nil { | |||
| 295 | - | _ = os.Rename(bak, s.path) | |||
| 296 | - | _ = os.Remove(tmp) | |||
| 297 | - | return err | |||
| 298 | - | } | |||
| 299 | - | f, err := os.OpenFile(s.path, os.O_RDWR, 0o600) | |||
| 300 | - | if err != nil { | |||
| 301 | - | _ = os.Remove(s.path) | |||
| 302 | - | _ = os.Rename(bak, s.path) | |||
| 303 | - | return err | |||
| 304 | - | } | |||
| 305 | - | if _, err := f.Seek(0, io.SeekEnd); err != nil { | |||
| 306 | - | _ = f.Close() | |||
| 307 | - | _ = os.Remove(s.path) | |||
| 308 | - | _ = os.Rename(bak, s.path) | |||
| 309 | - | return err | |||
| 310 | - | } | |||
| 311 | - | old := s.file | |||
| 312 | - | s.file = f | |||
| 313 | - | s.entrySize = newSizes | |||
| 314 | - | s.total = newTotal | |||
| 315 | - | s.live = newLive | |||
| 316 | - | _ = old.Close() | |||
| 317 | - | _ = os.Remove(bak) | |||
| 318 | return nil | 184 | return nil | ||
| 319 | } | 185 | } | ||
| 320 | 186 | ||||
| 321 | - | func replayFile(f *os.File) (map[string][]byte, map[string]int64, int64, int64, int64, error) { | 187 | + | func acquireLock(path string) (*os.File, error) { |
| 322 | - | info, err := f.Stat() | 188 | + | file, err := os.OpenFile(path+".lock", os.O_RDWR|os.O_CREATE, 0o600) |
| 323 | if err != nil { | 189 | if err != nil { | ||
| 324 | - | return nil, nil, 0, 0, 0, err | 190 | + | return nil, err |
| 325 | } | 191 | } | ||
| 326 | - | fileSize := info.Size() | 192 | + | if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { |
| 327 | - | if _, err := f.Seek(0, io.SeekStart); err != nil { | 193 | + | _ = file.Close() |
| 328 | - | return nil, nil, 0, 0, 0, err | 194 | + | return nil, err |
| 329 | } | 195 | } | ||
| 330 | - | data := make(map[string][]byte) | 196 | + | return file, nil |
| 331 | - | sizes := make(map[string]int64) | 197 | + | } |
| 332 | - | var total, live, off int64 | |||
| 333 | - | header := make([]byte, headerLen) | |||
| 334 | - | for { | |||
| 335 | - | n, err := io.ReadFull(f, header) | |||
| 336 | - | if errors.Is(err, io.EOF) { | |||
| 337 | - | break | |||
| 338 | - | } | |||
| 339 | - | if errors.Is(err, io.ErrUnexpectedEOF) { | |||
| 340 | - | _ = n | |||
| 341 | - | break | |||
| 342 | - | } | |||
| 343 | - | if err != nil { | |||
| 344 | - | return nil, nil, 0, 0, 0, err | |||
| 345 | - | } | |||
| 346 | 198 | ||||
| 347 | - | payloadLen := binary.BigEndian.Uint32(header[0:4]) | 199 | + | func releaseLock(file *os.File) error { |
| 348 | - | if int64(payloadLen) > fileSize-off-headerLen { | 200 | + | if file == nil { |
| 349 | - | break | 201 | + | return nil |
| 350 | - | } | 202 | + | } |
| 351 | - | wantCRC := binary.BigEndian.Uint32(header[4:8]) | 203 | + | return errors.Join(syscall.Flock(int(file.Fd()), syscall.LOCK_UN), file.Close()) |
| 352 | - | payload := make([]byte, int(payloadLen)) | 204 | + | } |
| 353 | - | if _, err := io.ReadFull(f, payload); errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { | |||
| 354 | - | break | |||
| 355 | - | } else if err != nil { | |||
| 356 | - | return nil, nil, 0, 0, 0, err | |||
| 357 | - | } | |||
| 358 | - | if crc32.ChecksumIEEE(payload) != wantCRC { | |||
| 359 | - | break | |||
| 360 | - | } | |||
| 361 | - | rec, err := decodePayload(payload) | |||
| 362 | - | if err != nil { | |||
| 363 | - | break | |||
| 364 | - | } | |||
| 365 | 205 | ||||
| 366 | - | sz := int64(headerLen + len(payload)) | 206 | + | func syncDir(dir string) error { |
| 367 | - | k := string(rec.key) | 207 | + | file, err := os.Open(dir) |
| 368 | - | if old, ok := sizes[k]; ok { | 208 | + | if err != nil { |
| 369 | - | live -= old | 209 | + | return err |
| 370 | - | } | |||
| 371 | - | total += sz | |||
| 372 | - | if rec.op == opSet { | |||
| 373 | - | data[k] = rec.value | |||
| 374 | - | sizes[k] = sz | |||
| 375 | - | live += sz | |||
| 376 | - | } else { | |||
| 377 | - | delete(data, k) | |||
| 378 | - | delete(sizes, k) | |||
| 379 | - | } | |||
| 380 | - | off += sz | |||
| 381 | } | 210 | } | ||
| 382 | - | return data, sizes, total, live, off, nil | 211 | + | return errors.Join(file.Sync(), file.Close()) |
| 383 | } | 212 | } | ||
M pkg/kv/kv_test.go +192 -230
| @@ -1,259 +1,221 @@ | |||||
| 1 | package kv | 1 | package kv | ||
| 2 | 2 | ||||
| 3 | import ( | 3 | import ( | ||
| 4 | - | "bytes" | |||
| 5 | "encoding/binary" | 4 | "encoding/binary" | ||
| 5 | + | "errors" | |||
| 6 | "os" | 6 | "os" | ||
| 7 | "path/filepath" | 7 | "path/filepath" | ||
| 8 | "testing" | 8 | "testing" | ||
| 9 | - | ) | |||
| 10 | 9 | ||||
| 11 | - | func privatePath(t *testing.T) string { | 10 | + | "github.com/stretchr/testify/require" |
| 12 | - | t.Helper() | 11 | + | ) |
| 13 | - | return filepath.Join(t.TempDir(), "vev", "store.kv") | |||
| 14 | - | } | |||
| 15 | 12 | ||||
| 16 | - | func closeStore(t *testing.T, s *Store) { | 13 | + | func TestSyncCrashBoundaries(t *testing.T) { |
| 17 | - | t.Helper() | 14 | + | t.Parallel() |
| 18 | - | if err := s.Close(); err != nil { | 15 | + | tests := []struct { |
| 19 | - | t.Fatal(err) | 16 | + | name string |
| 17 | + | leaveTmp bool | |||
| 18 | + | corrupt bool | |||
| 19 | + | wantOpen bool | |||
| 20 | + | wantValue string | |||
| 21 | + | }{ | |||
| 22 | + | {name: "clean file opens", wantOpen: true, wantValue: "v1"}, | |||
| 23 | + | {name: "stray tmp is ignored", leaveTmp: true, wantOpen: true, wantValue: "v1"}, | |||
| 24 | + | {name: "corrupt file fails closed", corrupt: true}, | |||
| 25 | + | } | |||
| 26 | + | for _, tt := range tests { | |||
| 27 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 28 | + | t.Parallel() | |||
| 29 | + | path := privateStorePath(t) | |||
| 30 | + | store, err := Open(path) | |||
| 31 | + | require.NoError(t, err) | |||
| 32 | + | require.NoError(t, store.Set([]byte("k"), []byte("v1"))) | |||
| 33 | + | require.NoError(t, store.Sync()) | |||
| 34 | + | require.NoError(t, store.Close()) | |||
| 35 | + | ||||
| 36 | + | if tt.leaveTmp { | |||
| 37 | + | require.NoError(t, os.WriteFile(path+".tmp", []byte("half written"), 0o600)) | |||
| 38 | + | } | |||
| 39 | + | if tt.corrupt { | |||
| 40 | + | data, err := os.ReadFile(path) | |||
| 41 | + | require.NoError(t, err) | |||
| 42 | + | data[len(data)-1] ^= 0xff | |||
| 43 | + | require.NoError(t, os.WriteFile(path, data, 0o600)) | |||
| 44 | + | } | |||
| 45 | + | ||||
| 46 | + | reopened, err := Open(path) | |||
| 47 | + | if !tt.wantOpen { | |||
| 48 | + | require.ErrorIs(t, err, ErrCorrupt) | |||
| 49 | + | return | |||
| 50 | + | } | |||
| 51 | + | require.NoError(t, err) | |||
| 52 | + | t.Cleanup(func() { _ = reopened.Close() }) | |||
| 53 | + | got, ok := reopened.Get([]byte("k")) | |||
| 54 | + | require.True(t, ok) | |||
| 55 | + | require.Equal(t, tt.wantValue, string(got)) | |||
| 56 | + | }) | |||
| 20 | } | 57 | } | ||
| 21 | } | 58 | } | ||
| 22 | 59 | ||||
| 23 | - | func TestSetGetDeleteRange(t *testing.T) { | 60 | + | func TestSyncIsAtomicAcrossKeys(t *testing.T) { |
| 24 | - | path := privatePath(t) | 61 | + | t.Parallel() |
| 25 | - | s, err := Open(path) | 62 | + | path := privateStorePath(t) |
| 26 | - | if err != nil { | 63 | + | store, err := Open(path) |
| 27 | - | t.Fatal(err) | 64 | + | require.NoError(t, err) |
| 28 | - | } | 65 | + | require.NoError(t, store.Set([]byte("old"), []byte("x"))) |
| 29 | - | defer closeStore(t, s) | 66 | + | require.NoError(t, store.Sync()) |
| 30 | - | 67 | + | require.NoError(t, store.Delete([]byte("old"))) | |
| 31 | - | if err := s.Set([]byte("b"), []byte("two")); err != nil { | 68 | + | require.NoError(t, store.Set([]byte("new"), []byte("x"))) |
| 32 | - | t.Fatal(err) | 69 | + | require.NoError(t, store.Sync()) |
| 33 | - | } | 70 | + | require.NoError(t, store.Close()) |
| 34 | - | if err := s.Set([]byte("a"), []byte("one")); err != nil { | 71 | + | |
| 35 | - | t.Fatal(err) | 72 | + | reopened, err := Open(path) |
| 36 | - | } | 73 | + | require.NoError(t, err) |
| 37 | - | got, ok := s.Get([]byte("a")) | 74 | + | t.Cleanup(func() { _ = reopened.Close() }) |
| 38 | - | if !ok || string(got) != "one" { | 75 | + | _, hadOld := reopened.Get([]byte("old")) |
| 39 | - | t.Fatalf("Get(a) = %q, %v", got, ok) | 76 | + | _, hasNew := reopened.Get([]byte("new")) |
| 40 | - | } | 77 | + | require.False(t, hadOld) |
| 41 | - | got[0] = 'X' | 78 | + | require.True(t, hasNew) |
| 42 | - | got, _ = s.Get([]byte("a")) | |||
| 43 | - | if string(got) != "one" { | |||
| 44 | - | t.Fatalf("Get returned mutable value: %q", got) | |||
| 45 | - | } | |||
| 46 | - | if err := s.Delete([]byte("b")); err != nil { | |||
| 47 | - | t.Fatal(err) | |||
| 48 | - | } | |||
| 49 | - | if _, ok := s.Get([]byte("b")); ok { | |||
| 50 | - | t.Fatal("deleted key still present") | |||
| 51 | - | } | |||
| 52 | - | ||||
| 53 | - | seen := map[string]string{} | |||
| 54 | - | s.Range(func(k, v []byte) bool { | |||
| 55 | - | seen[string(k)] = string(v) | |||
| 56 | - | return true | |||
| 57 | - | }) | |||
| 58 | - | if len(seen) != 1 || seen["a"] != "one" { | |||
| 59 | - | t.Fatalf("Range saw %#v", seen) | |||
| 60 | - | } | |||
| 61 | } | 79 | } | ||
| 62 | 80 | ||||
| 63 | - | func TestReopenDurability(t *testing.T) { | 81 | + | func TestOpenFailsClosedWithoutMutation(t *testing.T) { |
| 64 | - | path := privatePath(t) | 82 | + | t.Parallel() |
| 65 | - | s, err := Open(path) | 83 | + | tests := []struct { |
| 66 | - | if err != nil { | 84 | + | name string |
| 67 | - | t.Fatal(err) | 85 | + | raw []byte |
| 68 | - | } | 86 | + | }{ |
| 69 | - | if err := s.Set([]byte("k"), []byte("v")); err != nil { | 87 | + | {name: "empty", raw: []byte{}}, |
| 70 | - | t.Fatal(err) | 88 | + | {name: "short magic", raw: []byte("VE")}, |
| 71 | - | } | 89 | + | {name: "unknown magic", raw: func() []byte { |
| 72 | - | if err := s.Close(); err != nil { | 90 | + | payload := append([]byte(nil), validEmptyFile()[:filePrefixLen]...) |
| 73 | - | t.Fatal(err) | 91 | + | copy(payload[:4], "NOPE") |
| 74 | - | } | 92 | + | return appendCRC(payload) |
| 75 | - | 93 | + | }()}, | |
| 76 | - | s, err = Open(path) | 94 | + | {name: "unknown version", raw: func() []byte { |
| 77 | - | if err != nil { | 95 | + | payload := append([]byte(nil), validEmptyFile()[:filePrefixLen]...) |
| 78 | - | t.Fatal(err) | 96 | + | binary.BigEndian.PutUint16(payload[4:6], 99) |
| 79 | - | } | 97 | + | return appendCRC(payload) |
| 80 | - | defer closeStore(t, s) | 98 | + | }()}, |
| 81 | - | got, ok := s.Get([]byte("k")) | 99 | + | {name: "truncated entry", raw: func() []byte { |
| 82 | - | if !ok || string(got) != "v" { | 100 | + | payload := append([]byte(nil), validEmptyFile()[:filePrefixLen]...) |
| 83 | - | t.Fatalf("after reopen Get(k) = %q, %v", got, ok) | 101 | + | binary.BigEndian.PutUint32(payload[6:10], 1) |
| 102 | + | payload = append(payload, 0, 0, 0, 1) | |||
| 103 | + | return appendCRC(payload) | |||
| 104 | + | }()}, | |||
| 105 | + | } | |||
| 106 | + | for _, tt := range tests { | |||
| 107 | + | t.Run(tt.name, func(t *testing.T) { | |||
| 108 | + | t.Parallel() | |||
| 109 | + | path := privateStorePath(t) | |||
| 110 | + | require.NoError(t, os.WriteFile(path, tt.raw, 0o600)) | |||
| 111 | + | _, err := Open(path) | |||
| 112 | + | require.ErrorIs(t, err, ErrCorrupt) | |||
| 113 | + | after, readErr := os.ReadFile(path) | |||
| 114 | + | require.NoError(t, readErr) | |||
| 115 | + | require.Equal(t, tt.raw, after) | |||
| 116 | + | }) | |||
| 84 | } | 117 | } | ||
| 85 | } | 118 | } | ||
| 86 | 119 | ||||
| 87 | - | func TestCompactionShrinksWithDataIntact(t *testing.T) { | 120 | + | func TestCloseSyncsDirtyStore(t *testing.T) { |
| 88 | - | path := privatePath(t) | 121 | + | t.Parallel() |
| 89 | - | s, err := Open(path) | 122 | + | path := privateStorePath(t) |
| 90 | - | if err != nil { | 123 | + | store, err := Open(path) |
| 91 | - | t.Fatal(err) | 124 | + | require.NoError(t, err) |
| 92 | - | } | 125 | + | require.NoError(t, store.Set([]byte("k"), []byte("v"))) |
| 93 | - | if err := s.Set([]byte("keep"), bytes.Repeat([]byte("x"), 128)); err != nil { | 126 | + | require.NoError(t, store.Close()) |
| 94 | - | t.Fatal(err) | 127 | + | |
| 95 | - | } | 128 | + | reopened, err := Open(path) |
| 96 | - | info, err := os.Stat(path) | 129 | + | require.NoError(t, err) |
| 97 | - | if err != nil { | 130 | + | t.Cleanup(func() { _ = reopened.Close() }) |
| 98 | - | t.Fatal(err) | 131 | + | got, ok := reopened.Get([]byte("k")) |
| 99 | - | } | 132 | + | require.True(t, ok) |
| 100 | - | before := info.Size() | 133 | + | require.Equal(t, []byte("v"), got) |
| 101 | - | if err := s.Set([]byte("keep"), []byte("y")); err != nil { | |||
| 102 | - | t.Fatal(err) | |||
| 103 | - | } | |||
| 104 | - | info, err = os.Stat(path) | |||
| 105 | - | if err != nil { | |||
| 106 | - | t.Fatal(err) | |||
| 107 | - | } | |||
| 108 | - | after := info.Size() | |||
| 109 | - | if after >= before { | |||
| 110 | - | t.Fatalf("expected compaction to shrink file: before=%d after=%d", before, after) | |||
| 111 | - | } | |||
| 112 | - | got, ok := s.Get([]byte("keep")) | |||
| 113 | - | if !ok || string(got) != "y" { | |||
| 114 | - | t.Fatalf("compacted value = %q, %v", got, ok) | |||
| 115 | - | } | |||
| 116 | - | if err := s.Close(); err != nil { | |||
| 117 | - | t.Fatal(err) | |||
| 118 | - | } | |||
| 119 | } | 134 | } | ||
| 120 | 135 | ||||
| 121 | - | func TestOpenTruncatesMidRecordTail(t *testing.T) { | 136 | + | func TestStoreCopiesValuesAndRangesInKeyOrder(t *testing.T) { |
| 122 | - | path := privatePath(t) | 137 | + | t.Parallel() |
| 123 | - | first, _ := encodeRecord(opSet, []byte("a"), []byte("one")) | 138 | + | store, err := Open(privateStorePath(t)) |
| 124 | - | second, _ := encodeRecord(opSet, []byte("b"), []byte("two")) | 139 | + | require.NoError(t, err) |
| 125 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | 140 | + | t.Cleanup(func() { _ = store.Close() }) |
| 126 | - | t.Fatal(err) | 141 | + | value := []byte("two") |
| 127 | - | } | 142 | + | require.NoError(t, store.Set([]byte("b"), value)) |
| 128 | - | if err := os.WriteFile(path, append(first, second[:len(second)/2]...), 0o600); err != nil { | 143 | + | value[0] = 'x' |
| 129 | - | t.Fatal(err) | 144 | + | require.NoError(t, store.Set([]byte("a"), []byte("one"))) |
| 130 | - | } | 145 | + | got, ok := store.Get([]byte("b")) |
| 131 | - | 146 | + | require.True(t, ok) | |
| 132 | - | s, err := Open(path) | 147 | + | require.Equal(t, []byte("two"), got) |
| 133 | - | if err != nil { | 148 | + | got[0] = 'x' |
| 134 | - | t.Fatal(err) | 149 | + | gotAgain, _ := store.Get([]byte("b")) |
| 135 | - | } | 150 | + | require.Equal(t, []byte("two"), gotAgain) |
| 136 | - | defer closeStore(t, s) | 151 | + | |
| 137 | - | if got, ok := s.Get([]byte("a")); !ok || string(got) != "one" { | 152 | + | var keys []string |
| 138 | - | t.Fatalf("good record missing: %q %v", got, ok) | 153 | + | store.Range(func(k, v []byte) bool { |
| 139 | - | } | 154 | + | keys = append(keys, string(k)) |
| 140 | - | if _, ok := s.Get([]byte("b")); ok { | 155 | + | return true |
| 141 | - | t.Fatal("torn record was replayed") | 156 | + | }) |
| 142 | - | } | 157 | + | require.Equal(t, []string{"a", "b"}, keys) |
| 143 | - | info, err := os.Stat(path) | |||
| 144 | - | if err != nil { | |||
| 145 | - | t.Fatal(err) | |||
| 146 | - | } | |||
| 147 | - | if info.Size() != int64(len(first)) { | |||
| 148 | - | t.Fatalf("tail not truncated: size=%d want=%d", info.Size(), len(first)) | |||
| 149 | - | } | |||
| 150 | } | 158 | } | ||
| 151 | 159 | ||||
| 152 | - | func TestCRCcorruptionOfLastRecordDropsIt(t *testing.T) { | 160 | + | func TestStoreLockAndClosedErrors(t *testing.T) { |
| 153 | - | path := privatePath(t) | 161 | + | t.Parallel() |
| 154 | - | first, _ := encodeRecord(opSet, []byte("k"), []byte("old")) | 162 | + | path := privateStorePath(t) |
| 155 | - | second, _ := encodeRecord(opSet, []byte("k"), []byte("new")) | 163 | + | store, err := Open(path) |
| 156 | - | buf := append(append([]byte{}, first...), second...) | 164 | + | require.NoError(t, err) |
| 157 | - | buf[len(first)+headerLen+payloadPrefixLen+1] ^= 0xff | 165 | + | _, err = Open(path) |
| 158 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | 166 | + | require.Error(t, err) |
| 159 | - | t.Fatal(err) | 167 | + | require.NoError(t, store.Close()) |
| 160 | - | } | 168 | + | require.NoError(t, store.Close()) |
| 161 | - | if err := os.WriteFile(path, buf, 0o600); err != nil { | 169 | + | require.ErrorIs(t, store.Set([]byte("k"), []byte("v")), os.ErrClosed) |
| 162 | - | t.Fatal(err) | 170 | + | require.ErrorIs(t, store.Delete([]byte("k")), os.ErrClosed) |
| 163 | - | } | 171 | + | require.ErrorIs(t, store.Sync(), os.ErrClosed) |
| 164 | - | ||||
| 165 | - | s, err := Open(path) | |||
| 166 | - | if err != nil { | |||
| 167 | - | t.Fatal(err) | |||
| 168 | - | } | |||
| 169 | - | defer closeStore(t, s) | |||
| 170 | - | got, ok := s.Get([]byte("k")) | |||
| 171 | - | if !ok || string(got) != "old" { | |||
| 172 | - | t.Fatalf("corrupt last record not dropped: %q %v", got, ok) | |||
| 173 | - | } | |||
| 174 | - | info, err := os.Stat(path) | |||
| 175 | - | if err != nil { | |||
| 176 | - | t.Fatal(err) | |||
| 177 | - | } | |||
| 178 | - | if info.Size() != int64(len(first)) { | |||
| 179 | - | t.Fatalf("corrupt tail not truncated: size=%d want=%d", info.Size(), len(first)) | |||
| 180 | - | } | |||
| 181 | } | 172 | } | ||
| 182 | 173 | ||||
| 183 | - | func TestAbsentAndEmptyFile(t *testing.T) { | 174 | + | func privateStorePath(t *testing.T) string { |
| 184 | - | path := filepath.Join(t.TempDir(), "missing", "store.kv") | 175 | + | t.Helper() |
| 185 | - | m, err := Replay(path) | 176 | + | dir := filepath.Join(t.TempDir(), "vev") |
| 186 | - | if err != nil { | 177 | + | require.NoError(t, os.Mkdir(dir, 0o700)) |
| 187 | - | t.Fatal(err) | 178 | + | return filepath.Join(dir, "sessions.kv") |
| 188 | - | } | |||
| 189 | - | if len(m) != 0 { | |||
| 190 | - | t.Fatalf("Replay(absent) = %#v", m) | |||
| 191 | - | } | |||
| 192 | - | s, err := Open(path) | |||
| 193 | - | if err != nil { | |||
| 194 | - | t.Fatal(err) | |||
| 195 | - | } | |||
| 196 | - | if err := s.Close(); err != nil { | |||
| 197 | - | t.Fatal(err) | |||
| 198 | - | } | |||
| 199 | - | m, err = Replay(path) | |||
| 200 | - | if err != nil { | |||
| 201 | - | t.Fatal(err) | |||
| 202 | - | } | |||
| 203 | - | if len(m) != 0 { | |||
| 204 | - | t.Fatalf("Replay(empty) = %#v", m) | |||
| 205 | - | } | |||
| 206 | } | 179 | } | ||
| 207 | 180 | ||||
| 208 | - | func TestTombstoneSurvivesCompaction(t *testing.T) { | 181 | + | func validEmptyFile() []byte { |
| 209 | - | path := privatePath(t) | 182 | + | return encodeFile(map[string][]byte{}) |
| 210 | - | s, err := Open(path) | 183 | + | } |
| 211 | - | if err != nil { | |||
| 212 | - | t.Fatal(err) | |||
| 213 | - | } | |||
| 214 | - | if err := s.Set([]byte("dead"), bytes.Repeat([]byte("d"), 128)); err != nil { | |||
| 215 | - | t.Fatal(err) | |||
| 216 | - | } | |||
| 217 | - | if err := s.Set([]byte("keep"), bytes.Repeat([]byte("k"), 128)); err != nil { | |||
| 218 | - | t.Fatal(err) | |||
| 219 | - | } | |||
| 220 | - | if err := s.Delete([]byte("dead")); err != nil { | |||
| 221 | - | t.Fatal(err) | |||
| 222 | - | } | |||
| 223 | - | if err := s.Close(); err != nil { | |||
| 224 | - | t.Fatal(err) | |||
| 225 | - | } | |||
| 226 | 184 | ||||
| 227 | - | s, err = Open(path) | 185 | + | func TestDecodeRejectsDuplicateKeysAndTrailingData(t *testing.T) { |
| 228 | - | if err != nil { | 186 | + | t.Parallel() |
| 229 | - | t.Fatal(err) | 187 | + | payload := make([]byte, 0) |
| 230 | - | } | 188 | + | payload = append(payload, fileMagic[:]...) |
| 231 | - | defer closeStore(t, s) | 189 | + | payload = binary.BigEndian.AppendUint16(payload, fileVersion) |
| 232 | - | if _, ok := s.Get([]byte("dead")); ok { | 190 | + | payload = binary.BigEndian.AppendUint32(payload, 2) |
| 233 | - | t.Fatal("deleted key resurrected after compaction/reopen") | 191 | + | for range 2 { |
| 234 | - | } | 192 | + | payload = binary.BigEndian.AppendUint32(payload, 1) |
| 235 | - | if got, ok := s.Get([]byte("keep")); !ok || len(got) != 128 || got[0] != 'k' { | 193 | + | payload = append(payload, 'k') |
| 236 | - | t.Fatalf("live key missing after compaction/reopen: len=%d ok=%v", len(got), ok) | 194 | + | payload = binary.BigEndian.AppendUint32(payload, 1) |
| 237 | - | } | 195 | + | payload = append(payload, 'v') |
| 196 | + | } | |||
| 197 | + | raw := appendCRC(payload) | |||
| 198 | + | _, err := decodeFile(raw) | |||
| 199 | + | require.ErrorIs(t, err, ErrCorrupt) | |||
| 200 | + | ||||
| 201 | + | payload = append([]byte(nil), validEmptyFile()[:filePrefixLen]...) | |||
| 202 | + | payload = append(payload, 0) | |||
| 203 | + | _, err = decodeFile(appendCRC(payload)) | |||
| 204 | + | require.True(t, errors.Is(err, ErrCorrupt)) | |||
| 238 | } | 205 | } | ||
| 239 | 206 | ||||
| 240 | - | func TestReplayStopsAtBadPayloadLengthWithoutAllocatingHugeBuffer(t *testing.T) { | 207 | + | func TestCloseWithoutSyncPreventsLaterCloseFromSyncing(t *testing.T) { |
| 241 | - | path := privatePath(t) | 208 | + | t.Parallel() |
| 242 | - | first, _ := encodeRecord(opSet, []byte("a"), []byte("one")) | 209 | + | path := privateStorePath(t) |
| 243 | - | header := make([]byte, headerLen) | 210 | + | store, err := Open(path) |
| 244 | - | binary.BigEndian.PutUint32(header[0:4], 1<<31) | 211 | + | require.NoError(t, err) |
| 245 | - | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { | 212 | + | require.NoError(t, store.Set([]byte("rejected"), []byte("value"))) |
| 246 | - | t.Fatal(err) | 213 | + | require.NoError(t, store.CloseWithoutSync()) |
| 247 | - | } | 214 | + | require.NoError(t, store.Close(), "a later normal Close must remain a no-op") |
| 248 | - | if err := os.WriteFile(path, append(first, header...), 0o600); err != nil { | 215 | + | |
| 249 | - | t.Fatal(err) | 216 | + | reopened, err := Open(path) |
| 250 | - | } | 217 | + | require.NoError(t, err) |
| 251 | - | s, err := Open(path) | 218 | + | t.Cleanup(func() { _ = reopened.Close() }) |
| 252 | - | if err != nil { | 219 | + | _, ok := reopened.Get([]byte("rejected")) |
| 253 | - | t.Fatal(err) | 220 | + | require.False(t, ok) |
| 254 | - | } | |||
| 255 | - | defer closeStore(t, s) | |||
| 256 | - | if got, ok := s.Get([]byte("a")); !ok || string(got) != "one" { | |||
| 257 | - | t.Fatalf("good record missing before bad length: %q %v", got, ok) | |||
| 258 | - | } | |||
| 259 | } | 221 | } | ||