1// SPDX-License-Identifier: AGPL-3.0-or-later
2
3package domain
4
5import (
6 "fmt"
7 "strings"
8 "time"
9)
10
11type MirrorState string
12
13const (
14 MirrorDisabled MirrorState = "disabled"
15 MirrorOK MirrorState = "ok"
16 MirrorFailing MirrorState = "failing"
17 MirrorPaused MirrorState = "paused"
18)
19
20type MirrorFailureClass string
21
22const (
23 MirrorClassNone MirrorFailureClass = ""
24 MirrorClassDiverged MirrorFailureClass = "diverged" // non-fast-forward / fetch-first -> paused
25 MirrorClassAuthDenied MirrorFailureClass = "auth_denied" // "Permission denied (publickey)" -> paused
26 MirrorClassRemoteNotFound MirrorFailureClass = "remote_not_found" // "Repository not found", "does not appear to be a git repository" -> paused
27 MirrorClassHostKey MirrorFailureClass = "host_key" // known_hosts mismatch -> paused
28 MirrorClassTransient MirrorFailureClass = "transient" // timeout, network, exit without a known marker -> backoff
29 MirrorClassInternal MirrorFailureClass = "internal" // no key, no secrets key, local git error -> backoff
30)
31
32// MirrorError is what the transport returns: Class decides the transition, Detail (capped,
33// URL-redacted stderr tail) is shown to admins only. Unwrap() yields ErrUnavailable for
34// transient, ErrPreconditionFailed for the paused classes.
35type MirrorError struct {
36 Class MirrorFailureClass
37 Detail string
38}
39
40// MirrorStatus is the persisted per-repository state.
41type MirrorStatus struct {
42 Repo RepoName
43 State MirrorState
44 Class MirrorFailureClass
45 Reason string // MirrorReason(...) output, human
46 Detail string // last error detail (admin only)
47 ConsecutiveFailures int
48 LastAttemptAt time.Time
49 LastSuccessAt time.Time
50 NextRetryAt time.Time // zero unless Failing
51 PausedAt time.Time
52 UpdatedAt time.Time
53}
54type MirrorOutcome struct {
55 Err *MirrorError
56 NoOp bool
57} // Err nil == success
58type MirrorTransition struct {
59 Status MirrorStatus
60 FailedEvent bool // State moved into Failing or Paused from OK/Disabled: once per incident
61 Recovered bool // State moved from Failing/Paused to OK
62}
63
64const (
65 MirrorBackoffBase = 5 * time.Minute
66 MirrorBackoffCap = 6 * time.Hour
67 MirrorPauseAfter = 5 // consecutive transient failures
68)
69
70func (e *MirrorError) Error() string { return fmt.Sprintf("mirror %s: %s", e.Class, e.Detail) }
71func mirrorFatal(c MirrorFailureClass) bool {
72 return c == MirrorClassDiverged || c == MirrorClassAuthDenied || c == MirrorClassRemoteNotFound || c == MirrorClassHostKey
73}
74func (e *MirrorError) Unwrap() error {
75 if mirrorFatal(e.Class) {
76 return ErrPreconditionFailed
77 }
78 return ErrUnavailable
79}
80
81// ClassifyMirrorOutput maps git/ssh stderr to a class (table of substrings, tested).
82func ClassifyMirrorOutput(stderr string, exitErr error, timedOut bool) MirrorFailureClass {
83 if timedOut {
84 return MirrorClassTransient
85 }
86 s := strings.ToLower(stderr)
87 for _, line := range strings.Split(s, "\n") {
88 if i := strings.Index(line, "permission denied ("); i >= 0 {
89 methods, _, closed := strings.Cut(line[i+len("permission denied ("):], ")")
90 if closed && strings.Contains(methods, "publickey") {
91 return MirrorClassAuthDenied
92 }
93 }
94 if strings.Contains(line, "permission to ") && strings.Contains(line, " denied") {
95 return MirrorClassAuthDenied
96 }
97 }
98 for _, entry := range []struct {
99 class MirrorFailureClass
100 markers []string
101 }{
102 {MirrorClassHostKey, []string{"host key verification failed", "remote host identification has changed", "host key has changed"}},
103 {MirrorClassAuthDenied, []string{"permission denied (publickey)", "authentication failed"}},
104 {MirrorClassRemoteNotFound, []string{"repository not found", "does not appear to be a git repository"}},
105 {MirrorClassDiverged, []string{"non-fast-forward", "fetch first", "fetch-first"}},
106 } {
107 for _, m := range entry.markers {
108 if strings.Contains(s, m) {
109 return entry.class
110 }
111 }
112 }
113 if exitErr != nil {
114 return MirrorClassTransient
115 }
116 return MirrorClassNone
117}
118func MirrorBackoff(n int) time.Duration {
119 d := MirrorBackoffBase
120 for i := 1; i < n && d < MirrorBackoffCap; i++ {
121 d *= 2
122 }
123 return min(d, MirrorBackoffCap)
124}
125
126// MirrorDisable/MirrorReset are the config-change transitions: reset lifts Paused/Failing to OK
127// with counters cleared (config changed); disable -> Disabled. sync --force does not reset: the
128// attempt is judged against the previous status so a recovery still reports Recovered.
129func MirrorReset(p MirrorStatus, now time.Time) MirrorStatus {
130 p.State = MirrorOK
131 p.Class = MirrorClassNone
132 p.Reason = ""
133 p.Detail = ""
134 p.ConsecutiveFailures = 0
135 p.NextRetryAt = time.Time{}
136 p.PausedAt = time.Time{}
137 p.UpdatedAt = now
138 return p
139}
140func MirrorDisable(p MirrorStatus, now time.Time) MirrorStatus {
141 p = MirrorReset(p, now)
142 p.State = MirrorDisabled
143 return p
144}
145
146// ApplyMirrorOutcome is the single transition function (pure):
147//
148// success -> OK, counters cleared, LastSuccessAt=now (NoOp also sets LastSuccessAt).
149// fatal class (Diverged/AuthDenied/RemoteNotFound/HostKey) -> Paused at once.
150// Transient/Internal -> Failing, n++, NextRetryAt=now+MirrorBackoff(n); n>=MirrorPauseAfter -> Paused.
151// A Paused status receiving an outcome was synced with force: it is judged like any other.
152func ApplyMirrorOutcome(prev MirrorStatus, out MirrorOutcome, now time.Time) MirrorTransition {
153 p := MirrorReset(prev, now)
154 p.LastAttemptAt = now
155 if out.Err == nil {
156 p.LastSuccessAt = now
157 return MirrorTransition{Status: p, Recovered: prev.State == MirrorFailing || prev.State == MirrorPaused}
158 }
159 p.Class = out.Err.Class
160 p.Detail = out.Err.Detail
161 p.ConsecutiveFailures = prev.ConsecutiveFailures + 1
162 p.State = MirrorFailing
163 if mirrorFatal(p.Class) || p.ConsecutiveFailures >= MirrorPauseAfter {
164 p.State = MirrorPaused
165 p.PausedAt = now
166 if prev.State == MirrorPaused && !prev.PausedAt.IsZero() {
167 p.PausedAt = prev.PausedAt
168 }
169 } else {
170 p.NextRetryAt = now.Add(MirrorBackoff(p.ConsecutiveFailures))
171 }
172 p.Reason = MirrorReason(p)
173 return MirrorTransition{Status: p, FailedEvent: prev.State != MirrorFailing && prev.State != MirrorPaused}
174}
175
176// MirrorReason is the human sentence of a status, e.g.
177//
178// diverged: "the remote has commits this repository does not: push is never forced; fix the remote and run `ermite mirror sync <repo> --force`"
179// auth_denied: "the remote refused the mirror key: add the output of `ermite mirror key show` to the remote"
180// remote_not_found: "the remote repository does not exist or is not reachable with this key"
181// host_key: "the host key of the remote changed: check <data dir>/mirror_known_hosts"
182// transient: "the last %d attempts failed (%s); next retry %s"
183// paused after transient: "paused after %d consecutive failures; run `ermite mirror sync <repo> --force`"
184func MirrorReason(s MirrorStatus) string {
185 if s.State != MirrorFailing && s.State != MirrorPaused {
186 return ""
187 }
188 switch s.Class {
189 case MirrorClassDiverged:
190 return "the remote has commits this repository does not: push is never forced; fix the remote and run `ermite mirror sync <repo> --force`"
191 case MirrorClassAuthDenied:
192 return "the remote refused the mirror key: add the output of `ermite mirror key show` to the remote"
193 case MirrorClassRemoteNotFound:
194 return "the remote repository does not exist or is not reachable with this key"
195 case MirrorClassHostKey:
196 return "the host key of the remote changed: check <data dir>/mirror_known_hosts"
197 }
198 if s.State == MirrorPaused {
199 return fmt.Sprintf("paused after %d consecutive failures; run `ermite mirror sync <repo> --force`", s.ConsecutiveFailures)
200 }
201 return fmt.Sprintf("the last %d attempts failed (%s); next retry %s", s.ConsecutiveFailures, s.Class, s.NextRetryAt.UTC().Format(time.RFC3339))
202}