fix(robustness): rollback partial attachments on deploy failure
Implement attachment rollback in deployAttachments to ensure consistency when deployment fails partway through. Only commit changes after all attachments are successfully deployed.
- commit
- f80817291dadce5bb54205f3c3be8a2dc764b24c
- author
- brice · · 2026-02-06 05:56 UTC
- parent
- 99f5a82
- signature
- unsigned
1 file · +38 -6
M internal/usecase/container/service.go +38 -6
| @@ -1216,16 +1216,48 @@ func (s *Service) deployAttachments(ctx context.Context, domainName, networkName | |||
| 1216 | 1216 | } | |
| 1217 | 1217 | ||
| 1218 | 1218 | log := zerowrap.FromCtx(ctx) | |
| 1219 | - | var failed int | |
| 1219 | + | var deployed []string // track successfully deployed attachment IDs for rollback | |
| 1220 | 1220 | for _, svc := range attachments { | |
| 1221 | 1221 | if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil { | |
| 1222 | 1222 | log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName}) | |
| 1223 | - | failed++ | |
| 1224 | - | } | |
| 1225 | - | } | |
| 1226 | 1223 | ||
| 1227 | - | if failed > 0 { | |
| 1228 | - | return fmt.Errorf("failed to deploy %d of %d attachment(s)", failed, len(attachments)) | |
| 1224 | + | // Rollback: clean up already-deployed attachments | |
| 1225 | + | for _, id := range deployed { | |
| 1226 | + | if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil { | |
| 1227 | + | log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment") | |
| 1228 | + | } | |
| 1229 | + | if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil { | |
| 1230 | + | log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment") | |
| 1231 | + | } | |
| 1232 | + | } | |
| 1233 | + | // Deregister rolled-back attachments | |
| 1234 | + | if len(deployed) > 0 { | |
| 1235 | + | s.mu.Lock() | |
| 1236 | + | remaining := s.attachments[domainName] | |
| 1237 | + | filtered := remaining[:0] | |
| 1238 | + | rollbackSet := make(map[string]bool, len(deployed)) | |
| 1239 | + | for _, id := range deployed { | |
| 1240 | + | rollbackSet[id] = true | |
| 1241 | + | } | |
| 1242 | + | for _, id := range remaining { | |
| 1243 | + | if !rollbackSet[id] { | |
| 1244 | + | filtered = append(filtered, id) | |
| 1245 | + | } | |
| 1246 | + | } | |
| 1247 | + | s.attachments[domainName] = filtered | |
| 1248 | + | s.mu.Unlock() | |
| 1249 | + | } | |
| 1250 | + | ||
| 1251 | + | return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed)) | |
| 1252 | + | } | |
| 1253 | + | // Collect IDs of attachments tracked under this domain after successful deploy | |
| 1254 | + | s.mu.RLock() | |
| 1255 | + | ids := s.attachments[domainName] | |
| 1256 | + | if len(ids) > 0 { | |
| 1257 | + | latest := ids[len(ids)-1] | |
| 1258 | + | deployed = append(deployed, latest) | |
| 1259 | + | } | |
| 1260 | + | s.mu.RUnlock() | |
| 1229 | 1261 | } | |
| 1230 | 1262 | return nil | |
| 1231 | 1263 | } | |
| @@ -1216,16 +1216,48 @@ func (s *Service) deployAttachments(ctx context.Context, domainName, networkName | |||||
| 1216 | } | 1216 | } | ||
| 1217 | 1217 | ||||
| 1218 | log := zerowrap.FromCtx(ctx) | 1218 | log := zerowrap.FromCtx(ctx) | ||
| 1219 | - | var failed int | 1219 | + | var deployed []string // track successfully deployed attachment IDs for rollback |
| 1220 | for _, svc := range attachments { | 1220 | for _, svc := range attachments { | ||
| 1221 | if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil { | 1221 | if err := s.deployAttachedService(ctx, domainName, svc, networkName); err != nil { | ||
| 1222 | log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName}) | 1222 | log.WrapErrWithFields(err, "failed to deploy attachment", map[string]any{zerowrap.FieldService: svc, "domain": domainName}) | ||
| 1223 | - | failed++ | |||
| 1224 | - | } | |||
| 1225 | - | } | |||
| 1226 | 1223 | ||||
| 1227 | - | if failed > 0 { | 1224 | + | // Rollback: clean up already-deployed attachments |
| 1228 | - | return fmt.Errorf("failed to deploy %d of %d attachment(s)", failed, len(attachments)) | 1225 | + | for _, id := range deployed { |
| 1226 | + | if stopErr := s.runtime.StopContainer(ctx, id); stopErr != nil { | |||
| 1227 | + | log.Warn().Err(stopErr).Str("attachment_id", id).Msg("rollback: failed to stop attachment") | |||
| 1228 | + | } | |||
| 1229 | + | if rmErr := s.runtime.RemoveContainer(ctx, id, true); rmErr != nil { | |||
| 1230 | + | log.Warn().Err(rmErr).Str("attachment_id", id).Msg("rollback: failed to remove attachment") | |||
| 1231 | + | } | |||
| 1232 | + | } | |||
| 1233 | + | // Deregister rolled-back attachments | |||
| 1234 | + | if len(deployed) > 0 { | |||
| 1235 | + | s.mu.Lock() | |||
| 1236 | + | remaining := s.attachments[domainName] | |||
| 1237 | + | filtered := remaining[:0] | |||
| 1238 | + | rollbackSet := make(map[string]bool, len(deployed)) | |||
| 1239 | + | for _, id := range deployed { | |||
| 1240 | + | rollbackSet[id] = true | |||
| 1241 | + | } | |||
| 1242 | + | for _, id := range remaining { | |||
| 1243 | + | if !rollbackSet[id] { | |||
| 1244 | + | filtered = append(filtered, id) | |||
| 1245 | + | } | |||
| 1246 | + | } | |||
| 1247 | + | s.attachments[domainName] = filtered | |||
| 1248 | + | s.mu.Unlock() | |||
| 1249 | + | } | |||
| 1250 | + | ||||
| 1251 | + | return fmt.Errorf("failed to deploy attachment %q (rolled back %d already-deployed)", svc, len(deployed)) | |||
| 1252 | + | } | |||
| 1253 | + | // Collect IDs of attachments tracked under this domain after successful deploy | |||
| 1254 | + | s.mu.RLock() | |||
| 1255 | + | ids := s.attachments[domainName] | |||
| 1256 | + | if len(ids) > 0 { | |||
| 1257 | + | latest := ids[len(ids)-1] | |||
| 1258 | + | deployed = append(deployed, latest) | |||
| 1259 | + | } | |||
| 1260 | + | s.mu.RUnlock() | |||
| 1229 | } | 1261 | } | ||
| 1230 | return nil | 1262 | return nil | ||
| 1231 | } | 1263 | } | ||