feat(ci): store CI artifacts on the blob store, keep legacy disk artifacts readable

commit: f9e69417727bbe6f46155e569b5bc1f66f8e3a9b
author: brice · 22h ago · 2026-10-21 14:41 UTC
committer: brice · 19h ago · 2026-10-21 17:53 UTC
parent: eb72c2f
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

16 files · +763 -601

M gen/ermite/storage/ci_artifact.pb.go +15 -5

@@ -1,8 +1,8 @@
1	1		// SPDX-License-Identifier: AGPL-3.0-or-later
2	2
3	3		// Internal, unversioned storage schema of the CI artifacts (artifacts v4,
4		-	// CI-4 D3). The contents are not stored here: they are files under
5		-	// $ERMITE_DATA_DIR/ci/artifacts.
	4	+	// CI-4 D3). Contents are stored separately, in the blob store or (for
	5	+	// legacy records) under $ERMITE_DATA_DIR/ci/artifacts.
6	6
7	7		// Code generated by protoc-gen-go. DO NOT EDIT.
8	8		// versions:

@@ -110,7 +110,9 @@ type CIArtifact struct {
110	110		Charged int64 `protobuf:"varint,14,opt,name=charged,proto3" json:"charged,omitempty"`
111	111		// list_digest: the SHA-256 of the block list of the commit, to recognize a
112	112		// repeated commit.
113		-	ListDigest []byte `protobuf:"bytes,15,opt,name=list_digest,json=listDigest,proto3" json:"list_digest,omitempty"`
	113	+	ListDigest []byte `protobuf:"bytes,15,opt,name=list_digest,json=listDigest,proto3" json:"list_digest,omitempty"`
	114	+	// "" or "disk": legacy files under ci/artifacts; "blob": the blob store.
	115	+	Store string `protobuf:"bytes,16,opt,name=store,proto3" json:"store,omitempty"`
114	116		unknownFields protoimpl.UnknownFields
115	117		sizeCache protoimpl.SizeCache
116	118		}

@@ -250,6 +252,13 @@ func (x *CIArtifact) GetListDigest() []byte {
250	252		return nil
251	253		}
252	254
	255	+	func (x *CIArtifact) GetStore() string {
	256	+	if x != nil {
	257	+	return x.Store
	258	+	}
	259	+	return ""
	260	+	}
	261	+
253	262		// CIArtifactBlock is stored under "artifactblk/<artifact:020d>/<hex(block id)>":
254	263		// a staged block of a pending artifact.
255	264		type CIArtifactBlock struct {

@@ -358,7 +367,7 @@ var File_ermite_storage_ci_artifact_proto protoreflect.FileDescriptor
358	367
359	368		const file_ermite_storage_ci_artifact_proto_rawDesc = "" +
360	369		"\n" +
361		-	" ermite/storage/ci_artifact.proto\x12\x0eermite.storage\x1a\x1fgoogle/protobuf/timestamp.proto\"\xdf\x03\n" +
	370	+	" ermite/storage/ci_artifact.proto\x12\x0eermite.storage\x1a\x1fgoogle/protobuf/timestamp.proto\"\xf5\x03\n" +
362	371		"\n" +
363	372		"CIArtifact\x12\x0e\n" +
364	373		"\x02id\x18\x01 \x01(\x03R\x02id\x12\x12\n" +

@@ -382,7 +391,8 @@ const file_ermite_storage_ci_artifact_proto_rawDesc = "" +
382	391		"expires_at\x18\r \x01(\v2\x1a.google.protobuf.TimestampR\texpiresAt\x12\x18\n" +
383	392		"\acharged\x18\x0e \x01(\x03R\acharged\x12\x1f\n" +
384	393		"\vlist_digest\x18\x0f \x01(\fR\n" +
385		-	"listDigest\"9\n" +
	394	+	"listDigest\x12\x14\n" +
	395	+	"\x05store\x18\x10 \x01(\tR\x05store\"9\n" +
386	396		"\x0fCIArtifactBlock\x12\x12\n" +
387	397		"\x04size\x18\x01 \x01(\x03R\x04size\x12\x12\n" +
388	398		"\x04file\x18\x02 \x01(\tR\x04file\"%\n" +

 // SPDX-License-Identifier: AGPL-3.0-or-later
 // Internal, unversioned storage schema of the CI artifacts (artifacts v4,
-// CI-4 D3). The contents are not stored here: they are files under
+// CI-4 D3). Contents are stored separately, in the blob store or (for
-// $ERMITE_DATA_DIR/ci/artifacts.
+// legacy records) under $ERMITE_DATA_DIR/ci/artifacts.
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 	Charged int64 `protobuf:"varint,14,opt,name=charged,proto3" json:"charged,omitempty"`
 	// list_digest: the SHA-256 of the block list of the commit, to recognize a
 	// repeated commit.
-	ListDigest    []byte `protobuf:"bytes,15,opt,name=list_digest,json=listDigest,proto3" json:"list_digest,omitempty"`
+	ListDigest []byte `protobuf:"bytes,15,opt,name=list_digest,json=listDigest,proto3" json:"list_digest,omitempty"`
+	// "" or "disk": legacy files under ci/artifacts; "blob": the blob store.
+	Store         string `protobuf:"bytes,16,opt,name=store,proto3" json:"store,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
 	return nil
 }
+func (x *CIArtifact) GetStore() string {
+	if x != nil {
+		return x.Store
+	}
+	return ""
+}
 // CIArtifactBlock is stored under "artifactblk/<artifact:020d>/<hex(block id)>":
 // a staged block of a pending artifact.
 type CIArtifactBlock struct {
 const file_ermite_storage_ci_artifact_proto_rawDesc = "" +
 	"\n" +
-	" ermite/storage/ci_artifact.proto\x12\x0eermite.storage\x1a\x1fgoogle/protobuf/timestamp.proto\"\xdf\x03\n" +
+	" ermite/storage/ci_artifact.proto\x12\x0eermite.storage\x1a\x1fgoogle/protobuf/timestamp.proto\"\xf5\x03\n" +
 	"\n" +
 	"CIArtifact\x12\x0e\n" +
 	"\x02id\x18\x01 \x01(\x03R\x02id\x12\x12\n" +
 	"expires_at\x18\r \x01(\v2\x1a.google.protobuf.TimestampR\texpiresAt\x12\x18\n" +
 	"\acharged\x18\x0e \x01(\x03R\acharged\x12\x1f\n" +
 	"\vlist_digest\x18\x0f \x01(\fR\n" +
-	"listDigest\"9\n" +
+	"listDigest\x12\x14\n" +
+	"\x05store\x18\x10 \x01(\tR\x05store\"9\n" +
 	"\x0fCIArtifactBlock\x12\x12\n" +
 	"\x04size\x18\x01 \x01(\x03R\x04size\x12\x12\n" +
 	"\x04file\x18\x02 \x01(\tR\x04file\"%\n" +

M internal/adapters/ciartifacts/ciartifacts.go +5 -171

@@ -1,23 +1,12 @@
1	1		// SPDX-License-Identifier: AGPL-3.0-or-later
2	2
3		-	// Package ciartifacts implements ports.CIArtifactFiles under
4		-	// $ERMITE_DATA_DIR/ci/artifacts, keyed by the global run id like the logs (a
5		-	// repository rename never moves them):
6		-	//
7		-	// <run:020d>/<id:020d> the content of a committed artifact
8		-	// <run:020d>/<id:020d>.blocks/<f> the staged blocks of a pending one
9		-	// <run:020d>/.tmp-<id:020d>-* a content being assembled
10		-	//
11		-	// Paths are made of ids and of block file names the use case builds from
12		-	// hexadecimal characters: an artifact name never reaches the file system.
13		-	// Files are 0600 in 0700 directories, synced before they count, and a content
14		-	// appears by rename, whole or not at all.
	3	+	// Package ciartifacts reads and removes legacy artifacts recorded before the
	4	+	// blob store (ADR 0034), under $ERMITE_DATA_DIR/ci/artifacts. Paths use ids
	5	+	// only. The adapter is removed once all legacy artifacts have expired.
15	6		package ciartifacts
16	7
17	8		import (
18	9		"context"
19		-	"crypto/sha256"
20		-	"encoding/hex"
21	10		"errors"
22	11		"fmt"
23	12		"io"

@@ -39,9 +28,6 @@ const (
39	28		idFormat = "%020d"
40	29		blocksSuffix = ".blocks"
41	30		tmpPrefix = ".tmp-"
42		-	// maxFileName bounds a block file name (well under the 255 bytes of a
43		-	// name on Linux).
44		-	maxFileName = 200
45	31		)
46	32
47	33		// Store is the file-backed artifact store.

@@ -54,15 +40,6 @@ func New(dir string) *Store { return &Store{root: dir} }
54	40
55	41		func validRef(ref ports.CIArtifactRef) bool { return ref.RunID > 0 && ref.ID > 0 }
56	42
57		-	// validFile accepts the block file names of the use case: lowercase
58		-	// hexadecimal characters and dots, not starting with a dot.
59		-	func validFile(name string) bool {
60		-	if name == "" \|\| len(name) > maxFileName \|\| name[0] == '.' {
61		-	return false
62		-	}
63		-	return strings.Trim(name, "0123456789abcdef.") == ""
64		-	}
65		-
66	43		func (s *Store) runDir(runID int64) string {
67	44		return filepath.Join(s.root, fmt.Sprintf(idFormat, runID))
68	45		}

@@ -75,138 +52,6 @@ func (s *Store) blocksDir(ref ports.CIArtifactRef) string {
75	52		return s.contentPath(ref) + blocksSuffix
76	53		}
77	54
78		-	// syncDir makes the entries of a directory durable.
79		-	func syncDir(dir string) error {
80		-	d, err := os.Open(dir)
81		-	if err != nil {
82		-	return err
83		-	}
84		-	serr := d.Sync()
85		-	return errors.Join(serr, d.Close())
86		-	}
87		-
88		-	// ctxReader stops a copy when its context ends.
89		-	type ctxReader struct {
90		-	ctx context.Context
91		-	r io.Reader
92		-	}
93		-
94		-	func (c ctxReader) Read(p []byte) (int, error) {
95		-	if err := c.ctx.Err(); err != nil {
96		-	return 0, err
97		-	}
98		-	return c.r.Read(p)
99		-	}
100		-
101		-	// PutBlock writes a new block file, created exclusively.
102		-	func (s *Store) PutBlock(ctx context.Context, ref ports.CIArtifactRef, file string, src io.Reader) (int64, error) {
103		-	if !validRef(ref) \|\| !validFile(file) {
104		-	return 0, fmt.Errorf("%w: block file %q", domain.ErrInvalid, file)
105		-	}
106		-	dir := s.blocksDir(ref)
107		-	if err := os.MkdirAll(dir, 0o700); err != nil {
108		-	return 0, fmt.Errorf("create block directory: %w", err)
109		-	}
110		-	path := filepath.Join(dir, file)
111		-	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600)
112		-	if err != nil {
113		-	return 0, fmt.Errorf("create block: %w", err)
114		-	}
115		-	n, werr := io.Copy(f, ctxReader{ctx, src})
116		-	var serr error
117		-	if werr == nil {
118		-	serr = f.Sync()
119		-	}
120		-	cerr := f.Close()
121		-	if err := errors.Join(werr, serr, cerr); err != nil {
122		-	_ = os.Remove(path)
123		-	if werr != nil {
124		-	return 0, werr // the source's error, as it is
125		-	}
126		-	return 0, fmt.Errorf("write block: %w", err)
127		-	}
128		-	if err := syncDir(dir); err != nil {
129		-	_ = os.Remove(path)
130		-	return 0, fmt.Errorf("sync block directory: %w", err)
131		-	}
132		-	return n, nil
133		-	}
134		-
135		-	// RemoveBlock deletes one block file.
136		-	func (s *Store) RemoveBlock(_ context.Context, ref ports.CIArtifactRef, file string) error {
137		-	if !validRef(ref) \|\| !validFile(file) {
138		-	return nil
139		-	}
140		-	if err := os.Remove(filepath.Join(s.blocksDir(ref), file)); err != nil && !errors.Is(err, fs.ErrNotExist) {
141		-	return fmt.Errorf("remove block: %w", err)
142		-	}
143		-	return nil
144		-	}
145		-
146		-	// Assemble writes the blocks into a temporary file, syncs it and renames it
147		-	// over the content.
148		-	func (s *Store) Assemble(ctx context.Context, ref ports.CIArtifactRef, files []string) (int64, string, error) {
149		-	if !validRef(ref) {
150		-	return 0, "", domain.ErrNotFound
151		-	}
152		-	for _, f := range files {
153		-	if !validFile(f) {
154		-	return 0, "", fmt.Errorf("%w: block file %q", domain.ErrInvalid, f)
155		-	}
156		-	}
157		-	dir := s.runDir(ref.RunID)
158		-	if err := os.MkdirAll(dir, 0o700); err != nil {
159		-	return 0, "", fmt.Errorf("create artifact directory: %w", err)
160		-	}
161		-	out, err := os.CreateTemp(dir, tmpPrefix+fmt.Sprintf(idFormat, ref.ID)+"-*")
162		-	if err != nil {
163		-	return 0, "", fmt.Errorf("create artifact: %w", err)
164		-	}
165		-	tmp := out.Name()
166		-	h := sha256.New()
167		-	size, err := s.concat(ctx, ref, files, io.MultiWriter(out, h))
168		-	if err == nil {
169		-	err = out.Sync()
170		-	}
171		-	if cerr := out.Close(); err == nil {
172		-	err = cerr
173		-	}
174		-	if err == nil {
175		-	err = os.Rename(tmp, s.contentPath(ref))
176		-	}
177		-	if err != nil {
178		-	_ = os.Remove(tmp)
179		-	if errors.Is(err, domain.ErrNotFound) \|\| ctx.Err() != nil {
180		-	return 0, "", err
181		-	}
182		-	return 0, "", fmt.Errorf("assemble artifact: %w", err)
183		-	}
184		-	if err := syncDir(dir); err != nil {
185		-	return 0, "", fmt.Errorf("sync artifact directory: %w", err)
186		-	}
187		-	return size, "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
188		-	}
189		-
190		-	func (s *Store) concat(ctx context.Context, ref ports.CIArtifactRef, files []string, w io.Writer) (int64, error) {
191		-	var size int64
192		-	for _, name := range files {
193		-	f, err := os.Open(filepath.Join(s.blocksDir(ref), name))
194		-	if errors.Is(err, fs.ErrNotExist) {
195		-	return 0, fmt.Errorf("%w: block file %s", domain.ErrNotFound, name)
196		-	}
197		-	if err != nil {
198		-	return 0, err
199		-	}
200		-	n, err := io.Copy(w, ctxReader{ctx, f})
201		-	_ = f.Close()
202		-	if err != nil {
203		-	return 0, err
204		-	}
205		-	size += n
206		-	}
207		-	return size, nil
208		-	}
209		-
210	55		// Open opens the content at offset.
211	56		func (s *Store) Open(_ context.Context, ref ports.CIArtifactRef, offset int64) (io.ReadCloser, error) {
212	57		if !validRef(ref) \|\| offset < 0 {

@@ -227,7 +72,7 @@ func (s *Store) Open(_ context.Context, ref ports.CIArtifactRef, offset int64) (
227	72		}
228	73
229	74		// Remove deletes the content, the blocks and any assembly in progress.
230		-	func (s *Store) Remove(ctx context.Context, ref ports.CIArtifactRef) error {
	75	+	func (s *Store) Remove(_ context.Context, ref ports.CIArtifactRef) error {
231	76		if !validRef(ref) {
232	77		return nil
233	78		}

@@ -235,7 +80,7 @@ func (s *Store) Remove(ctx context.Context, ref ports.CIArtifactRef) error {
235	80		if err := os.Remove(s.contentPath(ref)); err != nil && !errors.Is(err, fs.ErrNotExist) {
236	81		errs = append(errs, err)
237	82		}
238		-	errs = append(errs, s.RemoveBlocks(ctx, ref))
	83	+	errs = append(errs, os.RemoveAll(s.blocksDir(ref)))
239	84		tmps, _ := filepath.Glob(filepath.Join(s.runDir(ref.RunID), tmpPrefix+fmt.Sprintf(idFormat, ref.ID)+"-*"))
240	85		for _, t := range tmps {
241	86		if err := os.Remove(t); err != nil && !errors.Is(err, fs.ErrNotExist) {

@@ -248,17 +93,6 @@ func (s *Store) Remove(ctx context.Context, ref ports.CIArtifactRef) error {
248	93		return nil
249	94		}
250	95
251		-	// RemoveBlocks deletes the staging directory of the artifact.
252		-	func (s *Store) RemoveBlocks(_ context.Context, ref ports.CIArtifactRef) error {
253		-	if !validRef(ref) {
254		-	return nil
255		-	}
256		-	if err := os.RemoveAll(s.blocksDir(ref)); err != nil {
257		-	return fmt.Errorf("remove blocks: %w", err)
258		-	}
259		-	return nil
260		-	}
261		-
262	96		// RemoveRuns deletes the directories of the runs.
263	97		func (s *Store) RemoveRuns(_ context.Context, runIDs []int64) error {
264	98		var errs []error

@@ -1,23 +1,12 @@
1		// SPDX-License-Identifier: AGPL-3.0-or-later	1		// SPDX-License-Identifier: AGPL-3.0-or-later
2			2
3	-	// Package ciartifacts implements ports.CIArtifactFiles under	3	+	// Package ciartifacts reads and removes legacy artifacts recorded before the
4	-	// $ERMITE_DATA_DIR/ci/artifacts, keyed by the global run id like the logs (a	4	+	// blob store (ADR 0034), under $ERMITE_DATA_DIR/ci/artifacts. Paths use ids
5	-	// repository rename never moves them):	5	+	// only. The adapter is removed once all legacy artifacts have expired.
6	-	//
7	-	// <run:020d>/<id:020d> the content of a committed artifact
8	-	// <run:020d>/<id:020d>.blocks/<f> the staged blocks of a pending one
9	-	// <run:020d>/.tmp-<id:020d>-* a content being assembled
10	-	//
11	-	// Paths are made of ids and of block file names the use case builds from
12	-	// hexadecimal characters: an artifact name never reaches the file system.
13	-	// Files are 0600 in 0700 directories, synced before they count, and a content
14	-	// appears by rename, whole or not at all.
15		package ciartifacts	6		package ciartifacts
16			7
17		import (	8		import (
18		"context"	9		"context"
19	-	"crypto/sha256"
20	-	"encoding/hex"
21		"errors"	10		"errors"
22		"fmt"	11		"fmt"
23		"io"	12		"io"

@@ -39,9 +28,6 @@ const (
39		idFormat = "%020d"	28		idFormat = "%020d"
40		blocksSuffix = ".blocks"	29		blocksSuffix = ".blocks"
41		tmpPrefix = ".tmp-"	30		tmpPrefix = ".tmp-"
42	-	// maxFileName bounds a block file name (well under the 255 bytes of a
43	-	// name on Linux).
44	-	maxFileName = 200
45		)	31		)
46			32
47		// Store is the file-backed artifact store.	33		// Store is the file-backed artifact store.

@@ -54,15 +40,6 @@ func New(dir string) *Store { return &Store{root: dir} }
54			40
55		func validRef(ref ports.CIArtifactRef) bool { return ref.RunID > 0 && ref.ID > 0 }	41		func validRef(ref ports.CIArtifactRef) bool { return ref.RunID > 0 && ref.ID > 0 }
56			42
57	-	// validFile accepts the block file names of the use case: lowercase
58	-	// hexadecimal characters and dots, not starting with a dot.
59	-	func validFile(name string) bool {
60	-	if name == "" \|\| len(name) > maxFileName \|\| name[0] == '.' {
61	-	return false
62	-	}
63	-	return strings.Trim(name, "0123456789abcdef.") == ""
64	-	}
65	-
66		func (s *Store) runDir(runID int64) string {	43		func (s *Store) runDir(runID int64) string {
67		return filepath.Join(s.root, fmt.Sprintf(idFormat, runID))	44		return filepath.Join(s.root, fmt.Sprintf(idFormat, runID))
68		}	45		}

@@ -75,138 +52,6 @@ func (s *Store) blocksDir(ref ports.CIArtifactRef) string {
75		return s.contentPath(ref) + blocksSuffix	52		return s.contentPath(ref) + blocksSuffix
76		}	53		}
77			54
78	-	// syncDir makes the entries of a directory durable.
79	-	func syncDir(dir string) error {
80	-	d, err := os.Open(dir)
81	-	if err != nil {
82	-	return err
83	-	}
84	-	serr := d.Sync()
85	-	return errors.Join(serr, d.Close())
86	-	}
87	-
88	-	// ctxReader stops a copy when its context ends.
89	-	type ctxReader struct {
90	-	ctx context.Context
91	-	r io.Reader
92	-	}
93	-
94	-	func (c ctxReader) Read(p []byte) (int, error) {
95	-	if err := c.ctx.Err(); err != nil {
96	-	return 0, err
97	-	}
98	-	return c.r.Read(p)
99	-	}
100	-
101	-	// PutBlock writes a new block file, created exclusively.
102	-	func (s *Store) PutBlock(ctx context.Context, ref ports.CIArtifactRef, file string, src io.Reader) (int64, error) {
103	-	if !validRef(ref) \|\| !validFile(file) {
104	-	return 0, fmt.Errorf("%w: block file %q", domain.ErrInvalid, file)
105	-	}
106	-	dir := s.blocksDir(ref)
107	-	if err := os.MkdirAll(dir, 0o700); err != nil {
108	-	return 0, fmt.Errorf("create block directory: %w", err)
109	-	}
110	-	path := filepath.Join(dir, file)
111	-	f, err := os.OpenFile(path, os.O_WRONLY\|os.O_CREATE\|os.O_EXCL, 0o600)
112	-	if err != nil {
113	-	return 0, fmt.Errorf("create block: %w", err)
114	-	}
115	-	n, werr := io.Copy(f, ctxReader{ctx, src})
116	-	var serr error
117	-	if werr == nil {
118	-	serr = f.Sync()
119	-	}
120	-	cerr := f.Close()
121	-	if err := errors.Join(werr, serr, cerr); err != nil {
122	-	_ = os.Remove(path)
123	-	if werr != nil {
124	-	return 0, werr // the source's error, as it is
125	-	}
126	-	return 0, fmt.Errorf("write block: %w", err)
127	-	}
128	-	if err := syncDir(dir); err != nil {
129	-	_ = os.Remove(path)
130	-	return 0, fmt.Errorf("sync block directory: %w", err)
131	-	}
132	-	return n, nil
133	-	}
134	-
135	-	// RemoveBlock deletes one block file.
136	-	func (s *Store) RemoveBlock(_ context.Context, ref ports.CIArtifactRef, file string) error {
137	-	if !validRef(ref) \|\| !validFile(file) {
138	-	return nil
139	-	}
140	-	if err := os.Remove(filepath.Join(s.blocksDir(ref), file)); err != nil && !errors.Is(err, fs.ErrNotExist) {
141	-	return fmt.Errorf("remove block: %w", err)
142	-	}
143	-	return nil
144	-	}
145	-
146	-	// Assemble writes the blocks into a temporary file, syncs it and renames it
147	-	// over the content.
148	-	func (s *Store) Assemble(ctx context.Context, ref ports.CIArtifactRef, files []string) (int64, string, error) {
149	-	if !validRef(ref) {
150	-	return 0, "", domain.ErrNotFound
151	-	}
152	-	for _, f := range files {
153	-	if !validFile(f) {
154	-	return 0, "", fmt.Errorf("%w: block file %q", domain.ErrInvalid, f)
155	-	}
156	-	}
157	-	dir := s.runDir(ref.RunID)
158	-	if err := os.MkdirAll(dir, 0o700); err != nil {
159	-	return 0, "", fmt.Errorf("create artifact directory: %w", err)
160	-	}
161	-	out, err := os.CreateTemp(dir, tmpPrefix+fmt.Sprintf(idFormat, ref.ID)+"-*")
162	-	if err != nil {
163	-	return 0, "", fmt.Errorf("create artifact: %w", err)
164	-	}
165	-	tmp := out.Name()
166	-	h := sha256.New()
167	-	size, err := s.concat(ctx, ref, files, io.MultiWriter(out, h))
168	-	if err == nil {
169	-	err = out.Sync()
170	-	}
171	-	if cerr := out.Close(); err == nil {
172	-	err = cerr
173	-	}
174	-	if err == nil {
175	-	err = os.Rename(tmp, s.contentPath(ref))
176	-	}
177	-	if err != nil {
178	-	_ = os.Remove(tmp)
179	-	if errors.Is(err, domain.ErrNotFound) \|\| ctx.Err() != nil {
180	-	return 0, "", err
181	-	}
182	-	return 0, "", fmt.Errorf("assemble artifact: %w", err)
183	-	}
184	-	if err := syncDir(dir); err != nil {
185	-	return 0, "", fmt.Errorf("sync artifact directory: %w", err)
186	-	}
187	-	return size, "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
188	-	}
189	-
190	-	func (s *Store) concat(ctx context.Context, ref ports.CIArtifactRef, files []string, w io.Writer) (int64, error) {
191	-	var size int64
192	-	for _, name := range files {
193	-	f, err := os.Open(filepath.Join(s.blocksDir(ref), name))
194	-	if errors.Is(err, fs.ErrNotExist) {
195	-	return 0, fmt.Errorf("%w: block file %s", domain.ErrNotFound, name)
196	-	}
197	-	if err != nil {
198	-	return 0, err
199	-	}
200	-	n, err := io.Copy(w, ctxReader{ctx, f})
201	-	_ = f.Close()
202	-	if err != nil {
203	-	return 0, err
204	-	}
205	-	size += n
206	-	}
207	-	return size, nil
208	-	}
209	-
210		// Open opens the content at offset.	55		// Open opens the content at offset.
211		func (s *Store) Open(_ context.Context, ref ports.CIArtifactRef, offset int64) (io.ReadCloser, error) {	56		func (s *Store) Open(_ context.Context, ref ports.CIArtifactRef, offset int64) (io.ReadCloser, error) {
212		if !validRef(ref) \|\| offset < 0 {	57		if !validRef(ref) \|\| offset < 0 {

@@ -227,7 +72,7 @@ func (s *Store) Open(_ context.Context, ref ports.CIArtifactRef, offset int64) (
227		}	72		}
228			73
229		// Remove deletes the content, the blocks and any assembly in progress.	74		// Remove deletes the content, the blocks and any assembly in progress.
230	-	func (s *Store) Remove(ctx context.Context, ref ports.CIArtifactRef) error {	75	+	func (s *Store) Remove(_ context.Context, ref ports.CIArtifactRef) error {
231		if !validRef(ref) {	76		if !validRef(ref) {
232		return nil	77		return nil
233		}	78		}

@@ -235,7 +80,7 @@ func (s *Store) Remove(ctx context.Context, ref ports.CIArtifactRef) error {
235		if err := os.Remove(s.contentPath(ref)); err != nil && !errors.Is(err, fs.ErrNotExist) {	80		if err := os.Remove(s.contentPath(ref)); err != nil && !errors.Is(err, fs.ErrNotExist) {
236		errs = append(errs, err)	81		errs = append(errs, err)
237		}	82		}
238	-	errs = append(errs, s.RemoveBlocks(ctx, ref))	83	+	errs = append(errs, os.RemoveAll(s.blocksDir(ref)))
239		tmps, _ := filepath.Glob(filepath.Join(s.runDir(ref.RunID), tmpPrefix+fmt.Sprintf(idFormat, ref.ID)+"-*"))	84		tmps, _ := filepath.Glob(filepath.Join(s.runDir(ref.RunID), tmpPrefix+fmt.Sprintf(idFormat, ref.ID)+"-*"))
240		for _, t := range tmps {	85		for _, t := range tmps {
241		if err := os.Remove(t); err != nil && !errors.Is(err, fs.ErrNotExist) {	86		if err := os.Remove(t); err != nil && !errors.Is(err, fs.ErrNotExist) {

@@ -248,17 +93,6 @@ func (s *Store) Remove(ctx context.Context, ref ports.CIArtifactRef) error {
248		return nil	93		return nil
249		}	94		}
250			95
251	-	// RemoveBlocks deletes the staging directory of the artifact.
252	-	func (s *Store) RemoveBlocks(_ context.Context, ref ports.CIArtifactRef) error {
253	-	if !validRef(ref) {
254	-	return nil
255	-	}
256	-	if err := os.RemoveAll(s.blocksDir(ref)); err != nil {
257	-	return fmt.Errorf("remove blocks: %w", err)
258	-	}
259	-	return nil
260	-	}
261	-
262		// RemoveRuns deletes the directories of the runs.	96		// RemoveRuns deletes the directories of the runs.
263		func (s *Store) RemoveRuns(_ context.Context, runIDs []int64) error {	97		func (s *Store) RemoveRuns(_ context.Context, runIDs []int64) error {
264		var errs []error	98		var errs []error

M internal/adapters/ciartifacts/ciartifacts_test.go +8 -105


@@ -3,13 +3,10 @@
3	3		package ciartifacts_test
4	4
5	5		import (
6		-	"crypto/sha256"
7		-	"encoding/hex"
8	6		"errors"
9	7		"io"
10	8		"os"
11	9		"path/filepath"
12		-	"strings"
13	10		"testing"
14	11
15	12		"github.com/stretchr/testify/assert"

@@ -22,11 +19,6 @@ import (
22	19
23	20		var ref = ports.CIArtifactRef{RunID: 7, ID: 42}
24	21
25		-	func digestOf(s string) string {
26		-	sum := sha256.Sum256([]byte(s))
27		-	return "sha256:" + hex.EncodeToString(sum[:])
28		-	}
29		-
30	22		func entries(t *testing.T, dir string) []string {
31	23		t.Helper()
32	24		es, err := os.ReadDir(dir)

@@ -41,121 +33,32 @@ func entries(t *testing.T, dir string) []string {
41	33		return out
42	34		}
43	35
44		-	func TestBlocksAssembleInTheOrderOfTheList(t *testing.T) {
	36	+	func TestLegacyReadAndRemoval(t *testing.T) {
45	37		ctx := t.Context()
46	38		root := t.TempDir()
47	39		s := ciartifacts.New(root)
48		-
49		-	// Uploaded out of order; the list decides.
50		-	for file, data := range map[string]string{"02.aa": "world", "01.bb": "hello ", "03.cc": "!"} {
51		-	n, err := s.PutBlock(ctx, ref, file, strings.NewReader(data))
52		-	require.NoError(t, err)
53		-	assert.EqualValues(t, len(data), n)
54		-	}
55		-	size, digest, err := s.Assemble(ctx, ref, []string{"01.bb", "02.aa", "03.cc", "03.cc"})
56		-	require.NoError(t, err)
57		-	assert.EqualValues(t, len("hello world!!"), size)
58		-	assert.Equal(t, digestOf("hello world!!"), digest, "a repeated id is repeated content")
59		-
60		-	// The documented layout, named by ids only.
61		-	b, err := os.ReadFile(filepath.Join(root, "00000000000000000007", "00000000000000000042"))
62		-	require.NoError(t, err)
63		-	assert.Equal(t, "hello world!!", string(b))
64		-	for _, e := range entries(t, filepath.Join(root, "00000000000000000007")) {
65		-	assert.False(t, strings.HasPrefix(e, ".tmp-"), "no temporary file left: %s", e)
66		-	}
67		-
68		-	// Offset reads.
	40	+	run := filepath.Join(root, "00000000000000000007")
	41	+	require.NoError(t, os.MkdirAll(run, 0o700))
	42	+	require.NoError(t, os.WriteFile(filepath.Join(run, "00000000000000000042"), []byte("hello world"), 0o600))
69	43		r, err := s.Open(ctx, ref, 6)
70	44		require.NoError(t, err)
71		-	rest, err := io.ReadAll(r)
	45	+	data, err := io.ReadAll(r)
72	46		require.NoError(t, err)
73	47		require.NoError(t, r.Close())
74		-	assert.Equal(t, "world!!", string(rest))
75		-
76		-	// A new assembly replaces the content whole.
77		-	size, digest, err = s.Assemble(ctx, ref, nil)
78		-	require.NoError(t, err)
79		-	assert.Zero(t, size, "an empty list is an empty content")
80		-	assert.Equal(t, digestOf(""), digest)
81		-
	48	+	assert.Equal(t, "world", string(data))
82	49		ids, err := s.ArtifactIDs(ctx, 7)
83	50		require.NoError(t, err)
84	51		assert.Equal(t, []int64{42}, ids)
85		-	runs, err := s.RunIDs(ctx)
86		-	require.NoError(t, err)
87		-	assert.Equal(t, []int64{7}, runs)
88		-
89		-	require.NoError(t, s.RemoveBlocks(ctx, ref))
90		-	assert.Equal(t, []string{"00000000000000000042"}, entries(t, filepath.Join(root, "00000000000000000007")))
91	52		require.NoError(t, s.Remove(ctx, ref))
92		-	assert.Empty(t, entries(t, filepath.Join(root, "00000000000000000007")))
	53	+	require.NoError(t, s.Remove(ctx, ref))
93	54		_, err = s.Open(ctx, ref, 0)
94	55		assert.ErrorIs(t, err, domain.ErrNotFound)
95	56		require.NoError(t, s.RemoveRuns(ctx, []int64{7}))
96	57		assert.Empty(t, entries(t, root))
97		-	}
98		-
99		-	func TestAMissingBlockIsNotFoundAndLeavesNoContent(t *testing.T) {
100		-	ctx := t.Context()
101		-	root := t.TempDir()
102		-	s := ciartifacts.New(root)
103		-	_, err := s.PutBlock(ctx, ref, "01.aa", strings.NewReader("x"))
104		-	require.NoError(t, err)
105		-	_, _, err = s.Assemble(ctx, ref, []string{"01.aa", "02.aa"})
106		-	assert.ErrorIs(t, err, domain.ErrNotFound)
107		-	_, err = s.Open(ctx, ref, 0)
108		-	assert.ErrorIs(t, err, domain.ErrNotFound, "no content appears")
109		-	for _, e := range entries(t, filepath.Join(root, "00000000000000000007")) {
110		-	assert.False(t, strings.HasPrefix(e, ".tmp-"), e)
111		-	}
112		-	}
113		-
114		-	type failingReader struct{ n int }
115		-
116		-	func (f *failingReader) Read(p []byte) (int, error) {
117		-	if f.n == 0 {
118		-	return 0, errors.New("stream broken")
119		-	}
120		-	f.n--
121		-	p[0] = 'x'
122		-	return 1, nil
123		-	}
124		-
125		-	func TestABrokenSourceLeavesNoBlock(t *testing.T) {
126		-	ctx := t.Context()
127		-	root := t.TempDir()
128		-	s := ciartifacts.New(root)
129		-	_, err := s.PutBlock(ctx, ref, "01.aa", &failingReader{n: 3})
130		-	require.EqualError(t, err, "stream broken", "the source's error as it is")
131		-	assert.Empty(t, entries(t, filepath.Join(root, "00000000000000000007", "00000000000000000042.blocks")))
132		-
133		-	// A file name is never reused: the second put of a name fails.
134		-	_, err = s.PutBlock(ctx, ref, "01.aa", strings.NewReader("a"))
135		-	require.NoError(t, err)
136		-	_, err = s.PutBlock(ctx, ref, "01.aa", strings.NewReader("b"))
137		-	assert.Error(t, err)
138		-	}
139		-
140		-	func TestNamesOutsideTheRulesNeverReachTheFileSystem(t *testing.T) {
141		-	ctx := t.Context()
142		-	root := t.TempDir()
143		-	s := ciartifacts.New(filepath.Join(root, "store"))
144		-	for _, name := range []string{"", "../x", "a/b", ".hidden", "..", "ZZ", "01.aa/../../x", strings.Repeat("a", 201)} {
145		-	_, err := s.PutBlock(ctx, ref, name, strings.NewReader("x"))
146		-	assert.ErrorIs(t, err, domain.ErrInvalid, "%q", name)
147		-	_, _, err = s.Assemble(ctx, ref, []string{name})
148		-	assert.Error(t, err, "%q", name)
149		-	}
150	58		for _, bad := range []ports.CIArtifactRef{{RunID: 0, ID: 1}, {RunID: 1, ID: -1}} {
151		-	_, err := s.PutBlock(ctx, bad, "01.aa", strings.NewReader("x"))
152		-	assert.ErrorIs(t, err, domain.ErrInvalid)
153		-	_, err = s.Open(ctx, bad, 0)
	59	+	_, err := s.Open(ctx, bad, 0)
154	60		assert.ErrorIs(t, err, domain.ErrNotFound)
155	61		}
156		-	for _, e := range entries(t, root) {
157		-	assert.Equal(t, "store", e, "nothing written beside the store")
158		-	}
159	62		}
160	63
161	64		func TestOrphanListingSeesEveryKindOfFile(t *testing.T) {


@@ -3,13 +3,10 @@
3		package ciartifacts_test	3		package ciartifacts_test
4			4
5		import (	5		import (
6	-	"crypto/sha256"
7	-	"encoding/hex"
8		"errors"	6		"errors"
9		"io"	7		"io"
10		"os"	8		"os"
11		"path/filepath"	9		"path/filepath"
12	-	"strings"
13		"testing"	10		"testing"
14			11
15		"github.com/stretchr/testify/assert"	12		"github.com/stretchr/testify/assert"

@@ -22,11 +19,6 @@ import (
22			19
23		var ref = ports.CIArtifactRef{RunID: 7, ID: 42}	20		var ref = ports.CIArtifactRef{RunID: 7, ID: 42}
24			21
25	-	func digestOf(s string) string {
26	-	sum := sha256.Sum256([]byte(s))
27	-	return "sha256:" + hex.EncodeToString(sum[:])
28	-	}
29	-
30		func entries(t *testing.T, dir string) []string {	22		func entries(t *testing.T, dir string) []string {
31		t.Helper()	23		t.Helper()
32		es, err := os.ReadDir(dir)	24		es, err := os.ReadDir(dir)

@@ -41,121 +33,32 @@ func entries(t *testing.T, dir string) []string {
41		return out	33		return out
42		}	34		}
43			35
44	-	func TestBlocksAssembleInTheOrderOfTheList(t *testing.T) {	36	+	func TestLegacyReadAndRemoval(t *testing.T) {
45		ctx := t.Context()	37		ctx := t.Context()
46		root := t.TempDir()	38		root := t.TempDir()
47		s := ciartifacts.New(root)	39		s := ciartifacts.New(root)
48	-		40	+	run := filepath.Join(root, "00000000000000000007")
49	-	// Uploaded out of order; the list decides.	41	+	require.NoError(t, os.MkdirAll(run, 0o700))
50	-	for file, data := range map[string]string{"02.aa": "world", "01.bb": "hello ", "03.cc": "!"} {	42	+	require.NoError(t, os.WriteFile(filepath.Join(run, "00000000000000000042"), []byte("hello world"), 0o600))
51	-	n, err := s.PutBlock(ctx, ref, file, strings.NewReader(data))
52	-	require.NoError(t, err)
53	-	assert.EqualValues(t, len(data), n)
54	-	}
55	-	size, digest, err := s.Assemble(ctx, ref, []string{"01.bb", "02.aa", "03.cc", "03.cc"})
56	-	require.NoError(t, err)
57	-	assert.EqualValues(t, len("hello world!!"), size)
58	-	assert.Equal(t, digestOf("hello world!!"), digest, "a repeated id is repeated content")
59	-
60	-	// The documented layout, named by ids only.
61	-	b, err := os.ReadFile(filepath.Join(root, "00000000000000000007", "00000000000000000042"))
62	-	require.NoError(t, err)
63	-	assert.Equal(t, "hello world!!", string(b))
64	-	for _, e := range entries(t, filepath.Join(root, "00000000000000000007")) {
65	-	assert.False(t, strings.HasPrefix(e, ".tmp-"), "no temporary file left: %s", e)
66	-	}
67	-
68	-	// Offset reads.
69		r, err := s.Open(ctx, ref, 6)	43		r, err := s.Open(ctx, ref, 6)
70		require.NoError(t, err)	44		require.NoError(t, err)
71	-	rest, err := io.ReadAll(r)	45	+	data, err := io.ReadAll(r)
72		require.NoError(t, err)	46		require.NoError(t, err)
73		require.NoError(t, r.Close())	47		require.NoError(t, r.Close())
74	-	assert.Equal(t, "world!!", string(rest))	48	+	assert.Equal(t, "world", string(data))
75	-
76	-	// A new assembly replaces the content whole.
77	-	size, digest, err = s.Assemble(ctx, ref, nil)
78	-	require.NoError(t, err)
79	-	assert.Zero(t, size, "an empty list is an empty content")
80	-	assert.Equal(t, digestOf(""), digest)
81	-
82		ids, err := s.ArtifactIDs(ctx, 7)	49		ids, err := s.ArtifactIDs(ctx, 7)
83		require.NoError(t, err)	50		require.NoError(t, err)
84		assert.Equal(t, []int64{42}, ids)	51		assert.Equal(t, []int64{42}, ids)
85	-	runs, err := s.RunIDs(ctx)
86	-	require.NoError(t, err)
87	-	assert.Equal(t, []int64{7}, runs)
88	-
89	-	require.NoError(t, s.RemoveBlocks(ctx, ref))
90	-	assert.Equal(t, []string{"00000000000000000042"}, entries(t, filepath.Join(root, "00000000000000000007")))
91		require.NoError(t, s.Remove(ctx, ref))	52		require.NoError(t, s.Remove(ctx, ref))
92	-	assert.Empty(t, entries(t, filepath.Join(root, "00000000000000000007")))	53	+	require.NoError(t, s.Remove(ctx, ref))
93		_, err = s.Open(ctx, ref, 0)	54		_, err = s.Open(ctx, ref, 0)
94		assert.ErrorIs(t, err, domain.ErrNotFound)	55		assert.ErrorIs(t, err, domain.ErrNotFound)
95		require.NoError(t, s.RemoveRuns(ctx, []int64{7}))	56		require.NoError(t, s.RemoveRuns(ctx, []int64{7}))
96		assert.Empty(t, entries(t, root))	57		assert.Empty(t, entries(t, root))
97	-	}
98	-
99	-	func TestAMissingBlockIsNotFoundAndLeavesNoContent(t *testing.T) {
100	-	ctx := t.Context()
101	-	root := t.TempDir()
102	-	s := ciartifacts.New(root)
103	-	_, err := s.PutBlock(ctx, ref, "01.aa", strings.NewReader("x"))
104	-	require.NoError(t, err)
105	-	_, _, err = s.Assemble(ctx, ref, []string{"01.aa", "02.aa"})
106	-	assert.ErrorIs(t, err, domain.ErrNotFound)
107	-	_, err = s.Open(ctx, ref, 0)
108	-	assert.ErrorIs(t, err, domain.ErrNotFound, "no content appears")
109	-	for _, e := range entries(t, filepath.Join(root, "00000000000000000007")) {
110	-	assert.False(t, strings.HasPrefix(e, ".tmp-"), e)
111	-	}
112	-	}
113	-
114	-	type failingReader struct{ n int }
115	-
116	-	func (f *failingReader) Read(p []byte) (int, error) {
117	-	if f.n == 0 {
118	-	return 0, errors.New("stream broken")
119	-	}
120	-	f.n--
121	-	p[0] = 'x'
122	-	return 1, nil
123	-	}
124	-
125	-	func TestABrokenSourceLeavesNoBlock(t *testing.T) {
126	-	ctx := t.Context()
127	-	root := t.TempDir()
128	-	s := ciartifacts.New(root)
129	-	_, err := s.PutBlock(ctx, ref, "01.aa", &failingReader{n: 3})
130	-	require.EqualError(t, err, "stream broken", "the source's error as it is")
131	-	assert.Empty(t, entries(t, filepath.Join(root, "00000000000000000007", "00000000000000000042.blocks")))
132	-
133	-	// A file name is never reused: the second put of a name fails.
134	-	_, err = s.PutBlock(ctx, ref, "01.aa", strings.NewReader("a"))
135	-	require.NoError(t, err)
136	-	_, err = s.PutBlock(ctx, ref, "01.aa", strings.NewReader("b"))
137	-	assert.Error(t, err)
138	-	}
139	-
140	-	func TestNamesOutsideTheRulesNeverReachTheFileSystem(t *testing.T) {
141	-	ctx := t.Context()
142	-	root := t.TempDir()
143	-	s := ciartifacts.New(filepath.Join(root, "store"))
144	-	for _, name := range []string{"", "../x", "a/b", ".hidden", "..", "ZZ", "01.aa/../../x", strings.Repeat("a", 201)} {
145	-	_, err := s.PutBlock(ctx, ref, name, strings.NewReader("x"))
146	-	assert.ErrorIs(t, err, domain.ErrInvalid, "%q", name)
147	-	_, _, err = s.Assemble(ctx, ref, []string{name})
148	-	assert.Error(t, err, "%q", name)
149	-	}
150		for _, bad := range []ports.CIArtifactRef{{RunID: 0, ID: 1}, {RunID: 1, ID: -1}} {	58		for _, bad := range []ports.CIArtifactRef{{RunID: 0, ID: 1}, {RunID: 1, ID: -1}} {
151	-	_, err := s.PutBlock(ctx, bad, "01.aa", strings.NewReader("x"))	59	+	_, err := s.Open(ctx, bad, 0)
152	-	assert.ErrorIs(t, err, domain.ErrInvalid)
153	-	_, err = s.Open(ctx, bad, 0)
154		assert.ErrorIs(t, err, domain.ErrNotFound)	60		assert.ErrorIs(t, err, domain.ErrNotFound)
155		}	61		}
156	-	for _, e := range entries(t, root) {
157	-	assert.Equal(t, "store", e, "nothing written beside the store")
158	-	}
159		}	62		}
160			63
161		func TestOrphanListingSeesEveryKindOfFile(t *testing.T) {	64		func TestOrphanListingSeesEveryKindOfFile(t *testing.T) {

M internal/adapters/kvstore/ci_artifact.go +2 -1


@@ -93,7 +93,7 @@ func encodeArtifact(a domain.CIArtifact) ([]byte, error) {
93	93		Id: a.ID, Repo: a.Repo.String(), RunId: a.RunID, Attempt: a.Attempt, JobId: a.JobID,
94	94		Name: a.Name, MediaType: a.MediaType, Size: a.Size, Digest: a.Digest,
95	95		CreatedAt: encodeTime(a.CreatedAt), ExpiresAt: encodeTime(a.ExpiresAt),
96		-	Charged: a.Charged, ListDigest: a.ListDigest,
	96	+	Charged: a.Charged, ListDigest: a.ListDigest, Store: string(a.Store),
97	97		}
98	98		if a.Assignment != (domain.TokenHash{}) {
99	99		pb.Assignment = a.Assignment[:]

@@ -123,6 +123,7 @@ func decodeArtifact(item kv.Item) (domain.CIArtifact, error) {
123	123		Name: pb.GetName(), MediaType: pb.GetMediaType(), Size: pb.GetSize(), Digest: pb.GetDigest(),
124	124		CreatedAt: decodeTime(pb.GetCreatedAt()), ExpiresAt: decodeTime(pb.GetExpiresAt()),
125	125		Charged: pb.GetCharged(), ListDigest: pb.GetListDigest(), Version: item.Version,
	126	+	Store: domain.CIArtifactStore(pb.GetStore()),
126	127		}
127	128		copy(a.Assignment[:], pb.GetAssignment())
128	129		for _, s := range artifactStates {

 		Id: a.ID, Repo: a.Repo.String(), RunId: a.RunID, Attempt: a.Attempt, JobId: a.JobID,
 		Name: a.Name, MediaType: a.MediaType, Size: a.Size, Digest: a.Digest,
 		CreatedAt: encodeTime(a.CreatedAt), ExpiresAt: encodeTime(a.ExpiresAt),
-		Charged: a.Charged, ListDigest: a.ListDigest,
+		Charged: a.Charged, ListDigest: a.ListDigest, Store: string(a.Store),
 	}
 	if a.Assignment != (domain.TokenHash{}) {
 		pb.Assignment = a.Assignment[:]
 		Name: pb.GetName(), MediaType: pb.GetMediaType(), Size: pb.GetSize(), Digest: pb.GetDigest(),
 		CreatedAt: decodeTime(pb.GetCreatedAt()), ExpiresAt: decodeTime(pb.GetExpiresAt()),
 		Charged: pb.GetCharged(), ListDigest: pb.GetListDigest(), Version: item.Version,
+		Store: domain.CIArtifactStore(pb.GetStore()),
 	}
 	copy(a.Assignment[:], pb.GetAssignment())
 	for _, s := range artifactStates {

M internal/adapters/kvstore/ci_artifact_test.go +5 -0


@@ -40,12 +40,17 @@ func TestCIArtifactsRoundTripAndIndexes(t *testing.T) {
40	40		assert.Greater(t, id2, id1, "ids increase")
41	41
42	42		a := pendingArtifact(n, 7, id1, "dist")
	43	+	a.Store = domain.CIArtifactStoreBlob
43	44		b := pendingArtifact(n, 7, id2, "logs")
44	45		require.NoError(t, arts.ApplyArtifacts(ctx, ports.CIArtifactWrite{Repo: n, Put: []domain.CIArtifact{a, b}}))
45	46
46	47		got, err := arts.GetArtifact(ctx, id1)
47	48		require.NoError(t, err)
48	49		assert.Equal(t, "dist", got.Name)
	50	+	assert.Equal(t, domain.CIArtifactStoreBlob, got.Store)
	51	+	legacy, err := arts.GetArtifact(ctx, id2)
	52	+	require.NoError(t, err)
	53	+	assert.Equal(t, domain.CIArtifactStoreLegacy, legacy.Store, "an absent proto field keeps the legacy layout")
49	54		assert.Equal(t, a.Assignment, got.Assignment)
50	55		assert.Equal(t, domain.CIArtifactPending, got.State)
51	56		assert.True(t, got.ExpiresAt.Equal(a.ExpiresAt))


@@ -40,12 +40,17 @@ func TestCIArtifactsRoundTripAndIndexes(t *testing.T) {
40	assert.Greater(t, id2, id1, "ids increase")	40		assert.Greater(t, id2, id1, "ids increase")
41		41
42	a := pendingArtifact(n, 7, id1, "dist")	42		a := pendingArtifact(n, 7, id1, "dist")
		43	+	a.Store = domain.CIArtifactStoreBlob
43	b := pendingArtifact(n, 7, id2, "logs")	44		b := pendingArtifact(n, 7, id2, "logs")
44	require.NoError(t, arts.ApplyArtifacts(ctx, ports.CIArtifactWrite{Repo: n, Put: []domain.CIArtifact{a, b}}))	45		require.NoError(t, arts.ApplyArtifacts(ctx, ports.CIArtifactWrite{Repo: n, Put: []domain.CIArtifact{a, b}}))
45		46
46	got, err := arts.GetArtifact(ctx, id1)	47		got, err := arts.GetArtifact(ctx, id1)
47	require.NoError(t, err)	48		require.NoError(t, err)
48	assert.Equal(t, "dist", got.Name)	49		assert.Equal(t, "dist", got.Name)
		50	+	assert.Equal(t, domain.CIArtifactStoreBlob, got.Store)
		51	+	legacy, err := arts.GetArtifact(ctx, id2)
		52	+	require.NoError(t, err)
		53	+	assert.Equal(t, domain.CIArtifactStoreLegacy, legacy.Store, "an absent proto field keeps the legacy layout")
49	assert.Equal(t, a.Assignment, got.Assignment)	54		assert.Equal(t, a.Assignment, got.Assignment)
50	assert.Equal(t, domain.CIArtifactPending, got.State)	55		assert.Equal(t, domain.CIArtifactPending, got.State)
51	assert.True(t, got.ExpiresAt.Equal(a.ExpiresAt))	56		assert.True(t, got.ExpiresAt.Equal(a.ExpiresAt))

M internal/app/ci_artifacts_test.go +111 -10


@@ -25,6 +25,7 @@ import (
25	25
26	26		pb "bnema.dev/ermite-api/gen/ermite"
27	27		"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
	28	+	"bnema.dev/ermite/internal/adapters/blobdisk"
28	29		"bnema.dev/ermite/internal/adapters/ciartifacts"
29	30		"bnema.dev/ermite/internal/domain"
30	31		"bnema.dev/ermite/internal/ports"

@@ -48,6 +49,21 @@ jobs:
48	49		- run: echo down
49	50		`
50	51
	52	+	// newArtifactHarness supplies the P8 blob wiring through the existing test
	53	+	// hook. Production wiring remains owned by P8.
	54	+	func newArtifactHarness(t testing.T, tweak func(usecase.CIConfig)) *ciHarness {
	55	+	t.Helper()
	56	+	root := t.TempDir()
	57	+	c := newCIHarness(t, func(cfg *usecase.CIConfig) {
	58	+	cfg.Blobs = blobdisk.New(blobdisk.Dir(root))
	59	+	if tweak != nil {
	60	+	tweak(cfg)
	61	+	}
	62	+	})
	63	+	c.artifactRoot = blobdisk.Dir(root)
	64	+	return c
	65	+	}
	66	+
51	67		// artifactAPI are the artifact clients of one caller.
52	68		type artifactAPI struct {
53	69		job ermiteconnect.JobArtifactServiceClient

@@ -202,11 +218,11 @@ func (c *ciHarness) usage() int64 {
202	218		return used
203	219		}
204	220
205		-	// artifactFiles lists every file under ci/artifacts.
	221	+	// artifactFiles lists the new CI objects under the test blob root.
206	222		func (c *ciHarness) artifactFiles() []string {
207	223		c.t.Helper()
208	224		var out []string
209		-	root := ciartifacts.Dir(c.data)
	225	+	root := filepath.Join(c.artifactRoot, "objects", "ci")
210	226		_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
211	227		if err == nil && !d.IsDir() {
212	228		rel, _ := filepath.Rel(root, path)

@@ -249,7 +265,7 @@ func (c ciHarness) newArtifactRun(r fakeRunner, files map[string]string) (runI
249	265		}
250	266
251	267		func TestCIArtifactsPassBetweenJobs(t *testing.T) {
252		-	c := newCIHarness(t, nil)
	268	+	c := newArtifactHarness(t, nil)
253	269		runner := c.registerRunner("alpha", "ubuntu-latest")
254	270		_, upJob, downJob := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
255	271		up, down := c.jobArtifacts(upJob), c.jobArtifacts(downJob)

@@ -417,7 +433,7 @@ func TestCIArtifactsPassBetweenJobs(t *testing.T) {
417	433		// AC3: a job token of another run reaches nothing of this run, and the calls
418	434		// are refused once the job ended.
419	435		func TestCIArtifactsAreIsolatedPerRunAndJob(t *testing.T) {
420		-	c := newCIHarness(t, nil)
	436	+	c := newArtifactHarness(t, nil)
421	437		runner := c.registerRunner("alpha", "ubuntu-latest")
422	438		runA, upA, downA := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
423	439		_, upB, _ := c.newArtifactRun(runner, map[string]string{"README": "two"})

@@ -505,7 +521,7 @@ func (s *shiftClock) since(t time.Time) time.Duration { return s.Now().Sub(t) }
505	521
506	522		func TestCIArtifactLimits(t *testing.T) {
507	523		const mib = 1 << 20
508		-	c := newCIHarness(t, func(cfg *usecase.CIConfig) {
	524	+	c := newArtifactHarness(t, func(cfg *usecase.CIConfig) {
509	525		cfg.ArtifactLimits = ports.ArtifactLimits{Retention: 10 * 24 * time.Hour, Quota: 3 * mib, MaxSize: 2 * mib}
510	526		})
511	527		runner := c.registerRunner("alpha", "ubuntu-latest")

@@ -567,7 +583,7 @@ func TestCIArtifactLimits(t *testing.T) {
567	583
568	584		func TestCIArtifactsSweep(t *testing.T) {
569	585		clk := &shiftClock{}
570		-	c := newCIHarness(t, func(cfg *usecase.CIConfig) {
	586	+	c := newArtifactHarness(t, func(cfg *usecase.CIConfig) {
571	587		cfg.Clock = clk
572	588		cfg.Tick = 20 * time.Millisecond
573	589		cfg.ArtifactLimits.Retention = 2 * 24 * time.Hour

@@ -584,6 +600,7 @@ func TestCIArtifactsSweep(t *testing.T) {
584	600		_, err := up.block(crashed, "x", []byte("stale"))
585	601		require.NoError(t, err)
586	602		stray := filepath.Join(ciartifacts.Dir(c.data), fmt.Sprintf("%020d", runID), fmt.Sprintf("%020d", int64(999999)))
	603	+	require.NoError(t, os.MkdirAll(filepath.Dir(stray), 0o700))
587	604		require.NoError(t, os.WriteFile(stray, []byte("orphan"), 0o600))
588	605		strayRun := filepath.Join(ciartifacts.Dir(c.data), fmt.Sprintf("%020d", int64(888888)))
589	606		require.NoError(t, os.MkdirAll(strayRun, 0o700))

@@ -619,7 +636,7 @@ func TestCIArtifactsSweep(t *testing.T) {
619	636
620	637		// Deleting the repository takes the records and the files of its artifacts.
621	638		func TestCIArtifactsGoWithTheRepository(t *testing.T) {
622		-	c := newCIHarness(t, nil)
	639	+	c := newArtifactHarness(t, nil)
623	640		runner := c.registerRunner("alpha", "ubuntu-latest")
624	641		_, upJob, _ := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
625	642		a := c.jobArtifacts(upJob).upload("dist", []byte("x"))

@@ -648,7 +665,7 @@ func TestCIArtifactsOff(t *testing.T) {
648	665		// before they are written, against the charges and the other writes.
649	666		func TestCIArtifactLimitsHoldUnderConcurrency(t *testing.T) {
650	667		const kib, mib = 1 << 10, 1 << 20
651		-	c := newCIHarness(t, func(cfg *usecase.CIConfig) {
	668	+	c := newArtifactHarness(t, func(cfg *usecase.CIConfig) {
652	669		cfg.ArtifactLimits = ports.ArtifactLimits{Quota: 3 * mib, MaxSize: 2 * mib}
653	670		})
654	671		runner := c.registerRunner("alpha", "ubuntu-latest")

@@ -657,7 +674,7 @@ func TestCIArtifactLimitsHoldUnderConcurrency(t *testing.T) {
657	674
658	675		blockFiles := func() (total int64) {
659	676		for _, f := range c.artifactFiles() {
660		-	if info, err := os.Stat(filepath.Join(ciartifacts.Dir(c.data), f)); err == nil {
	677	+	if info, err := os.Stat(filepath.Join(c.artifactRoot, "objects", "ci", f)); err == nil {
661	678		total += info.Size()
662	679		}
663	680		}

@@ -733,7 +750,7 @@ func TestCIArtifactLimitsHoldUnderConcurrency(t *testing.T) {
733	750		}
734	751
735	752		func TestCIArtifactCaps(t *testing.T) {
736		-	c := newCIHarness(t, nil)
	753	+	c := newArtifactHarness(t, nil)
737	754		runner := c.registerRunner("alpha", "ubuntu-latest")
738	755		_, upJob, downJob := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
739	756		up, down := c.jobArtifacts(upJob), c.jobArtifacts(downJob)

@@ -787,3 +804,87 @@ func TestCIArtifactCaps(t *testing.T) {
787	804		assert.True(t, bytes.Equal(content, body))
788	805		})
789	806		}
	807	+
	808	+	// Both storage generations remain downloadable through the real RPC handlers.
	809	+	func TestCIArtifactBlobAndLegacyDownload(t *testing.T) {
	810	+	c := newArtifactHarness(t, nil)
	811	+	runner := c.registerRunner("alpha", "ubuntu-latest")
	812	+	_, job, _ := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
	813	+	up := c.jobArtifacts(job)
	814	+	current := up.upload("new", []byte("new blob"))
	815	+	store := c.forge.store.Artifacts()
	816	+	a, err := store.GetArtifact(c.ctx, current.GetId())
	817	+	require.NoError(t, err)
	818	+	assert.Equal(t, domain.CIArtifactStoreBlob, a.Store)
	819	+	path := filepath.Join(c.artifactRoot, "objects", filepath.FromSlash(a.BlobKey()))
	820	+	data, err := os.ReadFile(path)
	821	+	require.NoError(t, err)
	822	+	assert.Equal(t, "new blob", string(data))
	823	+	_, got, err := up.download(a.ID, 0)
	824	+	require.NoError(t, err)
	825	+	assert.Equal(t, data, got)
	826	+
	827	+	// Simulate an on-disk record written by the pre-upgrade server: no store
	828	+	// field and contents at the old id-only path, absent from the blob store.
	829	+	legacy := up.upload("old", []byte("old disk"))
	830	+	a, err = store.GetArtifact(c.ctx, legacy.GetId())
	831	+	require.NoError(t, err)
	832	+	oldPath := filepath.Join(ciartifacts.Dir(c.data), fmt.Sprintf("%020d", a.RunID), fmt.Sprintf("%020d", a.ID))
	833	+	require.NoError(t, os.MkdirAll(filepath.Dir(oldPath), 0o700))
	834	+	require.NoError(t, os.WriteFile(oldPath, []byte("old disk"), 0o600))
	835	+	require.NoError(t, os.Remove(filepath.Join(c.artifactRoot, "objects", filepath.FromSlash(a.BlobKey()))))
	836	+	used, version, err := store.ArtifactUsage(c.ctx, a.Repo)
	837	+	require.NoError(t, err)
	838	+	a.Store = domain.CIArtifactStoreLegacy
	839	+	require.NoError(t, store.ApplyArtifacts(c.ctx, ports.CIArtifactWrite{Repo: a.Repo, Used: used, UsageVersion: version, Put: []domain.CIArtifact{a}}))
	840	+	_, got, err = up.download(a.ID, 0)
	841	+	require.NoError(t, err)
	842	+	assert.Equal(t, "old disk", string(got))
	843	+	require.NoError(t, up.remove("old"))
	844	+	_, err = os.Stat(oldPath)
	845	+	assert.True(t, os.IsNotExist(err))
	846	+	}
	847	+
	848	+	func TestCIArtifactWithoutLegacyReader(t *testing.T) {
	849	+	c := newArtifactHarness(t, func(cfg *usecase.CIConfig) { cfg.ArtifactFiles = nil })
	850	+	runner := c.registerRunner("alpha", "ubuntu-latest")
	851	+	_, job, _ := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
	852	+	up := c.jobArtifacts(job)
	853	+	a := up.upload("blob only", []byte("contents"))
	854	+	_, got, err := up.download(a.GetId(), 0)
	855	+	require.NoError(t, err)
	856	+	assert.Equal(t, "contents", string(got))
	857	+	require.NoError(t, c.forge.Repos.Delete(c.ctx, domain.Root(), c.name(c.repo)))
	858	+	assert.Empty(t, c.artifactFiles())
	859	+	}
	860	+
	861	+	func TestCIArtifactMultipartAndOversizedBlock(t *testing.T) {
	862	+	c := newArtifactHarness(t, nil)
	863	+	runner := c.registerRunner("alpha", "ubuntu-latest")
	864	+	_, job, _ := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
	865	+	up := c.jobArtifacts(job)
	866	+	id := up.mustCreate("multipart")
	867	+	first, second := pattern(6<<20, 1), pattern(3<<20, 2)
	868	+	_, err := up.block(id, "a", first)
	869	+	require.NoError(t, err)
	870	+	_, err = up.block(id, "b", second)
	871	+	require.NoError(t, err)
	872	+	committed, err := up.commit(id, "a", "b")
	873	+	require.NoError(t, err)
	874	+	_, err = up.finalize("multipart", committed.GetDigest())
	875	+	require.NoError(t, err)
	876	+	_, got, err := up.download(id, 0)
	877	+	require.NoError(t, err)
	878	+	assert.Equal(t, append(first, second...), got)
	879	+	before := c.usage()
	880	+	refused := up.mustCreate("too big")
	881	+	_, err = up.block(refused, "a", make([]byte, domain.CIArtifactMaxBlockSize+1))
	882	+	require.Error(t, err)
	883	+	assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err))
	884	+	assert.Contains(t, err.Error(), "a block has at most 8 MiB")
	885	+	assert.Equal(t, before, c.usage())
	886	+	a, err := c.forge.store.Artifacts().GetArtifact(c.ctx, refused)
	887	+	require.NoError(t, err)
	888	+	assert.Zero(t, a.Charged)
	889	+	require.NoError(t, blobdisk.New(c.artifactRoot).List(c.ctx, a.BlobKey()+".blocks/", func(ports.BlobInfo) error { t.Error("refused block survived"); return nil }))
	890	+	}


@@ -25,6 +25,7 @@ import (
25			25
26		pb "bnema.dev/ermite-api/gen/ermite"	26		pb "bnema.dev/ermite-api/gen/ermite"
27		"bnema.dev/ermite-api/gen/ermite/ermiteconnect"	27		"bnema.dev/ermite-api/gen/ermite/ermiteconnect"
			28	+	"bnema.dev/ermite/internal/adapters/blobdisk"
28		"bnema.dev/ermite/internal/adapters/ciartifacts"	29		"bnema.dev/ermite/internal/adapters/ciartifacts"
29		"bnema.dev/ermite/internal/domain"	30		"bnema.dev/ermite/internal/domain"
30		"bnema.dev/ermite/internal/ports"	31		"bnema.dev/ermite/internal/ports"

@@ -48,6 +49,21 @@ jobs:
48		- run: echo down	49		- run: echo down
49		`	50		`
50			51
			52	+	// newArtifactHarness supplies the P8 blob wiring through the existing test
			53	+	// hook. Production wiring remains owned by P8.
			54	+	func newArtifactHarness(t testing.T, tweak func(usecase.CIConfig)) *ciHarness {
			55	+	t.Helper()
			56	+	root := t.TempDir()
			57	+	c := newCIHarness(t, func(cfg *usecase.CIConfig) {
			58	+	cfg.Blobs = blobdisk.New(blobdisk.Dir(root))
			59	+	if tweak != nil {
			60	+	tweak(cfg)
			61	+	}
			62	+	})
			63	+	c.artifactRoot = blobdisk.Dir(root)
			64	+	return c
			65	+	}
			66	+
51		// artifactAPI are the artifact clients of one caller.	67		// artifactAPI are the artifact clients of one caller.
52		type artifactAPI struct {	68		type artifactAPI struct {
53		job ermiteconnect.JobArtifactServiceClient	69		job ermiteconnect.JobArtifactServiceClient

@@ -202,11 +218,11 @@ func (c *ciHarness) usage() int64 {
202		return used	218		return used
203		}	219		}
204			220
205	-	// artifactFiles lists every file under ci/artifacts.	221	+	// artifactFiles lists the new CI objects under the test blob root.
206		func (c *ciHarness) artifactFiles() []string {	222		func (c *ciHarness) artifactFiles() []string {
207		c.t.Helper()	223		c.t.Helper()
208		var out []string	224		var out []string
209	-	root := ciartifacts.Dir(c.data)	225	+	root := filepath.Join(c.artifactRoot, "objects", "ci")
210		_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {	226		_ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
211		if err == nil && !d.IsDir() {	227		if err == nil && !d.IsDir() {
212		rel, _ := filepath.Rel(root, path)	228		rel, _ := filepath.Rel(root, path)

@@ -249,7 +265,7 @@ func (c ciHarness) newArtifactRun(r fakeRunner, files map[string]string) (runI
249		}	265		}
250			266
251		func TestCIArtifactsPassBetweenJobs(t *testing.T) {	267		func TestCIArtifactsPassBetweenJobs(t *testing.T) {
252	-	c := newCIHarness(t, nil)	268	+	c := newArtifactHarness(t, nil)
253		runner := c.registerRunner("alpha", "ubuntu-latest")	269		runner := c.registerRunner("alpha", "ubuntu-latest")
254		_, upJob, downJob := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})	270		_, upJob, downJob := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
255		up, down := c.jobArtifacts(upJob), c.jobArtifacts(downJob)	271		up, down := c.jobArtifacts(upJob), c.jobArtifacts(downJob)

@@ -417,7 +433,7 @@ func TestCIArtifactsPassBetweenJobs(t *testing.T) {
417		// AC3: a job token of another run reaches nothing of this run, and the calls	433		// AC3: a job token of another run reaches nothing of this run, and the calls
418		// are refused once the job ended.	434		// are refused once the job ended.
419		func TestCIArtifactsAreIsolatedPerRunAndJob(t *testing.T) {	435		func TestCIArtifactsAreIsolatedPerRunAndJob(t *testing.T) {
420	-	c := newCIHarness(t, nil)	436	+	c := newArtifactHarness(t, nil)
421		runner := c.registerRunner("alpha", "ubuntu-latest")	437		runner := c.registerRunner("alpha", "ubuntu-latest")
422		runA, upA, downA := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})	438		runA, upA, downA := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
423		_, upB, _ := c.newArtifactRun(runner, map[string]string{"README": "two"})	439		_, upB, _ := c.newArtifactRun(runner, map[string]string{"README": "two"})

@@ -505,7 +521,7 @@ func (s *shiftClock) since(t time.Time) time.Duration { return s.Now().Sub(t) }
505			521
506		func TestCIArtifactLimits(t *testing.T) {	522		func TestCIArtifactLimits(t *testing.T) {
507		const mib = 1 << 20	523		const mib = 1 << 20
508	-	c := newCIHarness(t, func(cfg *usecase.CIConfig) {	524	+	c := newArtifactHarness(t, func(cfg *usecase.CIConfig) {
509		cfg.ArtifactLimits = ports.ArtifactLimits{Retention: 10 * 24 * time.Hour, Quota: 3 * mib, MaxSize: 2 * mib}	525		cfg.ArtifactLimits = ports.ArtifactLimits{Retention: 10 * 24 * time.Hour, Quota: 3 * mib, MaxSize: 2 * mib}
510		})	526		})
511		runner := c.registerRunner("alpha", "ubuntu-latest")	527		runner := c.registerRunner("alpha", "ubuntu-latest")

@@ -567,7 +583,7 @@ func TestCIArtifactLimits(t *testing.T) {
567			583
568		func TestCIArtifactsSweep(t *testing.T) {	584		func TestCIArtifactsSweep(t *testing.T) {
569		clk := &shiftClock{}	585		clk := &shiftClock{}
570	-	c := newCIHarness(t, func(cfg *usecase.CIConfig) {	586	+	c := newArtifactHarness(t, func(cfg *usecase.CIConfig) {
571		cfg.Clock = clk	587		cfg.Clock = clk
572		cfg.Tick = 20 * time.Millisecond	588		cfg.Tick = 20 * time.Millisecond
573		cfg.ArtifactLimits.Retention = 2 * 24 * time.Hour	589		cfg.ArtifactLimits.Retention = 2 * 24 * time.Hour

@@ -584,6 +600,7 @@ func TestCIArtifactsSweep(t *testing.T) {
584		_, err := up.block(crashed, "x", []byte("stale"))	600		_, err := up.block(crashed, "x", []byte("stale"))
585		require.NoError(t, err)	601		require.NoError(t, err)
586		stray := filepath.Join(ciartifacts.Dir(c.data), fmt.Sprintf("%020d", runID), fmt.Sprintf("%020d", int64(999999)))	602		stray := filepath.Join(ciartifacts.Dir(c.data), fmt.Sprintf("%020d", runID), fmt.Sprintf("%020d", int64(999999)))
			603	+	require.NoError(t, os.MkdirAll(filepath.Dir(stray), 0o700))
587		require.NoError(t, os.WriteFile(stray, []byte("orphan"), 0o600))	604		require.NoError(t, os.WriteFile(stray, []byte("orphan"), 0o600))
588		strayRun := filepath.Join(ciartifacts.Dir(c.data), fmt.Sprintf("%020d", int64(888888)))	605		strayRun := filepath.Join(ciartifacts.Dir(c.data), fmt.Sprintf("%020d", int64(888888)))
589		require.NoError(t, os.MkdirAll(strayRun, 0o700))	606		require.NoError(t, os.MkdirAll(strayRun, 0o700))

@@ -619,7 +636,7 @@ func TestCIArtifactsSweep(t *testing.T) {
619			636
620		// Deleting the repository takes the records and the files of its artifacts.	637		// Deleting the repository takes the records and the files of its artifacts.
621		func TestCIArtifactsGoWithTheRepository(t *testing.T) {	638		func TestCIArtifactsGoWithTheRepository(t *testing.T) {
622	-	c := newCIHarness(t, nil)	639	+	c := newArtifactHarness(t, nil)
623		runner := c.registerRunner("alpha", "ubuntu-latest")	640		runner := c.registerRunner("alpha", "ubuntu-latest")
624		_, upJob, _ := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})	641		_, upJob, _ := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
625		a := c.jobArtifacts(upJob).upload("dist", []byte("x"))	642		a := c.jobArtifacts(upJob).upload("dist", []byte("x"))

@@ -648,7 +665,7 @@ func TestCIArtifactsOff(t *testing.T) {
648		// before they are written, against the charges and the other writes.	665		// before they are written, against the charges and the other writes.
649		func TestCIArtifactLimitsHoldUnderConcurrency(t *testing.T) {	666		func TestCIArtifactLimitsHoldUnderConcurrency(t *testing.T) {
650		const kib, mib = 1 << 10, 1 << 20	667		const kib, mib = 1 << 10, 1 << 20
651	-	c := newCIHarness(t, func(cfg *usecase.CIConfig) {	668	+	c := newArtifactHarness(t, func(cfg *usecase.CIConfig) {
652		cfg.ArtifactLimits = ports.ArtifactLimits{Quota: 3 * mib, MaxSize: 2 * mib}	669		cfg.ArtifactLimits = ports.ArtifactLimits{Quota: 3 * mib, MaxSize: 2 * mib}
653		})	670		})
654		runner := c.registerRunner("alpha", "ubuntu-latest")	671		runner := c.registerRunner("alpha", "ubuntu-latest")

@@ -657,7 +674,7 @@ func TestCIArtifactLimitsHoldUnderConcurrency(t *testing.T) {
657			674
658		blockFiles := func() (total int64) {	675		blockFiles := func() (total int64) {
659		for _, f := range c.artifactFiles() {	676		for _, f := range c.artifactFiles() {
660	-	if info, err := os.Stat(filepath.Join(ciartifacts.Dir(c.data), f)); err == nil {	677	+	if info, err := os.Stat(filepath.Join(c.artifactRoot, "objects", "ci", f)); err == nil {
661		total += info.Size()	678		total += info.Size()
662		}	679		}
663		}	680		}

@@ -733,7 +750,7 @@ func TestCIArtifactLimitsHoldUnderConcurrency(t *testing.T) {
733		}	750		}
734			751
735		func TestCIArtifactCaps(t *testing.T) {	752		func TestCIArtifactCaps(t *testing.T) {
736	-	c := newCIHarness(t, nil)	753	+	c := newArtifactHarness(t, nil)
737		runner := c.registerRunner("alpha", "ubuntu-latest")	754		runner := c.registerRunner("alpha", "ubuntu-latest")
738		_, upJob, downJob := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})	755		_, upJob, downJob := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
739		up, down := c.jobArtifacts(upJob), c.jobArtifacts(downJob)	756		up, down := c.jobArtifacts(upJob), c.jobArtifacts(downJob)

@@ -787,3 +804,87 @@ func TestCIArtifactCaps(t *testing.T) {
787		assert.True(t, bytes.Equal(content, body))	804		assert.True(t, bytes.Equal(content, body))
788		})	805		})
789		}	806		}
			807	+
			808	+	// Both storage generations remain downloadable through the real RPC handlers.
			809	+	func TestCIArtifactBlobAndLegacyDownload(t *testing.T) {
			810	+	c := newArtifactHarness(t, nil)
			811	+	runner := c.registerRunner("alpha", "ubuntu-latest")
			812	+	_, job, _ := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
			813	+	up := c.jobArtifacts(job)
			814	+	current := up.upload("new", []byte("new blob"))
			815	+	store := c.forge.store.Artifacts()
			816	+	a, err := store.GetArtifact(c.ctx, current.GetId())
			817	+	require.NoError(t, err)
			818	+	assert.Equal(t, domain.CIArtifactStoreBlob, a.Store)
			819	+	path := filepath.Join(c.artifactRoot, "objects", filepath.FromSlash(a.BlobKey()))
			820	+	data, err := os.ReadFile(path)
			821	+	require.NoError(t, err)
			822	+	assert.Equal(t, "new blob", string(data))
			823	+	_, got, err := up.download(a.ID, 0)
			824	+	require.NoError(t, err)
			825	+	assert.Equal(t, data, got)
			826	+
			827	+	// Simulate an on-disk record written by the pre-upgrade server: no store
			828	+	// field and contents at the old id-only path, absent from the blob store.
			829	+	legacy := up.upload("old", []byte("old disk"))
			830	+	a, err = store.GetArtifact(c.ctx, legacy.GetId())
			831	+	require.NoError(t, err)
			832	+	oldPath := filepath.Join(ciartifacts.Dir(c.data), fmt.Sprintf("%020d", a.RunID), fmt.Sprintf("%020d", a.ID))
			833	+	require.NoError(t, os.MkdirAll(filepath.Dir(oldPath), 0o700))
			834	+	require.NoError(t, os.WriteFile(oldPath, []byte("old disk"), 0o600))
			835	+	require.NoError(t, os.Remove(filepath.Join(c.artifactRoot, "objects", filepath.FromSlash(a.BlobKey()))))
			836	+	used, version, err := store.ArtifactUsage(c.ctx, a.Repo)
			837	+	require.NoError(t, err)
			838	+	a.Store = domain.CIArtifactStoreLegacy
			839	+	require.NoError(t, store.ApplyArtifacts(c.ctx, ports.CIArtifactWrite{Repo: a.Repo, Used: used, UsageVersion: version, Put: []domain.CIArtifact{a}}))
			840	+	_, got, err = up.download(a.ID, 0)
			841	+	require.NoError(t, err)
			842	+	assert.Equal(t, "old disk", string(got))
			843	+	require.NoError(t, up.remove("old"))
			844	+	_, err = os.Stat(oldPath)
			845	+	assert.True(t, os.IsNotExist(err))
			846	+	}
			847	+
			848	+	func TestCIArtifactWithoutLegacyReader(t *testing.T) {
			849	+	c := newArtifactHarness(t, func(cfg *usecase.CIConfig) { cfg.ArtifactFiles = nil })
			850	+	runner := c.registerRunner("alpha", "ubuntu-latest")
			851	+	_, job, _ := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
			852	+	up := c.jobArtifacts(job)
			853	+	a := up.upload("blob only", []byte("contents"))
			854	+	_, got, err := up.download(a.GetId(), 0)
			855	+	require.NoError(t, err)
			856	+	assert.Equal(t, "contents", string(got))
			857	+	require.NoError(t, c.forge.Repos.Delete(c.ctx, domain.Root(), c.name(c.repo)))
			858	+	assert.Empty(t, c.artifactFiles())
			859	+	}
			860	+
			861	+	func TestCIArtifactMultipartAndOversizedBlock(t *testing.T) {
			862	+	c := newArtifactHarness(t, nil)
			863	+	runner := c.registerRunner("alpha", "ubuntu-latest")
			864	+	_, job, _ := c.newArtifactRun(runner, map[string]string{".github/workflows/ci.yml": artifactWorkflow})
			865	+	up := c.jobArtifacts(job)
			866	+	id := up.mustCreate("multipart")
			867	+	first, second := pattern(6<<20, 1), pattern(3<<20, 2)
			868	+	_, err := up.block(id, "a", first)
			869	+	require.NoError(t, err)
			870	+	_, err = up.block(id, "b", second)
			871	+	require.NoError(t, err)
			872	+	committed, err := up.commit(id, "a", "b")
			873	+	require.NoError(t, err)
			874	+	_, err = up.finalize("multipart", committed.GetDigest())
			875	+	require.NoError(t, err)
			876	+	_, got, err := up.download(id, 0)
			877	+	require.NoError(t, err)
			878	+	assert.Equal(t, append(first, second...), got)
			879	+	before := c.usage()
			880	+	refused := up.mustCreate("too big")
			881	+	_, err = up.block(refused, "a", make([]byte, domain.CIArtifactMaxBlockSize+1))
			882	+	require.Error(t, err)
			883	+	assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err))
			884	+	assert.Contains(t, err.Error(), "a block has at most 8 MiB")
			885	+	assert.Equal(t, before, c.usage())
			886	+	a, err := c.forge.store.Artifacts().GetArtifact(c.ctx, refused)
			887	+	require.NoError(t, err)
			888	+	assert.Zero(t, a.Charged)
			889	+	require.NoError(t, blobdisk.New(c.artifactRoot).List(c.ctx, a.BlobKey()+".blocks/", func(ports.BlobInfo) error { t.Error("refused block survived"); return nil }))
			890	+	}

M internal/app/ci_e2e_test.go +2 -0


@@ -28,6 +28,8 @@ type ciHarness struct {
28	28		root *clients
29	29		wt string
30	30		repo string
	31	+	// artifactRoot is supplied by the artifact test rig until P8 wires blobs.
	32	+	artifactRoot string
31	33		}
32	34
33	35		const ciWorkflow = `name: ci


@@ -28,6 +28,8 @@ type ciHarness struct {
28	root *clients	28		root *clients
29	wt string	29		wt string
30	repo string	30		repo string
		31	+	// artifactRoot is supplied by the artifact test rig until P8 wires blobs.
		32	+	artifactRoot string
31	}	33		}
32		34
33	const ciWorkflow = `name: ci	35		const ciWorkflow = `name: ci

M internal/domain/ci_artifact.go +21 -0


@@ -28,6 +28,8 @@ const (
28	28		CIArtifactMaxPerRun = 1000
29	29		// CIArtifactChunk is the largest piece of data of one message, both ways.
30	30		CIArtifactChunk = 1 << 20
	31	+	// CIArtifactMaxBlockSize bounds the in-memory staging of a toolkit block.
	32	+	CIArtifactMaxBlockSize = 8 << 20
31	33		// CIArtifactMinRetention is the shortest retention of an artifact.
32	34		CIArtifactMinRetention = 24 * time.Hour
33	35		// CIArtifactPendingMax is how long an upload may stay unfinished before the

@@ -42,6 +44,16 @@ const (
42	44		CIArtifactZip = "application/zip"
43	45		)
44	46
	47	+	// CIArtifactStore identifies the storage of an artifact's contents.
	48	+	type CIArtifactStore string
	49	+
	50	+	const (
	51	+	// CIArtifactStoreLegacy is the pre-blob-store layout under ci/artifacts.
	52	+	CIArtifactStoreLegacy CIArtifactStore = ""
	53	+	// CIArtifactStoreBlob stores the contents through ports.BlobStore.
	54	+	CIArtifactStoreBlob CIArtifactStore = "blob"
	55	+	)
	56	+
45	57		// CIArtifactState is where an artifact is in its upload.
46	58		type CIArtifactState string
47	59

@@ -74,6 +86,7 @@ type CIArtifact struct {
74	86		// Digest is "sha256:<hex>" of the content, known from the commit.
75	87		Digest string
76	88		State CIArtifactState
	89	+	Store CIArtifactStore
77	90		CreatedAt time.Time
78	91		ExpiresAt time.Time
79	92		// Charged is what the artifact counts against the quota of its

@@ -85,6 +98,14 @@ type CIArtifact struct {
85	98		Version uint64
86	99		}
87	100
	101	+	// BlobKey names the contents by immutable run and artifact ids only.
	102	+	func (a CIArtifact) BlobKey() string {
	103	+	return fmt.Sprintf("ci/%020d/%020d", a.RunID, a.ID)
	104	+	}
	105	+
	106	+	// BlockKey names a staged block; file is generated by the use case, not a user.
	107	+	func (a CIArtifact) BlockKey(file string) string { return a.BlobKey() + ".blocks/" + file }
	108	+
88	109		// Final reports whether the artifact is finalized.
89	110		func (a CIArtifact) Final() bool { return a.State == CIArtifactFinal }
90	111


@@ -28,6 +28,8 @@ const (
28	CIArtifactMaxPerRun = 1000	28		CIArtifactMaxPerRun = 1000
29	// CIArtifactChunk is the largest piece of data of one message, both ways.	29		// CIArtifactChunk is the largest piece of data of one message, both ways.
30	CIArtifactChunk = 1 << 20	30		CIArtifactChunk = 1 << 20
		31	+	// CIArtifactMaxBlockSize bounds the in-memory staging of a toolkit block.
		32	+	CIArtifactMaxBlockSize = 8 << 20
31	// CIArtifactMinRetention is the shortest retention of an artifact.	33		// CIArtifactMinRetention is the shortest retention of an artifact.
32	CIArtifactMinRetention = 24 * time.Hour	34		CIArtifactMinRetention = 24 * time.Hour
33	// CIArtifactPendingMax is how long an upload may stay unfinished before the	35		// CIArtifactPendingMax is how long an upload may stay unfinished before the

@@ -42,6 +44,16 @@ const (
42	CIArtifactZip = "application/zip"	44		CIArtifactZip = "application/zip"
43	)	45		)
44		46
		47	+	// CIArtifactStore identifies the storage of an artifact's contents.
		48	+	type CIArtifactStore string
		49	+
		50	+	const (
		51	+	// CIArtifactStoreLegacy is the pre-blob-store layout under ci/artifacts.
		52	+	CIArtifactStoreLegacy CIArtifactStore = ""
		53	+	// CIArtifactStoreBlob stores the contents through ports.BlobStore.
		54	+	CIArtifactStoreBlob CIArtifactStore = "blob"
		55	+	)
		56	+
45	// CIArtifactState is where an artifact is in its upload.	57		// CIArtifactState is where an artifact is in its upload.
46	type CIArtifactState string	58		type CIArtifactState string
47		59

@@ -74,6 +86,7 @@ type CIArtifact struct {
74	// Digest is "sha256:<hex>" of the content, known from the commit.	86		// Digest is "sha256:<hex>" of the content, known from the commit.
75	Digest string	87		Digest string
76	State CIArtifactState	88		State CIArtifactState
		89	+	Store CIArtifactStore
77	CreatedAt time.Time	90		CreatedAt time.Time
78	ExpiresAt time.Time	91		ExpiresAt time.Time
79	// Charged is what the artifact counts against the quota of its	92		// Charged is what the artifact counts against the quota of its

@@ -85,6 +98,14 @@ type CIArtifact struct {
85	Version uint64	98		Version uint64
86	}	99		}
87		100
		101	+	// BlobKey names the contents by immutable run and artifact ids only.
		102	+	func (a CIArtifact) BlobKey() string {
		103	+	return fmt.Sprintf("ci/%020d/%020d", a.RunID, a.ID)
		104	+	}
		105	+
		106	+	// BlockKey names a staged block; file is generated by the use case, not a user.
		107	+	func (a CIArtifact) BlockKey(file string) string { return a.BlobKey() + ".blocks/" + file }
		108	+
88	// Final reports whether the artifact is finalized.	109		// Final reports whether the artifact is finalized.
89	func (a CIArtifact) Final() bool { return a.State == CIArtifactFinal }	110		func (a CIArtifact) Final() bool { return a.State == CIArtifactFinal }
90		111

M internal/domain/ci_artifact_test.go +7 -0


@@ -13,6 +13,13 @@ import (
13	13		"bnema.dev/ermite/internal/domain"
14	14		)
15	15
	16	+	func TestCIArtifactBlobKeys(t *testing.T) {
	17	+	a := domain.CIArtifact{RunID: 7, ID: 42, Name: "not-used-in-keys"}
	18	+	assert.Equal(t, "ci/00000000000000000007/00000000000000000042", a.BlobKey())
	19	+	assert.Equal(t, a.BlobKey()+".blocks/abcd.0123", a.BlockKey("abcd.0123"))
	20	+	assert.Equal(t, domain.CIArtifactStoreLegacy, a.Store)
	21	+	}
	22	+
16	23		func TestParseArtifactName(t *testing.T) {
17	24		for _, ok := range []string{"dist", "my artifact (linux) v1.2", "é", strings.Repeat("é", 256), "..", "a.b-c_d"} {
18	25		assert.NoError(t, domain.ParseArtifactName(ok), "%q", ok)


@@ -13,6 +13,13 @@ import (
13	"bnema.dev/ermite/internal/domain"	13		"bnema.dev/ermite/internal/domain"
14	)	14		)
15		15
		16	+	func TestCIArtifactBlobKeys(t *testing.T) {
		17	+	a := domain.CIArtifact{RunID: 7, ID: 42, Name: "not-used-in-keys"}
		18	+	assert.Equal(t, "ci/00000000000000000007/00000000000000000042", a.BlobKey())
		19	+	assert.Equal(t, a.BlobKey()+".blocks/abcd.0123", a.BlockKey("abcd.0123"))
		20	+	assert.Equal(t, domain.CIArtifactStoreLegacy, a.Store)
		21	+	}
		22	+
16	func TestParseArtifactName(t *testing.T) {	23		func TestParseArtifactName(t *testing.T) {
17	for _, ok := range []string{"dist", "my artifact (linux) v1.2", "é", strings.Repeat("é", 256), "..", "a.b-c_d"} {	24		for _, ok := range []string{"dist", "my artifact (linux) v1.2", "é", strings.Repeat("é", 256), "..", "a.b-c_d"} {
18	assert.NoError(t, domain.ParseArtifactName(ok), "%q", ok)	25		assert.NoError(t, domain.ParseArtifactName(ok), "%q", ok)

M internal/ports/ci_artifact.go +3 -13


@@ -75,24 +75,14 @@ type CIArtifactRef struct {
75	75		ID int64
76	76		}
77	77
78		-	// CIArtifactFiles keeps the contents of the CI artifacts and their staged
79		-	// blocks, never in the metadata store. Files are named by ids only.
	78	+	// CIArtifactFiles reads and removes the artifacts recorded before the blob
	79	+	// store (ADR 0034); removed once they have all expired. *ciartifacts.Store
	80	+	// implements it. Files are named by ids only.
80	81		type CIArtifactFiles interface {
81		-	// PutBlock writes src into a new block file of the artifact, synced, and
82		-	// returns its size. A failure of src or of the write leaves no file.
83		-	PutBlock(ctx context.Context, ref CIArtifactRef, file string, src io.Reader) (int64, error)
84		-	// RemoveBlock deletes one block file; a missing one is not an error.
85		-	RemoveBlock(ctx context.Context, ref CIArtifactRef, file string) error
86		-	// Assemble concatenates the block files in order into the content of the
87		-	// artifact, which replaces any earlier one whole, and returns its size and
88		-	// digest ("sha256:<hex>"). A missing block file is domain.ErrNotFound.
89		-	Assemble(ctx context.Context, ref CIArtifactRef, files []string) (size int64, digest string, err error)
90	82		// Open returns the content from offset, or domain.ErrNotFound.
91	83		Open(ctx context.Context, ref CIArtifactRef, offset int64) (io.ReadCloser, error)
92	84		// Remove deletes the content and the blocks of the artifact.
93	85		Remove(ctx context.Context, ref CIArtifactRef) error
94		-	// RemoveBlocks deletes the blocks of the artifact.
95		-	RemoveBlocks(ctx context.Context, ref CIArtifactRef) error
96	86		// RemoveRuns deletes the files of the runs.
97	87		RemoveRuns(ctx context.Context, runIDs []int64) error
98	88		// RunIDs lists the runs that have a directory, ArtifactIDs the artifacts

 	ID    int64
 }
-// CIArtifactFiles keeps the contents of the CI artifacts and their staged
+// CIArtifactFiles reads and removes the artifacts recorded before the blob
-// blocks, never in the metadata store. Files are named by ids only.
+// store (ADR 0034); removed once they have all expired. *ciartifacts.Store
+// implements it. Files are named by ids only.
 type CIArtifactFiles interface {
-	// PutBlock writes src into a new block file of the artifact, synced, and
-	// returns its size. A failure of src or of the write leaves no file.
-	PutBlock(ctx context.Context, ref CIArtifactRef, file string, src io.Reader) (int64, error)
-	// RemoveBlock deletes one block file; a missing one is not an error.
-	RemoveBlock(ctx context.Context, ref CIArtifactRef, file string) error
-	// Assemble concatenates the block files in order into the content of the
-	// artifact, which replaces any earlier one whole, and returns its size and
-	// digest ("sha256:<hex>"). A missing block file is domain.ErrNotFound.
-	Assemble(ctx context.Context, ref CIArtifactRef, files []string) (size int64, digest string, err error)
 	// Open returns the content from offset, or domain.ErrNotFound.
 	Open(ctx context.Context, ref CIArtifactRef, offset int64) (io.ReadCloser, error)
 	// Remove deletes the content and the blocks of the artifact.
 	Remove(ctx context.Context, ref CIArtifactRef) error
-	// RemoveBlocks deletes the blocks of the artifact.
-	RemoveBlocks(ctx context.Context, ref CIArtifactRef) error
 	// RemoveRuns deletes the files of the runs.
 	RemoveRuns(ctx context.Context, runIDs []int64) error
 	// RunIDs lists the runs that have a directory, ArtifactIDs the artifacts

M internal/ports/mocks/CIArtifactFiles.go +0 -276


@@ -116,84 +116,6 @@ func (_c *CIArtifactFiles_ArtifactIDs_Call) RunAndReturn(run func(ctx context.Co
116	116		return _c
117	117		}
118	118
119		-	// Assemble provides a mock function for the type CIArtifactFiles
120		-	func (_mock *CIArtifactFiles) Assemble(ctx context.Context, ref ports.CIArtifactRef, files []string) (int64, string, error) {
121		-	ret := _mock.Called(ctx, ref, files)
122		-
123		-	if len(ret) == 0 {
124		-	panic("no return value specified for Assemble")
125		-	}
126		-
127		-	var r0 int64
128		-	var r1 string
129		-	var r2 error
130		-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.CIArtifactRef, []string) (int64, string, error)); ok {
131		-	return returnFunc(ctx, ref, files)
132		-	}
133		-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.CIArtifactRef, []string) int64); ok {
134		-	r0 = returnFunc(ctx, ref, files)
135		-	} else {
136		-	r0 = ret.Get(0).(int64)
137		-	}
138		-	if returnFunc, ok := ret.Get(1).(func(context.Context, ports.CIArtifactRef, []string) string); ok {
139		-	r1 = returnFunc(ctx, ref, files)
140		-	} else {
141		-	r1 = ret.Get(1).(string)
142		-	}
143		-	if returnFunc, ok := ret.Get(2).(func(context.Context, ports.CIArtifactRef, []string) error); ok {
144		-	r2 = returnFunc(ctx, ref, files)
145		-	} else {
146		-	r2 = ret.Error(2)
147		-	}
148		-	return r0, r1, r2
149		-	}
150		-
151		-	// CIArtifactFiles_Assemble_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Assemble'
152		-	type CIArtifactFiles_Assemble_Call struct {
153		-	*mock.Call
154		-	}
155		-
156		-	// Assemble is a helper method to define mock.On call
157		-	// - ctx context.Context
158		-	// - ref ports.CIArtifactRef
159		-	// - files []string
160		-	func (_e CIArtifactFiles_Expecter) Assemble(ctx any, ref any, files any) CIArtifactFiles_Assemble_Call {
161		-	return &CIArtifactFiles_Assemble_Call{Call: _e.mock.On("Assemble", ctx, ref, files)}
162		-	}
163		-
164		-	func (_c CIArtifactFiles_Assemble_Call) Run(run func(ctx context.Context, ref ports.CIArtifactRef, files []string)) CIArtifactFiles_Assemble_Call {
165		-	_c.Call.Run(func(args mock.Arguments) {
166		-	var arg0 context.Context
167		-	if args[0] != nil {
168		-	arg0 = args[0].(context.Context)
169		-	}
170		-	var arg1 ports.CIArtifactRef
171		-	if args[1] != nil {
172		-	arg1 = args[1].(ports.CIArtifactRef)
173		-	}
174		-	var arg2 []string
175		-	if args[2] != nil {
176		-	arg2 = args[2].([]string)
177		-	}
178		-	run(
179		-	arg0,
180		-	arg1,
181		-	arg2,
182		-	)
183		-	})
184		-	return _c
185		-	}
186		-
187		-	func (_c CIArtifactFiles_Assemble_Call) Return(size int64, digest string, err error) CIArtifactFiles_Assemble_Call {
188		-	_c.Call.Return(size, digest, err)
189		-	return _c
190		-	}
191		-
192		-	func (_c CIArtifactFiles_Assemble_Call) RunAndReturn(run func(ctx context.Context, ref ports.CIArtifactRef, files []string) (int64, string, error)) CIArtifactFiles_Assemble_Call {
193		-	_c.Call.Return(run)
194		-	return _c
195		-	}
196		-
197	119		// Open provides a mock function for the type CIArtifactFiles
198	120		func (_mock *CIArtifactFiles) Open(ctx context.Context, ref ports.CIArtifactRef, offset int64) (io.ReadCloser, error) {
199	121		ret := _mock.Called(ctx, ref, offset)

@@ -268,84 +190,6 @@ func (_c *CIArtifactFiles_Open_Call) RunAndReturn(run func(ctx context.Context,
268	190		return _c
269	191		}
270	192
271		-	// PutBlock provides a mock function for the type CIArtifactFiles
272		-	func (_mock *CIArtifactFiles) PutBlock(ctx context.Context, ref ports.CIArtifactRef, file string, src io.Reader) (int64, error) {
273		-	ret := _mock.Called(ctx, ref, file, src)
274		-
275		-	if len(ret) == 0 {
276		-	panic("no return value specified for PutBlock")
277		-	}
278		-
279		-	var r0 int64
280		-	var r1 error
281		-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.CIArtifactRef, string, io.Reader) (int64, error)); ok {
282		-	return returnFunc(ctx, ref, file, src)
283		-	}
284		-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.CIArtifactRef, string, io.Reader) int64); ok {
285		-	r0 = returnFunc(ctx, ref, file, src)
286		-	} else {
287		-	r0 = ret.Get(0).(int64)
288		-	}
289		-	if returnFunc, ok := ret.Get(1).(func(context.Context, ports.CIArtifactRef, string, io.Reader) error); ok {
290		-	r1 = returnFunc(ctx, ref, file, src)
291		-	} else {
292		-	r1 = ret.Error(1)
293		-	}
294		-	return r0, r1
295		-	}
296		-
297		-	// CIArtifactFiles_PutBlock_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PutBlock'
298		-	type CIArtifactFiles_PutBlock_Call struct {
299		-	*mock.Call
300		-	}
301		-
302		-	// PutBlock is a helper method to define mock.On call
303		-	// - ctx context.Context
304		-	// - ref ports.CIArtifactRef
305		-	// - file string
306		-	// - src io.Reader
307		-	func (_e CIArtifactFiles_Expecter) PutBlock(ctx any, ref any, file any, src any) CIArtifactFiles_PutBlock_Call {
308		-	return &CIArtifactFiles_PutBlock_Call{Call: _e.mock.On("PutBlock", ctx, ref, file, src)}
309		-	}
310		-
311		-	func (_c CIArtifactFiles_PutBlock_Call) Run(run func(ctx context.Context, ref ports.CIArtifactRef, file string, src io.Reader)) CIArtifactFiles_PutBlock_Call {
312		-	_c.Call.Run(func(args mock.Arguments) {
313		-	var arg0 context.Context
314		-	if args[0] != nil {
315		-	arg0 = args[0].(context.Context)
316		-	}
317		-	var arg1 ports.CIArtifactRef
318		-	if args[1] != nil {
319		-	arg1 = args[1].(ports.CIArtifactRef)
320		-	}
321		-	var arg2 string
322		-	if args[2] != nil {
323		-	arg2 = args[2].(string)
324		-	}
325		-	var arg3 io.Reader
326		-	if args[3] != nil {
327		-	arg3 = args[3].(io.Reader)
328		-	}
329		-	run(
330		-	arg0,
331		-	arg1,
332		-	arg2,
333		-	arg3,
334		-	)
335		-	})
336		-	return _c
337		-	}
338		-
339		-	func (_c CIArtifactFiles_PutBlock_Call) Return(n int64, err error) CIArtifactFiles_PutBlock_Call {
340		-	_c.Call.Return(n, err)
341		-	return _c
342		-	}
343		-
344		-	func (_c CIArtifactFiles_PutBlock_Call) RunAndReturn(run func(ctx context.Context, ref ports.CIArtifactRef, file string, src io.Reader) (int64, error)) CIArtifactFiles_PutBlock_Call {
345		-	_c.Call.Return(run)
346		-	return _c
347		-	}
348		-
349	193		// Remove provides a mock function for the type CIArtifactFiles
350	194		func (_mock *CIArtifactFiles) Remove(ctx context.Context, ref ports.CIArtifactRef) error {
351	195		ret := _mock.Called(ctx, ref)

@@ -403,126 +247,6 @@ func (_c *CIArtifactFiles_Remove_Call) RunAndReturn(run func(ctx context.Context
403	247		return _c
404	248		}
405	249
406		-	// RemoveBlock provides a mock function for the type CIArtifactFiles
407		-	func (_mock *CIArtifactFiles) RemoveBlock(ctx context.Context, ref ports.CIArtifactRef, file string) error {
408		-	ret := _mock.Called(ctx, ref, file)
409		-
410		-	if len(ret) == 0 {
411		-	panic("no return value specified for RemoveBlock")
412		-	}
413		-
414		-	var r0 error
415		-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.CIArtifactRef, string) error); ok {
416		-	r0 = returnFunc(ctx, ref, file)
417		-	} else {
418		-	r0 = ret.Error(0)
419		-	}
420		-	return r0
421		-	}
422		-
423		-	// CIArtifactFiles_RemoveBlock_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RemoveBlock'
424		-	type CIArtifactFiles_RemoveBlock_Call struct {
425		-	*mock.Call
426		-	}
427		-
428		-	// RemoveBlock is a helper method to define mock.On call
429		-	// - ctx context.Context
430		-	// - ref ports.CIArtifactRef
431		-	// - file string
432		-	func (_e CIArtifactFiles_Expecter) RemoveBlock(ctx any, ref any, file any) CIArtifactFiles_RemoveBlock_Call {
433		-	return &CIArtifactFiles_RemoveBlock_Call{Call: _e.mock.On("RemoveBlock", ctx, ref, file)}
434		-	}
435		-
436		-	func (_c CIArtifactFiles_RemoveBlock_Call) Run(run func(ctx context.Context, ref ports.CIArtifactRef, file string)) CIArtifactFiles_RemoveBlock_Call {
437		-	_c.Call.Run(func(args mock.Arguments) {
438		-	var arg0 context.Context
439		-	if args[0] != nil {
440		-	arg0 = args[0].(context.Context)
441		-	}
442		-	var arg1 ports.CIArtifactRef
443		-	if args[1] != nil {
444		-	arg1 = args[1].(ports.CIArtifactRef)
445		-	}
446		-	var arg2 string
447		-	if args[2] != nil {
448		-	arg2 = args[2].(string)
449		-	}
450		-	run(
451		-	arg0,
452		-	arg1,
453		-	arg2,
454		-	)
455		-	})
456		-	return _c
457		-	}
458		-
459		-	func (_c CIArtifactFiles_RemoveBlock_Call) Return(err error) CIArtifactFiles_RemoveBlock_Call {
460		-	_c.Call.Return(err)
461		-	return _c
462		-	}
463		-
464		-	func (_c CIArtifactFiles_RemoveBlock_Call) RunAndReturn(run func(ctx context.Context, ref ports.CIArtifactRef, file string) error) CIArtifactFiles_RemoveBlock_Call {
465		-	_c.Call.Return(run)
466		-	return _c
467		-	}
468		-
469		-	// RemoveBlocks provides a mock function for the type CIArtifactFiles
470		-	func (_mock *CIArtifactFiles) RemoveBlocks(ctx context.Context, ref ports.CIArtifactRef) error {
471		-	ret := _mock.Called(ctx, ref)
472		-
473		-	if len(ret) == 0 {
474		-	panic("no return value specified for RemoveBlocks")
475		-	}
476		-
477		-	var r0 error
478		-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.CIArtifactRef) error); ok {
479		-	r0 = returnFunc(ctx, ref)
480		-	} else {
481		-	r0 = ret.Error(0)
482		-	}
483		-	return r0
484		-	}
485		-
486		-	// CIArtifactFiles_RemoveBlocks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RemoveBlocks'
487		-	type CIArtifactFiles_RemoveBlocks_Call struct {
488		-	*mock.Call
489		-	}
490		-
491		-	// RemoveBlocks is a helper method to define mock.On call
492		-	// - ctx context.Context
493		-	// - ref ports.CIArtifactRef
494		-	func (_e CIArtifactFiles_Expecter) RemoveBlocks(ctx any, ref any) CIArtifactFiles_RemoveBlocks_Call {
495		-	return &CIArtifactFiles_RemoveBlocks_Call{Call: _e.mock.On("RemoveBlocks", ctx, ref)}
496		-	}
497		-
498		-	func (_c CIArtifactFiles_RemoveBlocks_Call) Run(run func(ctx context.Context, ref ports.CIArtifactRef)) CIArtifactFiles_RemoveBlocks_Call {
499		-	_c.Call.Run(func(args mock.Arguments) {
500		-	var arg0 context.Context
501		-	if args[0] != nil {
502		-	arg0 = args[0].(context.Context)
503		-	}
504		-	var arg1 ports.CIArtifactRef
505		-	if args[1] != nil {
506		-	arg1 = args[1].(ports.CIArtifactRef)
507		-	}
508		-	run(
509		-	arg0,
510		-	arg1,
511		-	)
512		-	})
513		-	return _c
514		-	}
515		-
516		-	func (_c CIArtifactFiles_RemoveBlocks_Call) Return(err error) CIArtifactFiles_RemoveBlocks_Call {
517		-	_c.Call.Return(err)
518		-	return _c
519		-	}
520		-
521		-	func (_c CIArtifactFiles_RemoveBlocks_Call) RunAndReturn(run func(ctx context.Context, ref ports.CIArtifactRef) error) CIArtifactFiles_RemoveBlocks_Call {
522		-	_c.Call.Return(run)
523		-	return _c
524		-	}
525		-
526	250		// RemoveRuns provides a mock function for the type CIArtifactFiles
527	251		func (_mock *CIArtifactFiles) RemoveRuns(ctx context.Context, runIDs []int64) error {
528	252		ret := _mock.Called(ctx, runIDs)


@@ -116,84 +116,6 @@ func (_c *CIArtifactFiles_ArtifactIDs_Call) RunAndReturn(run func(ctx context.Co
116		return _c	116	return _c
117		}	117	}
118			118
119	-	// Assemble provides a mock function for the type CIArtifactFiles
120	-	func (_mock *CIArtifactFiles) Assemble(ctx context.Context, ref ports.CIArtifactRef, files []string) (int64, string, error) {
121	-	ret := _mock.Called(ctx, ref, files)
122	-
123	-	if len(ret) == 0 {
124	-	panic("no return value specified for Assemble")
125	-	}
126	-
127	-	var r0 int64
128	-	var r1 string
129	-	var r2 error
130	-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.CIArtifactRef, []string) (int64, string, error)); ok {
131	-	return returnFunc(ctx, ref, files)
132	-	}
133	-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.CIArtifactRef, []string) int64); ok {
134	-	r0 = returnFunc(ctx, ref, files)
135	-	} else {
136	-	r0 = ret.Get(0).(int64)
137	-	}
138	-	if returnFunc, ok := ret.Get(1).(func(context.Context, ports.CIArtifactRef, []string) string); ok {
139	-	r1 = returnFunc(ctx, ref, files)
140	-	} else {
141	-	r1 = ret.Get(1).(string)
142	-	}
143	-	if returnFunc, ok := ret.Get(2).(func(context.Context, ports.CIArtifactRef, []string) error); ok {
144	-	r2 = returnFunc(ctx, ref, files)
145	-	} else {
146	-	r2 = ret.Error(2)
147	-	}
148	-	return r0, r1, r2
149	-	}
150	-
151	-	// CIArtifactFiles_Assemble_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Assemble'
152	-	type CIArtifactFiles_Assemble_Call struct {
153	-	*mock.Call
154	-	}
155	-
156	-	// Assemble is a helper method to define mock.On call
157	-	// - ctx context.Context
158	-	// - ref ports.CIArtifactRef
159	-	// - files []string
160	-	func (_e CIArtifactFiles_Expecter) Assemble(ctx any, ref any, files any) CIArtifactFiles_Assemble_Call {
161	-	return &CIArtifactFiles_Assemble_Call{Call: _e.mock.On("Assemble", ctx, ref, files)}
162	-	}
163	-
164	-	func (_c CIArtifactFiles_Assemble_Call) Run(run func(ctx context.Context, ref ports.CIArtifactRef, files []string)) CIArtifactFiles_Assemble_Call {
165	-	_c.Call.Run(func(args mock.Arguments) {
166	-	var arg0 context.Context
167	-	if args[0] != nil {
168	-	arg0 = args[0].(context.Context)
169	-	}
170	-	var arg1 ports.CIArtifactRef
171	-	if args[1] != nil {
172	-	arg1 = args[1].(ports.CIArtifactRef)
173	-	}
174	-	var arg2 []string
175	-	if args[2] != nil {
176	-	arg2 = args[2].([]string)
177	-	}
178	-	run(
179	-	arg0,
180	-	arg1,
181	-	arg2,
182	-	)
183	-	})
184	-	return _c
185	-	}
186	-
187	-	func (_c CIArtifactFiles_Assemble_Call) Return(size int64, digest string, err error) CIArtifactFiles_Assemble_Call {
188	-	_c.Call.Return(size, digest, err)
189	-	return _c
190	-	}
191	-
192	-	func (_c CIArtifactFiles_Assemble_Call) RunAndReturn(run func(ctx context.Context, ref ports.CIArtifactRef, files []string) (int64, string, error)) CIArtifactFiles_Assemble_Call {
193	-	_c.Call.Return(run)
194	-	return _c
195	-	}
196	-
197		// Open provides a mock function for the type CIArtifactFiles	119	// Open provides a mock function for the type CIArtifactFiles
198		func (_mock *CIArtifactFiles) Open(ctx context.Context, ref ports.CIArtifactRef, offset int64) (io.ReadCloser, error) {	120	func (_mock *CIArtifactFiles) Open(ctx context.Context, ref ports.CIArtifactRef, offset int64) (io.ReadCloser, error) {
199		ret := _mock.Called(ctx, ref, offset)	121	ret := _mock.Called(ctx, ref, offset)

@@ -268,84 +190,6 @@ func (_c *CIArtifactFiles_Open_Call) RunAndReturn(run func(ctx context.Context,
268		return _c	190	return _c
269		}	191	}
270			192
271	-	// PutBlock provides a mock function for the type CIArtifactFiles
272	-	func (_mock *CIArtifactFiles) PutBlock(ctx context.Context, ref ports.CIArtifactRef, file string, src io.Reader) (int64, error) {
273	-	ret := _mock.Called(ctx, ref, file, src)
274	-
275	-	if len(ret) == 0 {
276	-	panic("no return value specified for PutBlock")
277	-	}
278	-
279	-	var r0 int64
280	-	var r1 error
281	-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.CIArtifactRef, string, io.Reader) (int64, error)); ok {
282	-	return returnFunc(ctx, ref, file, src)
283	-	}
284	-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.CIArtifactRef, string, io.Reader) int64); ok {
285	-	r0 = returnFunc(ctx, ref, file, src)
286	-	} else {
287	-	r0 = ret.Get(0).(int64)
288	-	}
289	-	if returnFunc, ok := ret.Get(1).(func(context.Context, ports.CIArtifactRef, string, io.Reader) error); ok {
290	-	r1 = returnFunc(ctx, ref, file, src)
291	-	} else {
292	-	r1 = ret.Error(1)
293	-	}
294	-	return r0, r1
295	-	}
296	-
297	-	// CIArtifactFiles_PutBlock_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PutBlock'
298	-	type CIArtifactFiles_PutBlock_Call struct {
299	-	*mock.Call
300	-	}
301	-
302	-	// PutBlock is a helper method to define mock.On call
303	-	// - ctx context.Context
304	-	// - ref ports.CIArtifactRef
305	-	// - file string
306	-	// - src io.Reader
307	-	func (_e CIArtifactFiles_Expecter) PutBlock(ctx any, ref any, file any, src any) CIArtifactFiles_PutBlock_Call {
308	-	return &CIArtifactFiles_PutBlock_Call{Call: _e.mock.On("PutBlock", ctx, ref, file, src)}
309	-	}
310	-
311	-	func (_c CIArtifactFiles_PutBlock_Call) Run(run func(ctx context.Context, ref ports.CIArtifactRef, file string, src io.Reader)) CIArtifactFiles_PutBlock_Call {
312	-	_c.Call.Run(func(args mock.Arguments) {
313	-	var arg0 context.Context
314	-	if args[0] != nil {
315	-	arg0 = args[0].(context.Context)
316	-	}
317	-	var arg1 ports.CIArtifactRef
318	-	if args[1] != nil {
319	-	arg1 = args[1].(ports.CIArtifactRef)
320	-	}
321	-	var arg2 string
322	-	if args[2] != nil {
323	-	arg2 = args[2].(string)
324	-	}
325	-	var arg3 io.Reader
326	-	if args[3] != nil {
327	-	arg3 = args[3].(io.Reader)
328	-	}
329	-	run(
330	-	arg0,
331	-	arg1,
332	-	arg2,
333	-	arg3,
334	-	)
335	-	})
336	-	return _c
337	-	}
338	-
339	-	func (_c CIArtifactFiles_PutBlock_Call) Return(n int64, err error) CIArtifactFiles_PutBlock_Call {
340	-	_c.Call.Return(n, err)
341	-	return _c
342	-	}
343	-
344	-	func (_c CIArtifactFiles_PutBlock_Call) RunAndReturn(run func(ctx context.Context, ref ports.CIArtifactRef, file string, src io.Reader) (int64, error)) CIArtifactFiles_PutBlock_Call {
345	-	_c.Call.Return(run)
346	-	return _c
347	-	}
348	-
349		// Remove provides a mock function for the type CIArtifactFiles	193	// Remove provides a mock function for the type CIArtifactFiles
350		func (_mock *CIArtifactFiles) Remove(ctx context.Context, ref ports.CIArtifactRef) error {	194	func (_mock *CIArtifactFiles) Remove(ctx context.Context, ref ports.CIArtifactRef) error {
351		ret := _mock.Called(ctx, ref)	195	ret := _mock.Called(ctx, ref)

@@ -403,126 +247,6 @@ func (_c *CIArtifactFiles_Remove_Call) RunAndReturn(run func(ctx context.Context
403		return _c	247	return _c
404		}	248	}
405			249
406	-	// RemoveBlock provides a mock function for the type CIArtifactFiles
407	-	func (_mock *CIArtifactFiles) RemoveBlock(ctx context.Context, ref ports.CIArtifactRef, file string) error {
408	-	ret := _mock.Called(ctx, ref, file)
409	-
410	-	if len(ret) == 0 {
411	-	panic("no return value specified for RemoveBlock")
412	-	}
413	-
414	-	var r0 error
415	-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.CIArtifactRef, string) error); ok {
416	-	r0 = returnFunc(ctx, ref, file)
417	-	} else {
418	-	r0 = ret.Error(0)
419	-	}
420	-	return r0
421	-	}
422	-
423	-	// CIArtifactFiles_RemoveBlock_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RemoveBlock'
424	-	type CIArtifactFiles_RemoveBlock_Call struct {
425	-	*mock.Call
426	-	}
427	-
428	-	// RemoveBlock is a helper method to define mock.On call
429	-	// - ctx context.Context
430	-	// - ref ports.CIArtifactRef
431	-	// - file string
432	-	func (_e CIArtifactFiles_Expecter) RemoveBlock(ctx any, ref any, file any) CIArtifactFiles_RemoveBlock_Call {
433	-	return &CIArtifactFiles_RemoveBlock_Call{Call: _e.mock.On("RemoveBlock", ctx, ref, file)}
434	-	}
435	-
436	-	func (_c CIArtifactFiles_RemoveBlock_Call) Run(run func(ctx context.Context, ref ports.CIArtifactRef, file string)) CIArtifactFiles_RemoveBlock_Call {
437	-	_c.Call.Run(func(args mock.Arguments) {
438	-	var arg0 context.Context
439	-	if args[0] != nil {
440	-	arg0 = args[0].(context.Context)
441	-	}
442	-	var arg1 ports.CIArtifactRef
443	-	if args[1] != nil {
444	-	arg1 = args[1].(ports.CIArtifactRef)
445	-	}
446	-	var arg2 string
447	-	if args[2] != nil {
448	-	arg2 = args[2].(string)
449	-	}
450	-	run(
451	-	arg0,
452	-	arg1,
453	-	arg2,
454	-	)
455	-	})
456	-	return _c
457	-	}
458	-
459	-	func (_c CIArtifactFiles_RemoveBlock_Call) Return(err error) CIArtifactFiles_RemoveBlock_Call {
460	-	_c.Call.Return(err)
461	-	return _c
462	-	}
463	-
464	-	func (_c CIArtifactFiles_RemoveBlock_Call) RunAndReturn(run func(ctx context.Context, ref ports.CIArtifactRef, file string) error) CIArtifactFiles_RemoveBlock_Call {
465	-	_c.Call.Return(run)
466	-	return _c
467	-	}
468	-
469	-	// RemoveBlocks provides a mock function for the type CIArtifactFiles
470	-	func (_mock *CIArtifactFiles) RemoveBlocks(ctx context.Context, ref ports.CIArtifactRef) error {
471	-	ret := _mock.Called(ctx, ref)
472	-
473	-	if len(ret) == 0 {
474	-	panic("no return value specified for RemoveBlocks")
475	-	}
476	-
477	-	var r0 error
478	-	if returnFunc, ok := ret.Get(0).(func(context.Context, ports.CIArtifactRef) error); ok {
479	-	r0 = returnFunc(ctx, ref)
480	-	} else {
481	-	r0 = ret.Error(0)
482	-	}
483	-	return r0
484	-	}
485	-
486	-	// CIArtifactFiles_RemoveBlocks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RemoveBlocks'
487	-	type CIArtifactFiles_RemoveBlocks_Call struct {
488	-	*mock.Call
489	-	}
490	-
491	-	// RemoveBlocks is a helper method to define mock.On call
492	-	// - ctx context.Context
493	-	// - ref ports.CIArtifactRef
494	-	func (_e CIArtifactFiles_Expecter) RemoveBlocks(ctx any, ref any) CIArtifactFiles_RemoveBlocks_Call {
495	-	return &CIArtifactFiles_RemoveBlocks_Call{Call: _e.mock.On("RemoveBlocks", ctx, ref)}
496	-	}
497	-
498	-	func (_c CIArtifactFiles_RemoveBlocks_Call) Run(run func(ctx context.Context, ref ports.CIArtifactRef)) CIArtifactFiles_RemoveBlocks_Call {
499	-	_c.Call.Run(func(args mock.Arguments) {
500	-	var arg0 context.Context
501	-	if args[0] != nil {
502	-	arg0 = args[0].(context.Context)
503	-	}
504	-	var arg1 ports.CIArtifactRef
505	-	if args[1] != nil {
506	-	arg1 = args[1].(ports.CIArtifactRef)
507	-	}
508	-	run(
509	-	arg0,
510	-	arg1,
511	-	)
512	-	})
513	-	return _c
514	-	}
515	-
516	-	func (_c CIArtifactFiles_RemoveBlocks_Call) Return(err error) CIArtifactFiles_RemoveBlocks_Call {
517	-	_c.Call.Return(err)
518	-	return _c
519	-	}
520	-
521	-	func (_c CIArtifactFiles_RemoveBlocks_Call) RunAndReturn(run func(ctx context.Context, ref ports.CIArtifactRef) error) CIArtifactFiles_RemoveBlocks_Call {
522	-	_c.Call.Return(run)
523	-	return _c
524	-	}
525	-
526		// RemoveRuns provides a mock function for the type CIArtifactFiles	250	// RemoveRuns provides a mock function for the type CIArtifactFiles
527		func (_mock *CIArtifactFiles) RemoveRuns(ctx context.Context, runIDs []int64) error {	251	func (_mock *CIArtifactFiles) RemoveRuns(ctx context.Context, runIDs []int64) error {
528		ret := _mock.Called(ctx, runIDs)	252	ret := _mock.Called(ctx, runIDs)

M internal/usecase/ci.go +6 -3


@@ -125,14 +125,17 @@ type CIConfig struct {
125	125		PublicURL string
126	126		// Retention is how long runs, jobs, checks and logs are kept.
127	127		Retention time.Duration
128		-	// Artifacts and ArtifactFiles keep the artifacts of the runs (CI-4 D3):
129		-	// the records and the contents. Without both the artifact services refuse
	128	+	// Artifacts and Blobs keep the records and contents of the artifacts.
	129	+	// Without both the artifact services refuse
130	130		// every call. ArtifactLimits are their settings; a zero field takes its
131	131		// default (the run retention of domain.CIRetentionDays,
132	132		// domain.DefaultCIArtifactQuota, domain.DefaultCIArtifactMaxSize). Leave
133	133		// them unset rather than assigning typed nil pointers.
134		-	Artifacts ports.CIArtifactStore
	134	+	Artifacts ports.CIArtifactStore
	135	+	// ArtifactFiles is the legacy reader; nil when $ERMITE_DATA_DIR/ci/artifacts
	136	+	// does not exist at startup; P8 wires it.
135	137		ArtifactFiles ports.CIArtifactFiles
	138	+	Blobs ports.BlobStore
136	139		ArtifactLimits ports.ArtifactLimits
137	140
138	141		// The durations below default to the contract's; tests shorten them.

 	PublicURL string
 	// Retention is how long runs, jobs, checks and logs are kept.
 	Retention time.Duration
-	// Artifacts and ArtifactFiles keep the artifacts of the runs (CI-4 D3):
+	// Artifacts and Blobs keep the records and contents of the artifacts.
-	// the records and the contents. Without both the artifact services refuse
+	// Without both the artifact services refuse
 	// every call. ArtifactLimits are their settings; a zero field takes its
 	// default (the run retention of domain.CIRetentionDays,
 	// domain.DefaultCIArtifactQuota, domain.DefaultCIArtifactMaxSize). Leave
 	// them unset rather than assigning typed nil pointers.
-	Artifacts      ports.CIArtifactStore
+	Artifacts ports.CIArtifactStore
+	// ArtifactFiles is the legacy reader; nil when $ERMITE_DATA_DIR/ci/artifacts
+	// does not exist at startup; P8 wires it.
 	ArtifactFiles  ports.CIArtifactFiles
+	Blobs          ports.BlobStore
 	ArtifactLimits ports.ArtifactLimits
 	// The durations below default to the contract's; tests shorten them.

M internal/usecase/ci_artifacts.go +272 -12


@@ -13,6 +13,7 @@ import (
13	13		"fmt"
14	14		"io"
15	15		"strconv"
	16	+	"strings"
16	17		"sync"
17	18		"time"
18	19

@@ -25,7 +26,7 @@ import (
25	26		// The artifacts of the runs (artifacts v4, CI-4 D3). The runner translates the
26	27		// toolkit's protocol and calls JobArtifactService with the job token; owners
27	28		// use ArtifactService. The records are in CIConfig.Artifacts, the contents in
28		-	// CIConfig.ArtifactFiles, named by ids only.
	29	+	// CIConfig.Blobs (ArtifactFiles for legacy records), named by ids only.
29	30		//
30	31		// Every write of the artifacts of a repository goes through its usage record
31	32		// (ports.CIArtifactWrite): a decision is made on the records read after it and

@@ -57,7 +58,7 @@ var (
57	58		errArtifactChanged = errors.New("artifact changed")
58	59		)
59	60
60		-	func (c *CI) artifactsOn() bool { return c.cfg.Artifacts != nil && c.cfg.ArtifactFiles != nil }
	61	+	func (c *CI) artifactsOn() bool { return c.cfg.Artifacts != nil && c.cfg.Blobs != nil }
61	62
62	63		func artifactRef(a domain.CIArtifact) ports.CIArtifactRef {
63	64		return ports.CIArtifactRef{RunID: a.RunID, ID: a.ID}

@@ -309,7 +310,13 @@ func (c *CI) removeArtifacts(ctx context.Context, repo domain.RepoName, runID in
309	310		// logged: the sweep removes the files that no record names.
310	311		func (c *CI) removeArtifactFiles(ctx context.Context, arts []domain.CIArtifact) {
311	312		for _, a := range arts {
312		-	if err := c.cfg.ArtifactFiles.Remove(ctx, artifactRef(a)); err != nil {
	313	+	var err error
	314	+	if a.Store == domain.CIArtifactStoreBlob {
	315	+	err = errors.Join(c.cfg.Blobs.Delete(ctx, a.BlobKey()), c.removeBlobBlocks(ctx, a))
	316	+	} else if c.cfg.ArtifactFiles != nil {
	317	+	err = c.cfg.ArtifactFiles.Remove(ctx, artifactRef(a))
	318	+	}
	319	+	if err != nil {
313	320		ctxLog(ctx).Warn().Err(err).Int64("artifact_id", a.ID).Msg("ci: remove the files of an artifact; the sweep will")
314	321		}
315	322		}

@@ -373,7 +380,7 @@ func (c *CI) CreateJobArtifact(ctx context.Context, secret string, params ports.
373	380		}
374	381		a := domain.CIArtifact{
375	382		ID: id, Repo: j.Repo, RunID: j.RunID, Attempt: j.RunAttempt, JobID: j.ID, Assignment: j.TokenHash,
376		-	Name: params.Name, MediaType: mediaType, State: domain.CIArtifactPending,
	383	+	Name: params.Name, MediaType: mediaType, State: domain.CIArtifactPending, Store: domain.CIArtifactStoreBlob,
377	384		CreatedAt: now, ExpiresAt: domain.ArtifactExpiry(params.ExpiresAt, now, c.cfg.ArtifactLimits.Retention),
378	385		}
379	386		var discarded []domain.CIArtifact

@@ -466,9 +473,12 @@ func (c *CI) UploadJobArtifactBlock(ctx context.Context, secret, jobID string, a
466	473		src := &reservingReader{ctx: ctx, c: c, a: a, r: data}
467	474		defer func() { c.release(a, src.reserved) }()
468	475		limits := c.cfg.ArtifactLimits
469		-	ref, file := artifactRef(a), newBlockFile(blockID)
470		-	size, err := c.cfg.ArtifactFiles.PutBlock(ctx, ref, file, src)
	476	+	file := newBlockFile(blockID)
	477	+	size, err := c.putUnsized(ctx, a.BlockKey(file), src)
471	478		if err != nil {
	479	+	if derr := c.cfg.Blobs.Delete(context.WithoutCancel(ctx), a.BlockKey(file)); derr != nil {
	480	+	ctxLog(ctx).Warn().Err(derr).Msg("ci: remove a refused block; the sweep will")
	481	+	}
472	482		return 0, err
473	483		}
474	484		var old *ports.CIArtifactBlock

@@ -509,13 +519,13 @@ func (c *CI) UploadJobArtifactBlock(ctx context.Context, secret, jobID string, a
509	519		return nil
510	520		})
511	521		if err != nil {
512		-	if rerr := c.cfg.ArtifactFiles.RemoveBlock(ctx, ref, file); rerr != nil {
	522	+	if rerr := c.cfg.Blobs.Delete(context.WithoutCancel(ctx), a.BlockKey(file)); rerr != nil {
513	523		ctxLog(ctx).Warn().Err(rerr).Int64("artifact_id", a.ID).Msg("ci: remove a refused block; the sweep will")
514	524		}
515	525		return 0, err
516	526		}
517	527		if old != nil {
518		-	if err := c.cfg.ArtifactFiles.RemoveBlock(ctx, ref, old.File); err != nil {
	528	+	if err := c.cfg.Blobs.Delete(ctx, a.BlockKey(old.File)); err != nil {
519	529		ctxLog(ctx).Warn().Err(err).Int64("artifact_id", a.ID).Msg("ci: remove a replaced block")
520	530		}
521	531		}

@@ -602,7 +612,7 @@ func (c *CI) CommitJobArtifact(ctx context.Context, secret, jobID string, artifa
602	612		return 0, "", err
603	613		}
604	614		}
605		-	size, digest, err := c.cfg.ArtifactFiles.Assemble(ctx, artifactRef(a), files)
	615	+	size, digest, err := c.assembleBlobs(ctx, a, files, total)
606	616		if err != nil {
607	617		c.release(a, excess)
608	618		}

@@ -638,7 +648,7 @@ func (c *CI) CommitJobArtifact(ctx context.Context, secret, jobID string, artifa
638	648		if err != nil {
639	649		return 0, "", err
640	650		}
641		-	if err := c.cfg.ArtifactFiles.RemoveBlocks(ctx, artifactRef(a)); err != nil {
	651	+	if err := c.removeBlobBlocks(ctx, a); err != nil {
642	652		ctxLog(ctx).Warn().Err(err).Int64("artifact_id", a.ID).Msg("ci: remove the blocks of a committed artifact")
643	653		}
644	654		c.dropIfEnded(ctx, j)

@@ -782,7 +792,23 @@ func (c *CI) streamArtifact(ctx context.Context, a domain.CIArtifact, offset int
782	792		if offset < 0 \|\| offset > a.Size {
783	793		return fmt.Errorf("%w: offset %d is beyond the size %d", domain.ErrOutOfRange, offset, a.Size)
784	794		}
785		-	f, err := c.cfg.ArtifactFiles.Open(ctx, artifactRef(a), offset)
	795	+	var f io.ReadCloser
	796	+	var err error
	797	+	if a.Store == domain.CIArtifactStoreBlob {
	798	+	// BlobStore rejects offsets at EOF, while artifact downloads permit
	799	+	// an empty resumed stream (including zero-byte artifacts).
	800	+	if offset == a.Size {
	801	+	_, err = c.cfg.Blobs.Stat(ctx, a.BlobKey())
	802	+	f = io.NopCloser(bytes.NewReader(nil))
	803	+	} else {
	804	+	f, _, err = c.cfg.Blobs.Open(ctx, a.BlobKey(), offset, -1)
	805	+	}
	806	+	} else if c.cfg.ArtifactFiles != nil {
	807	+	f, err = c.cfg.ArtifactFiles.Open(ctx, artifactRef(a), offset)
	808	+	} else {
	809	+	ctxLog(ctx).Error().Int64("artifact_id", a.ID).Msg("ci: legacy artifact reader is unavailable")
	810	+	return domain.ErrNotFound
	811	+	}
786	812		if err != nil {
787	813		return err
788	814		}

@@ -1008,6 +1034,10 @@ func (c *CI) sweepArtifactOrphans(ctx context.Context) {
1008	1034		}
1009	1035		}
1010	1036		}
	1037	+	c.sweepArtifactBlobOrphans(ctx)
	1038	+	if c.cfg.ArtifactFiles == nil {
	1039	+	return
	1040	+	}
1011	1041		c.removeOrphans(ctx, "artifacts", "artifact directories", c.cfg.ArtifactFiles.RunIDs, c.cfg.ArtifactFiles.RemoveRuns)
1012	1042		runIDs, err := c.cfg.ArtifactFiles.RunIDs(ctx)
1013	1043		if err != nil {

@@ -1022,7 +1052,7 @@ func (c *CI) sweepArtifactOrphans(ctx context.Context) {
1022	1052		}
1023	1053		for _, id := range ids {
1024	1054		a, err := c.cfg.Artifacts.GetArtifact(ctx, id)
1025		-	if err == nil && a.RunID == run {
	1055	+	if err == nil && a.RunID == run && a.Store != domain.CIArtifactStoreBlob {
1026	1056		continue
1027	1057		}
1028	1058		if err != nil && !errors.Is(err, domain.ErrNotFound) {

@@ -1035,3 +1065,233 @@ func (c *CI) sweepArtifactOrphans(ctx context.Context) {
1035	1065		}
1036	1066		}
1037	1067		}
	1068	+
	1069	+	// putUnsized buffers at most one toolkit block; Put requires an exact size.
	1070	+	func (c *CI) putUnsized(ctx context.Context, key string, r io.Reader) (int64, error) {
	1071	+	// A fixed backing array prevents bytes.Buffer's geometric growth from
	1072	+	// allocating beyond the block bound. Probe the next byte separately.
	1073	+	buf := make([]byte, domain.CIArtifactMaxBlockSize)
	1074	+	n, err := io.ReadFull(r, buf)
	1075	+	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
	1076	+	return 0, err
	1077	+	}
	1078	+	if n == len(buf) {
	1079	+	var extra [1]byte
	1080	+	m, err := io.ReadFull(r, extra[:])
	1081	+	if m != 0 {
	1082	+	return 0, fmt.Errorf("%w: a block has at most 8 MiB", domain.ErrInvalid)
	1083	+	}
	1084	+	if err != io.EOF {
	1085	+	return 0, err
	1086	+	}
	1087	+	}
	1088	+	if err := ctx.Err(); err != nil {
	1089	+	return 0, err
	1090	+	}
	1091	+	_, err = c.cfg.Blobs.Put(ctx, key, bytes.NewReader(buf[:n]), int64(n))
	1092	+	return int64(n), err
	1093	+	}
	1094	+
	1095	+	func (c *CI) removeBlobBlocks(ctx context.Context, a domain.CIArtifact) error {
	1096	+	return c.cfg.Blobs.List(ctx, a.BlobKey()+".blocks/", func(b ports.BlobInfo) error {
	1097	+	return c.cfg.Blobs.Delete(ctx, b.Key)
	1098	+	})
	1099	+	}
	1100	+
	1101	+	// assembleBlobs keeps one 8 MiB part in memory, even when a block crosses
	1102	+	// part boundaries or is repeated in the commit list.
	1103	+	func (c *CI) assembleBlobs(ctx context.Context, a domain.CIArtifact, files []string, total int64) (int64, string, error) {
	1104	+	var upload ports.MultipartUpload
	1105	+	if total > ports.BlobPartSize {
	1106	+	var err error
	1107	+	upload, err = c.cfg.Blobs.NewMultipart(ctx, a.BlobKey())
	1108	+	if err != nil {
	1109	+	return 0, "", err
	1110	+	}
	1111	+	}
	1112	+	completed := false
	1113	+	defer func() {
	1114	+	if upload != nil && !completed {
	1115	+	if err := upload.Abort(context.WithoutCancel(ctx)); err != nil {
	1116	+	ctxLog(ctx).Warn().Err(err).Msg("ci: abort artifact assembly; the sweep will")
	1117	+	}
	1118	+	}
	1119	+	}()
	1120	+	buf := make([]byte, ports.BlobPartSize)
	1121	+	used, part := 0, 1
	1122	+	var size int64
	1123	+	h := sha256.New()
	1124	+	flush := func() error {
	1125	+	if err := upload.UploadPart(ctx, part, bytes.NewReader(buf[:used]), int64(used)); err != nil {
	1126	+	return err
	1127	+	}
	1128	+	used = 0
	1129	+	part++
	1130	+	return nil
	1131	+	}
	1132	+	for _, file := range files {
	1133	+	r, info, err := c.cfg.Blobs.Open(ctx, a.BlockKey(file), 0, -1)
	1134	+	// Empty blocks cannot be opened under BlobStore's EOF contract.
	1135	+	if errors.Is(err, domain.ErrOutOfRange) {
	1136	+	info, err = c.cfg.Blobs.Stat(ctx, a.BlockKey(file))
	1137	+	if err == nil && info.Size == 0 {
	1138	+	continue
	1139	+	}
	1140	+	if err == nil {
	1141	+	return 0, "", domain.ErrOutOfRange
	1142	+	}
	1143	+	}
	1144	+	if err != nil {
	1145	+	return 0, "", err
	1146	+	}
	1147	+	if info.Size > total-size {
	1148	+	_ = r.Close()
	1149	+	return 0, "", fmt.Errorf("%w: artifact block size changed", domain.ErrInvalid)
	1150	+	}
	1151	+	left := info.Size
	1152	+	for left > 0 {
	1153	+	if err := ctx.Err(); err != nil {
	1154	+	_ = r.Close()
	1155	+	return 0, "", err
	1156	+	}
	1157	+	want := min(left, int64(len(buf)-used))
	1158	+	if want == 0 {
	1159	+	_ = r.Close()
	1160	+	return 0, "", fmt.Errorf("%w: artifact block size changed", domain.ErrInvalid)
	1161	+	}
	1162	+	n, err := io.ReadFull(r, buf[used:used+int(want)])
	1163	+	if err != nil {
	1164	+	_ = r.Close()
	1165	+	return 0, "", err
	1166	+	}
	1167	+	h.Write(buf[used : used+n])
	1168	+	used += n
	1169	+	size += int64(n)
	1170	+	left -= int64(n)
	1171	+	if used == len(buf) && upload != nil {
	1172	+	if err := flush(); err != nil {
	1173	+	_ = r.Close()
	1174	+	return 0, "", err
	1175	+	}
	1176	+	}
	1177	+	}
	1178	+	var extra [1]byte
	1179	+	n, err := io.ReadFull(r, extra[:])
	1180	+	cerr := r.Close()
	1181	+	if n != 0 \|\| err != io.EOF {
	1182	+	return 0, "", fmt.Errorf("%w: artifact block size changed", domain.ErrInvalid)
	1183	+	}
	1184	+	if cerr != nil {
	1185	+	return 0, "", cerr
	1186	+	}
	1187	+	}
	1188	+	if size != total {
	1189	+	return 0, "", fmt.Errorf("%w: artifact block size changed", domain.ErrInvalid)
	1190	+	}
	1191	+	if upload == nil {
	1192	+	if _, err := c.cfg.Blobs.Put(ctx, a.BlobKey(), bytes.NewReader(buf[:used]), size); err != nil {
	1193	+	return 0, "", err
	1194	+	}
	1195	+	} else {
	1196	+	if used > 0 {
	1197	+	if err := flush(); err != nil {
	1198	+	return 0, "", err
	1199	+	}
	1200	+	}
	1201	+	if _, err := upload.Complete(ctx); err != nil {
	1202	+	return 0, "", err
	1203	+	}
	1204	+	completed = true
	1205	+	}
	1206	+	return size, "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
	1207	+	}
	1208	+
	1209	+	func (c *CI) sweepArtifactBlobOrphans(ctx context.Context) {
	1210	+	log := ctxLog(ctx)
	1211	+	cutoff := c.now().Add(-domain.CIArtifactPendingMax)
	1212	+	err := c.cfg.Blobs.List(ctx, "ci/", func(b ports.BlobInfo) error {
	1213	+	if !b.ModTime.Before(cutoff) {
	1214	+	return nil
	1215	+	}
	1216	+	run, id, block, ok := artifactBlobIDs(b.Key)
	1217	+	if !ok {
	1218	+	return nil
	1219	+	}
	1220	+	a, err := c.cfg.Artifacts.GetArtifact(ctx, id)
	1221	+	if err != nil && !errors.Is(err, domain.ErrNotFound) {
	1222	+	return nil
	1223	+	}
	1224	+	if err == nil && a.RunID == run && a.Store == domain.CIArtifactStoreBlob && (!block \|\| a.State == domain.CIArtifactPending) {
	1225	+	return nil
	1226	+	}
	1227	+	if err := c.cfg.Blobs.Delete(ctx, b.Key); err != nil {
	1228	+	log.Error().Err(err).Msg("ci: remove orphan artifact blob")
	1229	+	}
	1230	+	return nil
	1231	+	})
	1232	+	if err != nil {
	1233	+	log.Error().Err(err).Msg("ci: list artifact blobs")
	1234	+	}
	1235	+	err = c.cfg.Blobs.ListMultipart(ctx, "ci/", func(m ports.MultipartInfo) error {
	1236	+	if m.Started.Before(cutoff) {
	1237	+	if err := c.cfg.Blobs.AbortMultipart(ctx, m.Key, m.ID); err != nil {
	1238	+	log.Error().Err(err).Msg("ci: abort abandoned artifact upload")
	1239	+	}
	1240	+	}
	1241	+	return nil
	1242	+	})
	1243	+	if err != nil {
	1244	+	log.Error().Err(err).Msg("ci: list artifact multipart uploads")
	1245	+	}
	1246	+	}
	1247	+
	1248	+	func artifactBlobIDs(key string) (run, id int64, block, ok bool) {
	1249	+	rest, found := strings.CutPrefix(key, "ci/")
	1250	+	if !found {
	1251	+	return
	1252	+	}
	1253	+	r, tail, found := strings.Cut(rest, "/")
	1254	+	if !found \|\| len(r) != 20 {
	1255	+	return
	1256	+	}
	1257	+	name, file, block := strings.Cut(tail, ".blocks/")
	1258	+	if len(name) != 20 \|\| (block && file == "") {
	1259	+	return 0, 0, false, false
	1260	+	}
	1261	+	run, err := strconv.ParseInt(r, 10, 64)
	1262	+	if err != nil \|\| run <= 0 {
	1263	+	return 0, 0, false, false
	1264	+	}
	1265	+	id, err = strconv.ParseInt(name, 10, 64)
	1266	+	return run, id, block, err == nil && id > 0
	1267	+	}
	1268	+
	1269	+	// removeArtifactRuns follows repository deletion, when records are already
	1270	+	// gone. Run ids are global, so their blob prefixes cannot name another repo.
	1271	+	func (c *CI) removeArtifactRuns(ctx context.Context, runIDs []int64) error {
	1272	+	var errs []error
	1273	+	if c.cfg.ArtifactFiles != nil {
	1274	+	errs = append(errs, c.cfg.ArtifactFiles.RemoveRuns(ctx, runIDs))
	1275	+	}
	1276	+	if c.cfg.Blobs != nil {
	1277	+	for _, run := range runIDs {
	1278	+	if run <= 0 {
	1279	+	continue
	1280	+	}
	1281	+	prefix := fmt.Sprintf("ci/%020d/", run)
	1282	+	errs = append(errs, c.cfg.Blobs.List(ctx, prefix, func(b ports.BlobInfo) error {
	1283	+	if err := c.cfg.Blobs.Delete(ctx, b.Key); err != nil {
	1284	+	errs = append(errs, err)
	1285	+	}
	1286	+	return nil
	1287	+	}))
	1288	+	errs = append(errs, c.cfg.Blobs.ListMultipart(ctx, prefix, func(m ports.MultipartInfo) error {
	1289	+	if err := c.cfg.Blobs.AbortMultipart(ctx, m.Key, m.ID); err != nil {
	1290	+	errs = append(errs, err)
	1291	+	}
	1292	+	return nil
	1293	+	}))
	1294	+	}
	1295	+	}
	1296	+	return errors.Join(errs...)
	1297	+	}


@@ -13,6 +13,7 @@ import (
13		"fmt"	13		"fmt"
14		"io"	14		"io"
15		"strconv"	15		"strconv"
			16	+	"strings"
16		"sync"	17		"sync"
17		"time"	18		"time"
18			19

@@ -25,7 +26,7 @@ import (
25		// The artifacts of the runs (artifacts v4, CI-4 D3). The runner translates the	26		// The artifacts of the runs (artifacts v4, CI-4 D3). The runner translates the
26		// toolkit's protocol and calls JobArtifactService with the job token; owners	27		// toolkit's protocol and calls JobArtifactService with the job token; owners
27		// use ArtifactService. The records are in CIConfig.Artifacts, the contents in	28		// use ArtifactService. The records are in CIConfig.Artifacts, the contents in
28	-	// CIConfig.ArtifactFiles, named by ids only.	29	+	// CIConfig.Blobs (ArtifactFiles for legacy records), named by ids only.
29		//	30		//
30		// Every write of the artifacts of a repository goes through its usage record	31		// Every write of the artifacts of a repository goes through its usage record
31		// (ports.CIArtifactWrite): a decision is made on the records read after it and	32		// (ports.CIArtifactWrite): a decision is made on the records read after it and

@@ -57,7 +58,7 @@ var (
57		errArtifactChanged = errors.New("artifact changed")	58		errArtifactChanged = errors.New("artifact changed")
58		)	59		)
59			60
60	-	func (c *CI) artifactsOn() bool { return c.cfg.Artifacts != nil && c.cfg.ArtifactFiles != nil }	61	+	func (c *CI) artifactsOn() bool { return c.cfg.Artifacts != nil && c.cfg.Blobs != nil }
61			62
62		func artifactRef(a domain.CIArtifact) ports.CIArtifactRef {	63		func artifactRef(a domain.CIArtifact) ports.CIArtifactRef {
63		return ports.CIArtifactRef{RunID: a.RunID, ID: a.ID}	64		return ports.CIArtifactRef{RunID: a.RunID, ID: a.ID}

@@ -309,7 +310,13 @@ func (c *CI) removeArtifacts(ctx context.Context, repo domain.RepoName, runID in
309		// logged: the sweep removes the files that no record names.	310		// logged: the sweep removes the files that no record names.
310		func (c *CI) removeArtifactFiles(ctx context.Context, arts []domain.CIArtifact) {	311		func (c *CI) removeArtifactFiles(ctx context.Context, arts []domain.CIArtifact) {
311		for _, a := range arts {	312		for _, a := range arts {
312	-	if err := c.cfg.ArtifactFiles.Remove(ctx, artifactRef(a)); err != nil {	313	+	var err error
			314	+	if a.Store == domain.CIArtifactStoreBlob {
			315	+	err = errors.Join(c.cfg.Blobs.Delete(ctx, a.BlobKey()), c.removeBlobBlocks(ctx, a))
			316	+	} else if c.cfg.ArtifactFiles != nil {
			317	+	err = c.cfg.ArtifactFiles.Remove(ctx, artifactRef(a))
			318	+	}
			319	+	if err != nil {
313		ctxLog(ctx).Warn().Err(err).Int64("artifact_id", a.ID).Msg("ci: remove the files of an artifact; the sweep will")	320		ctxLog(ctx).Warn().Err(err).Int64("artifact_id", a.ID).Msg("ci: remove the files of an artifact; the sweep will")
314		}	321		}
315		}	322		}

@@ -373,7 +380,7 @@ func (c *CI) CreateJobArtifact(ctx context.Context, secret string, params ports.
373		}	380		}
374		a := domain.CIArtifact{	381		a := domain.CIArtifact{
375		ID: id, Repo: j.Repo, RunID: j.RunID, Attempt: j.RunAttempt, JobID: j.ID, Assignment: j.TokenHash,	382		ID: id, Repo: j.Repo, RunID: j.RunID, Attempt: j.RunAttempt, JobID: j.ID, Assignment: j.TokenHash,
376	-	Name: params.Name, MediaType: mediaType, State: domain.CIArtifactPending,	383	+	Name: params.Name, MediaType: mediaType, State: domain.CIArtifactPending, Store: domain.CIArtifactStoreBlob,
377		CreatedAt: now, ExpiresAt: domain.ArtifactExpiry(params.ExpiresAt, now, c.cfg.ArtifactLimits.Retention),	384		CreatedAt: now, ExpiresAt: domain.ArtifactExpiry(params.ExpiresAt, now, c.cfg.ArtifactLimits.Retention),
378		}	385		}
379		var discarded []domain.CIArtifact	386		var discarded []domain.CIArtifact

@@ -466,9 +473,12 @@ func (c *CI) UploadJobArtifactBlock(ctx context.Context, secret, jobID string, a
466		src := &reservingReader{ctx: ctx, c: c, a: a, r: data}	473		src := &reservingReader{ctx: ctx, c: c, a: a, r: data}
467		defer func() { c.release(a, src.reserved) }()	474		defer func() { c.release(a, src.reserved) }()
468		limits := c.cfg.ArtifactLimits	475		limits := c.cfg.ArtifactLimits
469	-	ref, file := artifactRef(a), newBlockFile(blockID)	476	+	file := newBlockFile(blockID)
470	-	size, err := c.cfg.ArtifactFiles.PutBlock(ctx, ref, file, src)	477	+	size, err := c.putUnsized(ctx, a.BlockKey(file), src)
471		if err != nil {	478		if err != nil {
			479	+	if derr := c.cfg.Blobs.Delete(context.WithoutCancel(ctx), a.BlockKey(file)); derr != nil {
			480	+	ctxLog(ctx).Warn().Err(derr).Msg("ci: remove a refused block; the sweep will")
			481	+	}
472		return 0, err	482		return 0, err
473		}	483		}
474		var old *ports.CIArtifactBlock	484		var old *ports.CIArtifactBlock

@@ -509,13 +519,13 @@ func (c *CI) UploadJobArtifactBlock(ctx context.Context, secret, jobID string, a
509		return nil	519		return nil
510		})	520		})
511		if err != nil {	521		if err != nil {
512	-	if rerr := c.cfg.ArtifactFiles.RemoveBlock(ctx, ref, file); rerr != nil {	522	+	if rerr := c.cfg.Blobs.Delete(context.WithoutCancel(ctx), a.BlockKey(file)); rerr != nil {
513		ctxLog(ctx).Warn().Err(rerr).Int64("artifact_id", a.ID).Msg("ci: remove a refused block; the sweep will")	523		ctxLog(ctx).Warn().Err(rerr).Int64("artifact_id", a.ID).Msg("ci: remove a refused block; the sweep will")
514		}	524		}
515		return 0, err	525		return 0, err
516		}	526		}
517		if old != nil {	527		if old != nil {
518	-	if err := c.cfg.ArtifactFiles.RemoveBlock(ctx, ref, old.File); err != nil {	528	+	if err := c.cfg.Blobs.Delete(ctx, a.BlockKey(old.File)); err != nil {
519		ctxLog(ctx).Warn().Err(err).Int64("artifact_id", a.ID).Msg("ci: remove a replaced block")	529		ctxLog(ctx).Warn().Err(err).Int64("artifact_id", a.ID).Msg("ci: remove a replaced block")
520		}	530		}
521		}	531		}

@@ -602,7 +612,7 @@ func (c *CI) CommitJobArtifact(ctx context.Context, secret, jobID string, artifa
602		return 0, "", err	612		return 0, "", err
603		}	613		}
604		}	614		}
605	-	size, digest, err := c.cfg.ArtifactFiles.Assemble(ctx, artifactRef(a), files)	615	+	size, digest, err := c.assembleBlobs(ctx, a, files, total)
606		if err != nil {	616		if err != nil {
607		c.release(a, excess)	617		c.release(a, excess)
608		}	618		}

@@ -638,7 +648,7 @@ func (c *CI) CommitJobArtifact(ctx context.Context, secret, jobID string, artifa
638		if err != nil {	648		if err != nil {
639		return 0, "", err	649		return 0, "", err
640		}	650		}
641	-	if err := c.cfg.ArtifactFiles.RemoveBlocks(ctx, artifactRef(a)); err != nil {	651	+	if err := c.removeBlobBlocks(ctx, a); err != nil {
642		ctxLog(ctx).Warn().Err(err).Int64("artifact_id", a.ID).Msg("ci: remove the blocks of a committed artifact")	652		ctxLog(ctx).Warn().Err(err).Int64("artifact_id", a.ID).Msg("ci: remove the blocks of a committed artifact")
643		}	653		}
644		c.dropIfEnded(ctx, j)	654		c.dropIfEnded(ctx, j)

@@ -782,7 +792,23 @@ func (c *CI) streamArtifact(ctx context.Context, a domain.CIArtifact, offset int
782		if offset < 0 \|\| offset > a.Size {	792		if offset < 0 \|\| offset > a.Size {
783		return fmt.Errorf("%w: offset %d is beyond the size %d", domain.ErrOutOfRange, offset, a.Size)	793		return fmt.Errorf("%w: offset %d is beyond the size %d", domain.ErrOutOfRange, offset, a.Size)
784		}	794		}
785	-	f, err := c.cfg.ArtifactFiles.Open(ctx, artifactRef(a), offset)	795	+	var f io.ReadCloser
			796	+	var err error
			797	+	if a.Store == domain.CIArtifactStoreBlob {
			798	+	// BlobStore rejects offsets at EOF, while artifact downloads permit
			799	+	// an empty resumed stream (including zero-byte artifacts).
			800	+	if offset == a.Size {
			801	+	_, err = c.cfg.Blobs.Stat(ctx, a.BlobKey())
			802	+	f = io.NopCloser(bytes.NewReader(nil))
			803	+	} else {
			804	+	f, _, err = c.cfg.Blobs.Open(ctx, a.BlobKey(), offset, -1)
			805	+	}
			806	+	} else if c.cfg.ArtifactFiles != nil {
			807	+	f, err = c.cfg.ArtifactFiles.Open(ctx, artifactRef(a), offset)
			808	+	} else {
			809	+	ctxLog(ctx).Error().Int64("artifact_id", a.ID).Msg("ci: legacy artifact reader is unavailable")
			810	+	return domain.ErrNotFound
			811	+	}
786		if err != nil {	812		if err != nil {
787		return err	813		return err
788		}	814		}

@@ -1008,6 +1034,10 @@ func (c *CI) sweepArtifactOrphans(ctx context.Context) {
1008		}	1034		}
1009		}	1035		}
1010		}	1036		}
			1037	+	c.sweepArtifactBlobOrphans(ctx)
			1038	+	if c.cfg.ArtifactFiles == nil {
			1039	+	return
			1040	+	}
1011		c.removeOrphans(ctx, "artifacts", "artifact directories", c.cfg.ArtifactFiles.RunIDs, c.cfg.ArtifactFiles.RemoveRuns)	1041		c.removeOrphans(ctx, "artifacts", "artifact directories", c.cfg.ArtifactFiles.RunIDs, c.cfg.ArtifactFiles.RemoveRuns)
1012		runIDs, err := c.cfg.ArtifactFiles.RunIDs(ctx)	1042		runIDs, err := c.cfg.ArtifactFiles.RunIDs(ctx)
1013		if err != nil {	1043		if err != nil {

@@ -1022,7 +1052,7 @@ func (c *CI) sweepArtifactOrphans(ctx context.Context) {
1022		}	1052		}
1023		for _, id := range ids {	1053		for _, id := range ids {
1024		a, err := c.cfg.Artifacts.GetArtifact(ctx, id)	1054		a, err := c.cfg.Artifacts.GetArtifact(ctx, id)
1025	-	if err == nil && a.RunID == run {	1055	+	if err == nil && a.RunID == run && a.Store != domain.CIArtifactStoreBlob {
1026		continue	1056		continue
1027		}	1057		}
1028		if err != nil && !errors.Is(err, domain.ErrNotFound) {	1058		if err != nil && !errors.Is(err, domain.ErrNotFound) {

@@ -1035,3 +1065,233 @@ func (c *CI) sweepArtifactOrphans(ctx context.Context) {
1035		}	1065		}
1036		}	1066		}
1037		}	1067		}
			1068	+
			1069	+	// putUnsized buffers at most one toolkit block; Put requires an exact size.
			1070	+	func (c *CI) putUnsized(ctx context.Context, key string, r io.Reader) (int64, error) {
			1071	+	// A fixed backing array prevents bytes.Buffer's geometric growth from
			1072	+	// allocating beyond the block bound. Probe the next byte separately.
			1073	+	buf := make([]byte, domain.CIArtifactMaxBlockSize)
			1074	+	n, err := io.ReadFull(r, buf)
			1075	+	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
			1076	+	return 0, err
			1077	+	}
			1078	+	if n == len(buf) {
			1079	+	var extra [1]byte
			1080	+	m, err := io.ReadFull(r, extra[:])
			1081	+	if m != 0 {
			1082	+	return 0, fmt.Errorf("%w: a block has at most 8 MiB", domain.ErrInvalid)
			1083	+	}
			1084	+	if err != io.EOF {
			1085	+	return 0, err
			1086	+	}
			1087	+	}
			1088	+	if err := ctx.Err(); err != nil {
			1089	+	return 0, err
			1090	+	}
			1091	+	_, err = c.cfg.Blobs.Put(ctx, key, bytes.NewReader(buf[:n]), int64(n))
			1092	+	return int64(n), err
			1093	+	}
			1094	+
			1095	+	func (c *CI) removeBlobBlocks(ctx context.Context, a domain.CIArtifact) error {
			1096	+	return c.cfg.Blobs.List(ctx, a.BlobKey()+".blocks/", func(b ports.BlobInfo) error {
			1097	+	return c.cfg.Blobs.Delete(ctx, b.Key)
			1098	+	})
			1099	+	}
			1100	+
			1101	+	// assembleBlobs keeps one 8 MiB part in memory, even when a block crosses
			1102	+	// part boundaries or is repeated in the commit list.
			1103	+	func (c *CI) assembleBlobs(ctx context.Context, a domain.CIArtifact, files []string, total int64) (int64, string, error) {
			1104	+	var upload ports.MultipartUpload
			1105	+	if total > ports.BlobPartSize {
			1106	+	var err error
			1107	+	upload, err = c.cfg.Blobs.NewMultipart(ctx, a.BlobKey())
			1108	+	if err != nil {
			1109	+	return 0, "", err
			1110	+	}
			1111	+	}
			1112	+	completed := false
			1113	+	defer func() {
			1114	+	if upload != nil && !completed {
			1115	+	if err := upload.Abort(context.WithoutCancel(ctx)); err != nil {
			1116	+	ctxLog(ctx).Warn().Err(err).Msg("ci: abort artifact assembly; the sweep will")
			1117	+	}
			1118	+	}
			1119	+	}()
			1120	+	buf := make([]byte, ports.BlobPartSize)
			1121	+	used, part := 0, 1
			1122	+	var size int64
			1123	+	h := sha256.New()
			1124	+	flush := func() error {
			1125	+	if err := upload.UploadPart(ctx, part, bytes.NewReader(buf[:used]), int64(used)); err != nil {
			1126	+	return err
			1127	+	}
			1128	+	used = 0
			1129	+	part++
			1130	+	return nil
			1131	+	}
			1132	+	for _, file := range files {
			1133	+	r, info, err := c.cfg.Blobs.Open(ctx, a.BlockKey(file), 0, -1)
			1134	+	// Empty blocks cannot be opened under BlobStore's EOF contract.
			1135	+	if errors.Is(err, domain.ErrOutOfRange) {
			1136	+	info, err = c.cfg.Blobs.Stat(ctx, a.BlockKey(file))
			1137	+	if err == nil && info.Size == 0 {
			1138	+	continue
			1139	+	}
			1140	+	if err == nil {
			1141	+	return 0, "", domain.ErrOutOfRange
			1142	+	}
			1143	+	}
			1144	+	if err != nil {
			1145	+	return 0, "", err
			1146	+	}
			1147	+	if info.Size > total-size {
			1148	+	_ = r.Close()
			1149	+	return 0, "", fmt.Errorf("%w: artifact block size changed", domain.ErrInvalid)
			1150	+	}
			1151	+	left := info.Size
			1152	+	for left > 0 {
			1153	+	if err := ctx.Err(); err != nil {
			1154	+	_ = r.Close()
			1155	+	return 0, "", err
			1156	+	}
			1157	+	want := min(left, int64(len(buf)-used))
			1158	+	if want == 0 {
			1159	+	_ = r.Close()
			1160	+	return 0, "", fmt.Errorf("%w: artifact block size changed", domain.ErrInvalid)
			1161	+	}
			1162	+	n, err := io.ReadFull(r, buf[used:used+int(want)])
			1163	+	if err != nil {
			1164	+	_ = r.Close()
			1165	+	return 0, "", err
			1166	+	}
			1167	+	h.Write(buf[used : used+n])
			1168	+	used += n
			1169	+	size += int64(n)
			1170	+	left -= int64(n)
			1171	+	if used == len(buf) && upload != nil {
			1172	+	if err := flush(); err != nil {
			1173	+	_ = r.Close()
			1174	+	return 0, "", err
			1175	+	}
			1176	+	}
			1177	+	}
			1178	+	var extra [1]byte
			1179	+	n, err := io.ReadFull(r, extra[:])
			1180	+	cerr := r.Close()
			1181	+	if n != 0 \|\| err != io.EOF {
			1182	+	return 0, "", fmt.Errorf("%w: artifact block size changed", domain.ErrInvalid)
			1183	+	}
			1184	+	if cerr != nil {
			1185	+	return 0, "", cerr
			1186	+	}
			1187	+	}
			1188	+	if size != total {
			1189	+	return 0, "", fmt.Errorf("%w: artifact block size changed", domain.ErrInvalid)
			1190	+	}
			1191	+	if upload == nil {
			1192	+	if _, err := c.cfg.Blobs.Put(ctx, a.BlobKey(), bytes.NewReader(buf[:used]), size); err != nil {
			1193	+	return 0, "", err
			1194	+	}
			1195	+	} else {
			1196	+	if used > 0 {
			1197	+	if err := flush(); err != nil {
			1198	+	return 0, "", err
			1199	+	}
			1200	+	}
			1201	+	if _, err := upload.Complete(ctx); err != nil {
			1202	+	return 0, "", err
			1203	+	}
			1204	+	completed = true
			1205	+	}
			1206	+	return size, "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
			1207	+	}
			1208	+
			1209	+	func (c *CI) sweepArtifactBlobOrphans(ctx context.Context) {
			1210	+	log := ctxLog(ctx)
			1211	+	cutoff := c.now().Add(-domain.CIArtifactPendingMax)
			1212	+	err := c.cfg.Blobs.List(ctx, "ci/", func(b ports.BlobInfo) error {
			1213	+	if !b.ModTime.Before(cutoff) {
			1214	+	return nil
			1215	+	}
			1216	+	run, id, block, ok := artifactBlobIDs(b.Key)
			1217	+	if !ok {
			1218	+	return nil
			1219	+	}
			1220	+	a, err := c.cfg.Artifacts.GetArtifact(ctx, id)
			1221	+	if err != nil && !errors.Is(err, domain.ErrNotFound) {
			1222	+	return nil
			1223	+	}
			1224	+	if err == nil && a.RunID == run && a.Store == domain.CIArtifactStoreBlob && (!block \|\| a.State == domain.CIArtifactPending) {
			1225	+	return nil
			1226	+	}
			1227	+	if err := c.cfg.Blobs.Delete(ctx, b.Key); err != nil {
			1228	+	log.Error().Err(err).Msg("ci: remove orphan artifact blob")
			1229	+	}
			1230	+	return nil
			1231	+	})
			1232	+	if err != nil {
			1233	+	log.Error().Err(err).Msg("ci: list artifact blobs")
			1234	+	}
			1235	+	err = c.cfg.Blobs.ListMultipart(ctx, "ci/", func(m ports.MultipartInfo) error {
			1236	+	if m.Started.Before(cutoff) {
			1237	+	if err := c.cfg.Blobs.AbortMultipart(ctx, m.Key, m.ID); err != nil {
			1238	+	log.Error().Err(err).Msg("ci: abort abandoned artifact upload")
			1239	+	}
			1240	+	}
			1241	+	return nil
			1242	+	})
			1243	+	if err != nil {
			1244	+	log.Error().Err(err).Msg("ci: list artifact multipart uploads")
			1245	+	}
			1246	+	}
			1247	+
			1248	+	func artifactBlobIDs(key string) (run, id int64, block, ok bool) {
			1249	+	rest, found := strings.CutPrefix(key, "ci/")
			1250	+	if !found {
			1251	+	return
			1252	+	}
			1253	+	r, tail, found := strings.Cut(rest, "/")
			1254	+	if !found \|\| len(r) != 20 {
			1255	+	return
			1256	+	}
			1257	+	name, file, block := strings.Cut(tail, ".blocks/")
			1258	+	if len(name) != 20 \|\| (block && file == "") {
			1259	+	return 0, 0, false, false
			1260	+	}
			1261	+	run, err := strconv.ParseInt(r, 10, 64)
			1262	+	if err != nil \|\| run <= 0 {
			1263	+	return 0, 0, false, false
			1264	+	}
			1265	+	id, err = strconv.ParseInt(name, 10, 64)
			1266	+	return run, id, block, err == nil && id > 0
			1267	+	}
			1268	+
			1269	+	// removeArtifactRuns follows repository deletion, when records are already
			1270	+	// gone. Run ids are global, so their blob prefixes cannot name another repo.
			1271	+	func (c *CI) removeArtifactRuns(ctx context.Context, runIDs []int64) error {
			1272	+	var errs []error
			1273	+	if c.cfg.ArtifactFiles != nil {
			1274	+	errs = append(errs, c.cfg.ArtifactFiles.RemoveRuns(ctx, runIDs))
			1275	+	}
			1276	+	if c.cfg.Blobs != nil {
			1277	+	for _, run := range runIDs {
			1278	+	if run <= 0 {
			1279	+	continue
			1280	+	}
			1281	+	prefix := fmt.Sprintf("ci/%020d/", run)
			1282	+	errs = append(errs, c.cfg.Blobs.List(ctx, prefix, func(b ports.BlobInfo) error {
			1283	+	if err := c.cfg.Blobs.Delete(ctx, b.Key); err != nil {
			1284	+	errs = append(errs, err)
			1285	+	}
			1286	+	return nil
			1287	+	}))
			1288	+	errs = append(errs, c.cfg.Blobs.ListMultipart(ctx, prefix, func(m ports.MultipartInfo) error {
			1289	+	if err := c.cfg.Blobs.AbortMultipart(ctx, m.Key, m.ID); err != nil {
			1290	+	errs = append(errs, err)
			1291	+	}
			1292	+	return nil
			1293	+	}))
			1294	+	}
			1295	+	}
			1296	+	return errors.Join(errs...)
			1297	+	}

A internal/usecase/ci_artifacts_blob_test.go +301 -0

@@ -0,0 +1,301 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"bytes"
	7	+	"context"
	8	+	"crypto/sha256"
	9	+	"encoding/hex"
	10	+	"errors"
	11	+	"io"
	12	+	"strconv"
	13	+	"strings"
	14	+	"testing"
	15	+	"time"
	16	+
	17	+	"github.com/stretchr/testify/assert"
	18	+	"github.com/stretchr/testify/mock"
	19	+	"github.com/stretchr/testify/require"
	20	+
	21	+	"bnema.dev/ermite/internal/domain"
	22	+	"bnema.dev/ermite/internal/ports"
	23	+	"bnema.dev/ermite/internal/ports/mocks"
	24	+	)
	25	+
	26	+	// artifactBlobMock uses generated port mocks with byte storage in callbacks.
	27	+	// Real-disk end-to-end behavior is exercised by internal/app's artifact rig.
	28	+	func artifactBlobMock(t testing.T, failPart bool) (mocks.BlobStore, int, int) {
	29	+	t.Helper()
	30	+	b := mocks.NewBlobStore(t)
	31	+	objects := map[string][]byte{}
	32	+	active := map[string]ports.MultipartInfo{}
	33	+	puts, multiparts := new(int), new(int)
	34	+	b.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key string, r io.Reader, size int64) (ports.BlobInfo, error) {
	35	+	*puts++
	36	+	data, err := io.ReadAll(r)
	37	+	if err != nil {
	38	+	return ports.BlobInfo{}, err
	39	+	}
	40	+	if int64(len(data)) != size {
	41	+	return ports.BlobInfo{}, domain.ErrInvalid
	42	+	}
	43	+	objects[key] = data
	44	+	return ports.BlobInfo{Key: key, Size: size}, nil
	45	+	}).Maybe()
	46	+	b.EXPECT().Open(mock.Anything, mock.Anything, mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key string, offset, length int64) (io.ReadCloser, ports.BlobInfo, error) {
	47	+	data, ok := objects[key]
	48	+	if !ok {
	49	+	return nil, ports.BlobInfo{}, domain.ErrNotFound
	50	+	}
	51	+	info := ports.BlobInfo{Key: key, Size: int64(len(data))}
	52	+	if offset >= info.Size {
	53	+	return nil, info, domain.ErrOutOfRange
	54	+	}
	55	+	end := info.Size
	56	+	if length >= 0 {
	57	+	end = min(end, offset+length)
	58	+	}
	59	+	return io.NopCloser(bytes.NewReader(data[offset:end])), info, nil
	60	+	}).Maybe()
	61	+	b.EXPECT().Stat(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key string) (ports.BlobInfo, error) {
	62	+	data, ok := objects[key]
	63	+	if !ok {
	64	+	return ports.BlobInfo{}, domain.ErrNotFound
	65	+	}
	66	+	return ports.BlobInfo{Key: key, Size: int64(len(data))}, nil
	67	+	}).Maybe()
	68	+	b.EXPECT().Delete(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key string) error { delete(objects, key); return nil }).Maybe()
	69	+	b.EXPECT().List(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, prefix string, fn func(ports.BlobInfo) error) error {
	70	+	for key, data := range objects {
	71	+	if strings.HasPrefix(key, prefix) {
	72	+	if err := fn(ports.BlobInfo{Key: key, Size: int64(len(data))}); err != nil {
	73	+	return err
	74	+	}
	75	+	}
	76	+	}
	77	+	return nil
	78	+	}).Maybe()
	79	+	b.EXPECT().ListMultipart(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, prefix string, fn func(ports.MultipartInfo) error) error {
	80	+	for key, m := range active {
	81	+	if strings.HasPrefix(key, prefix) {
	82	+	if err := fn(m); err != nil {
	83	+	return err
	84	+	}
	85	+	}
	86	+	}
	87	+	return nil
	88	+	}).Maybe()
	89	+	b.EXPECT().AbortMultipart(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key, id string) error { delete(active, key); return nil }).Maybe()
	90	+	b.EXPECT().NewMultipart(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key string) (ports.MultipartUpload, error) {
	91	+	*multiparts++
	92	+	active[key] = ports.MultipartInfo{Key: key, ID: "upload"}
	93	+	u := mocks.NewMultipartUpload(t)
	94	+	var data []byte
	95	+	u.EXPECT().UploadPart(mock.Anything, mock.Anything, mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, _ int, r io.Reader, size int64) error {
	96	+	if failPart {
	97	+	return errors.New("part failed")
	98	+	}
	99	+	part, err := io.ReadAll(r)
	100	+	data = append(data, part...)
	101	+	return err
	102	+	}).Maybe()
	103	+	u.EXPECT().Complete(mock.Anything).RunAndReturn(func(context.Context) (ports.BlobInfo, error) {
	104	+	objects[key] = data
	105	+	delete(active, key)
	106	+	return ports.BlobInfo{Key: key, Size: int64(len(data))}, nil
	107	+	}).Maybe()
	108	+	u.EXPECT().Abort(mock.Anything).RunAndReturn(func(context.Context) error { delete(active, key); return nil }).Maybe()
	109	+	return u, nil
	110	+	}).Maybe()
	111	+	return b, puts, multiparts
	112	+	}
	113	+
	114	+	func TestArtifactBlobAssembly(t *testing.T) {
	115	+	for _, size := range []int{0, 1, ports.BlobPartSize, ports.BlobPartSize + 1, 2*ports.BlobPartSize + 17} {
	116	+	t.Run(strconv.Itoa(size), func(t *testing.T) {
	117	+	ctx := t.Context()
	118	+	b, puts, multiparts := artifactBlobMock(t, false)
	119	+	c := &CI{cfg: CIConfig{Blobs: b}}
	120	+	a := domain.CIArtifact{RunID: 1, ID: 2, Store: domain.CIArtifactStoreBlob}
	121	+	data := bytes.Repeat([]byte("x"), size)
	122	+	// Parts straddle block boundaries, with an empty block in the list.
	123	+	cut := min(size, 3<<20)
	124	+	_, err := b.Put(ctx, a.BlockKey("a"), bytes.NewReader(data[:cut]), int64(cut))
	125	+	require.NoError(t, err)
	126	+	_, err = b.Put(ctx, a.BlockKey("b"), bytes.NewReader(data[cut:]), int64(size-cut))
	127	+	require.NoError(t, err)
	128	+	_, err = b.Put(ctx, a.BlockKey("empty"), bytes.NewReader(nil), 0)
	129	+	require.NoError(t, err)
	130	+	*puts = 0
	131	+	n, digest, err := c.assembleBlobs(ctx, a, []string{"a", "empty", "b"}, int64(size))
	132	+	require.NoError(t, err)
	133	+	assert.EqualValues(t, size, n)
	134	+	sum := sha256.Sum256(data)
	135	+	assert.Equal(t, "sha256:"+hex.EncodeToString(sum[:]), digest)
	136	+	if size <= ports.BlobPartSize {
	137	+	assert.Equal(t, 1, *puts)
	138	+	assert.Zero(t, *multiparts)
	139	+	} else {
	140	+	assert.Zero(t, *puts)
	141	+	assert.Equal(t, 1, *multiparts)
	142	+	}
	143	+	if size > 0 {
	144	+	r, _, err := b.Open(ctx, a.BlobKey(), 0, -1)
	145	+	require.NoError(t, err)
	146	+	got, err := io.ReadAll(r)
	147	+	require.NoError(t, err)
	148	+	require.NoError(t, r.Close())
	149	+	assert.Equal(t, data, got)
	150	+	} else {
	151	+	info, err := b.Stat(ctx, a.BlobKey())
	152	+	require.NoError(t, err)
	153	+	assert.Zero(t, info.Size)
	154	+	}
	155	+	// No legacy reader is needed for a blob download, including EOF resumes.
	156	+	a.Size = int64(size)
	157	+	var got []byte
	158	+	require.NoError(t, c.streamArtifact(ctx, a, 0, func(ch ports.ArtifactChunk) error { got = append(got, ch.Data...); return nil }))
	159	+	assert.Equal(t, data, append([]byte{}, got...))
	160	+	require.NoError(t, c.streamArtifact(ctx, a, a.Size, func(ports.ArtifactChunk) error { return nil }))
	161	+	c.removeArtifactFiles(ctx, []domain.CIArtifact{a})
	162	+	_, err = b.Stat(ctx, a.BlobKey())
	163	+	assert.ErrorIs(t, err, domain.ErrNotFound)
	164	+	require.NoError(t, b.List(ctx, a.BlobKey()+".blocks/", func(ports.BlobInfo) error { t.Error("block survived deletion"); return nil }))
	165	+	})
	166	+	}
	167	+	}
	168	+
	169	+	func TestArtifactBlobAssemblyFailureAborts(t *testing.T) {
	170	+	for _, failPart := range []bool{false, true} {
	171	+	t.Run(map[bool]string{false: "missing block", true: "failed part"}[failPart], func(t *testing.T) {
	172	+	b, _, multiparts := artifactBlobMock(t, failPart)
	173	+	c := &CI{cfg: CIConfig{Blobs: b}}
	174	+	a := domain.CIArtifact{RunID: 1, ID: 2}
	175	+	data := bytes.Repeat([]byte("a"), ports.BlobPartSize)
	176	+	_, err := b.Put(t.Context(), a.BlockKey("a"), bytes.NewReader(data), int64(len(data)))
	177	+	require.NoError(t, err)
	178	+	_, _, err = c.assembleBlobs(t.Context(), a, []string{"a", "missing"}, ports.BlobPartSize+1)
	179	+	require.Error(t, err)
	180	+	assert.Equal(t, 1, *multiparts)
	181	+	_, err = b.Stat(t.Context(), a.BlobKey())
	182	+	assert.ErrorIs(t, err, domain.ErrNotFound)
	183	+	require.NoError(t, b.ListMultipart(t.Context(), "ci/", func(ports.MultipartInfo) error { t.Error("multipart survived failure"); return nil }))
	184	+	})
	185	+	}
	186	+	}
	187	+
	188	+	func TestArtifactPutUnsizedBound(t *testing.T) {
	189	+	b, _, _ := artifactBlobMock(t, false)
	190	+	c := &CI{cfg: CIConfig{Blobs: b}}
	191	+	for _, size := range []int{0, 1, domain.CIArtifactMaxBlockSize, domain.CIArtifactMaxBlockSize + 1} {
	192	+	n, err := c.putUnsized(t.Context(), "ci/1/2", bytes.NewReader(make([]byte, size)))
	193	+	if size > domain.CIArtifactMaxBlockSize {
	194	+	assert.ErrorIs(t, err, domain.ErrInvalid)
	195	+	assert.Contains(t, err.Error(), "a block has at most 8 MiB")
	196	+	} else {
	197	+	require.NoError(t, err)
	198	+	assert.EqualValues(t, size, n)
	199	+	}
	200	+	}
	201	+	ctx, cancel := context.WithCancel(t.Context())
	202	+	cancel()
	203	+	_, err := c.putUnsized(ctx, "ci/1/cancelled", bytes.NewReader([]byte("x")))
	204	+	assert.ErrorIs(t, err, context.Canceled)
	205	+	_, err = b.Stat(t.Context(), "ci/1/cancelled")
	206	+	assert.ErrorIs(t, err, domain.ErrNotFound)
	207	+	}
	208	+
	209	+	func TestArtifactLegacyStream(t *testing.T) {
	210	+	a := domain.CIArtifact{RunID: 1, ID: 2, Size: 6}
	211	+	legacy := mocks.NewCIArtifactFiles(t)
	212	+	legacy.EXPECT().Open(mock.Anything, artifactRef(a), int64(2)).RunAndReturn(func(context.Context, ports.CIArtifactRef, int64) (io.ReadCloser, error) {
	213	+	return io.NopCloser(bytes.NewReader([]byte("gacy"))), nil
	214	+	}).Twice()
	215	+	c := &CI{cfg: CIConfig{ArtifactFiles: legacy}}
	216	+	for _, store := range []domain.CIArtifactStore{domain.CIArtifactStoreLegacy, "disk"} {
	217	+	a.Store = store
	218	+	var got []byte
	219	+	require.NoError(t, c.streamArtifact(t.Context(), a, 2, func(ch ports.ArtifactChunk) error { got = append(got, ch.Data...); return nil }))
	220	+	assert.Equal(t, "gacy", string(got))
	221	+	}
	222	+	c.cfg.ArtifactFiles = nil
	223	+	assert.ErrorIs(t, c.streamArtifact(t.Context(), a, 0, func(ports.ArtifactChunk) error { return nil }), domain.ErrNotFound)
	224	+	}
	225	+
	226	+	func TestArtifactBlobOrphanSweep(t *testing.T) {
	227	+	ctx := t.Context()
	228	+	b := mocks.NewBlobStore(t)
	229	+	s := mocks.NewCIArtifactStore(t)
	230	+	c := &CI{cfg: CIConfig{Blobs: b, Artifacts: s, Clock: stillClock{}}}
	231	+	now := c.now()
	232	+	old := now.Add(-25 * time.Hour)
	233	+	pending := domain.CIArtifact{RunID: 1, ID: 2, Store: domain.CIArtifactStoreBlob, State: domain.CIArtifactPending}
	234	+	committed := pending
	235	+	committed.ID = 3
	236	+	committed.State = domain.CIArtifactCommitted
	237	+	missing := pending
	238	+	missing.ID = 4
	239	+	b.EXPECT().List(mock.Anything, "ci/", mock.Anything).RunAndReturn(func(_ context.Context, _ string, fn func(ports.BlobInfo) error) error {
	240	+	for _, info := range []ports.BlobInfo{
	241	+	{Key: missing.BlobKey(), ModTime: now}, // young: no lookup, no deletion
	242	+	{Key: missing.BlobKey(), ModTime: old},
	243	+	{Key: pending.BlockKey("a"), ModTime: old},
	244	+	{Key: committed.BlockKey("a"), ModTime: old},
	245	+	{Key: committed.BlobKey(), ModTime: old},
	246	+	} {
	247	+	require.NoError(t, fn(info))
	248	+	}
	249	+	return nil
	250	+	})
	251	+	s.EXPECT().GetArtifact(mock.Anything, missing.ID).Return(domain.CIArtifact{}, domain.ErrNotFound).Once()
	252	+	s.EXPECT().GetArtifact(mock.Anything, pending.ID).Return(pending, nil).Once()
	253	+	s.EXPECT().GetArtifact(mock.Anything, committed.ID).Return(committed, nil).Twice()
	254	+	b.EXPECT().Delete(mock.Anything, missing.BlobKey()).Return(nil).Once()
	255	+	b.EXPECT().Delete(mock.Anything, committed.BlockKey("a")).Return(nil).Once()
	256	+	b.EXPECT().ListMultipart(mock.Anything, "ci/", mock.Anything).RunAndReturn(func(_ context.Context, _ string, fn func(ports.MultipartInfo) error) error {
	257	+	require.NoError(t, fn(ports.MultipartInfo{Key: missing.BlobKey(), ID: "old", Started: old}))
	258	+	require.NoError(t, fn(ports.MultipartInfo{Key: missing.BlobKey(), ID: "young", Started: now}))
	259	+	return nil
	260	+	})
	261	+	b.EXPECT().AbortMultipart(mock.Anything, missing.BlobKey(), "old").Return(nil).Once()
	262	+	c.sweepArtifactBlobOrphans(ctx)
	263	+	}
	264	+
	265	+	func TestArtifactRemoveRunsWithoutLegacyReader(t *testing.T) {
	266	+	ctx := t.Context()
	267	+	b, _, _ := artifactBlobMock(t, false)
	268	+	c := &CI{cfg: CIConfig{Blobs: b}}
	269	+	a := domain.CIArtifact{RunID: 1, ID: 2}
	270	+	other := a
	271	+	other.RunID = 2
	272	+	for _, key := range []string{a.BlobKey(), a.BlockKey("a"), other.BlobKey()} {
	273	+	_, err := b.Put(ctx, key, bytes.NewReader([]byte("x")), 1)
	274	+	require.NoError(t, err)
	275	+	}
	276	+	_, err := b.NewMultipart(ctx, a.BlobKey())
	277	+	require.NoError(t, err)
	278	+	require.NoError(t, c.removeArtifactRuns(ctx, []int64{1}))
	279	+	_, err = b.Stat(ctx, a.BlobKey())
	280	+	assert.ErrorIs(t, err, domain.ErrNotFound)
	281	+	_, err = b.Stat(ctx, a.BlockKey("a"))
	282	+	assert.ErrorIs(t, err, domain.ErrNotFound)
	283	+	_, err = b.Stat(ctx, other.BlobKey())
	284	+	require.NoError(t, err)
	285	+	require.NoError(t, b.ListMultipart(ctx, "ci/", func(ports.MultipartInfo) error { t.Error("upload survived run deletion"); return nil }))
	286	+	}
	287	+
	288	+	func TestArtifactBlobIDs(t *testing.T) {
	289	+	a := domain.CIArtifact{RunID: 7, ID: 42}
	290	+	for _, key := range []string{a.BlobKey(), a.BlockKey("abcdef.0123")} {
	291	+	run, id, block, ok := artifactBlobIDs(key)
	292	+	assert.True(t, ok)
	293	+	assert.EqualValues(t, 7, run)
	294	+	assert.EqualValues(t, 42, id)
	295	+	assert.Equal(t, strings.Contains(key, ".blocks/"), block)
	296	+	}
	297	+	for _, key := range []string{"release/1", "ci/1/2", a.BlobKey() + ".blocks/", a.BlobKey() + "/wrong"} {
	298	+	_, _, _, ok := artifactBlobIDs(key)
	299	+	assert.False(t, ok, key)
	300	+	}
	301	+	}

@@ -0,0 +1,301 @@
	1	+	// SPDX-License-Identifier: AGPL-3.0-or-later
	2	+
	3	+	package usecase
	4	+
	5	+	import (
	6	+	"bytes"
	7	+	"context"
	8	+	"crypto/sha256"
	9	+	"encoding/hex"
	10	+	"errors"
	11	+	"io"
	12	+	"strconv"
	13	+	"strings"
	14	+	"testing"
	15	+	"time"
	16	+
	17	+	"github.com/stretchr/testify/assert"
	18	+	"github.com/stretchr/testify/mock"
	19	+	"github.com/stretchr/testify/require"
	20	+
	21	+	"bnema.dev/ermite/internal/domain"
	22	+	"bnema.dev/ermite/internal/ports"
	23	+	"bnema.dev/ermite/internal/ports/mocks"
	24	+	)
	25	+
	26	+	// artifactBlobMock uses generated port mocks with byte storage in callbacks.
	27	+	// Real-disk end-to-end behavior is exercised by internal/app's artifact rig.
	28	+	func artifactBlobMock(t testing.T, failPart bool) (mocks.BlobStore, int, int) {
	29	+	t.Helper()
	30	+	b := mocks.NewBlobStore(t)
	31	+	objects := map[string][]byte{}
	32	+	active := map[string]ports.MultipartInfo{}
	33	+	puts, multiparts := new(int), new(int)
	34	+	b.EXPECT().Put(mock.Anything, mock.Anything, mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key string, r io.Reader, size int64) (ports.BlobInfo, error) {
	35	+	*puts++
	36	+	data, err := io.ReadAll(r)
	37	+	if err != nil {
	38	+	return ports.BlobInfo{}, err
	39	+	}
	40	+	if int64(len(data)) != size {
	41	+	return ports.BlobInfo{}, domain.ErrInvalid
	42	+	}
	43	+	objects[key] = data
	44	+	return ports.BlobInfo{Key: key, Size: size}, nil
	45	+	}).Maybe()
	46	+	b.EXPECT().Open(mock.Anything, mock.Anything, mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key string, offset, length int64) (io.ReadCloser, ports.BlobInfo, error) {
	47	+	data, ok := objects[key]
	48	+	if !ok {
	49	+	return nil, ports.BlobInfo{}, domain.ErrNotFound
	50	+	}
	51	+	info := ports.BlobInfo{Key: key, Size: int64(len(data))}
	52	+	if offset >= info.Size {
	53	+	return nil, info, domain.ErrOutOfRange
	54	+	}
	55	+	end := info.Size
	56	+	if length >= 0 {
	57	+	end = min(end, offset+length)
	58	+	}
	59	+	return io.NopCloser(bytes.NewReader(data[offset:end])), info, nil
	60	+	}).Maybe()
	61	+	b.EXPECT().Stat(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key string) (ports.BlobInfo, error) {
	62	+	data, ok := objects[key]
	63	+	if !ok {
	64	+	return ports.BlobInfo{}, domain.ErrNotFound
	65	+	}
	66	+	return ports.BlobInfo{Key: key, Size: int64(len(data))}, nil
	67	+	}).Maybe()
	68	+	b.EXPECT().Delete(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key string) error { delete(objects, key); return nil }).Maybe()
	69	+	b.EXPECT().List(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, prefix string, fn func(ports.BlobInfo) error) error {
	70	+	for key, data := range objects {
	71	+	if strings.HasPrefix(key, prefix) {
	72	+	if err := fn(ports.BlobInfo{Key: key, Size: int64(len(data))}); err != nil {
	73	+	return err
	74	+	}
	75	+	}
	76	+	}
	77	+	return nil
	78	+	}).Maybe()
	79	+	b.EXPECT().ListMultipart(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, prefix string, fn func(ports.MultipartInfo) error) error {
	80	+	for key, m := range active {
	81	+	if strings.HasPrefix(key, prefix) {
	82	+	if err := fn(m); err != nil {
	83	+	return err
	84	+	}
	85	+	}
	86	+	}
	87	+	return nil
	88	+	}).Maybe()
	89	+	b.EXPECT().AbortMultipart(mock.Anything, mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key, id string) error { delete(active, key); return nil }).Maybe()
	90	+	b.EXPECT().NewMultipart(mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, key string) (ports.MultipartUpload, error) {
	91	+	*multiparts++
	92	+	active[key] = ports.MultipartInfo{Key: key, ID: "upload"}
	93	+	u := mocks.NewMultipartUpload(t)
	94	+	var data []byte
	95	+	u.EXPECT().UploadPart(mock.Anything, mock.Anything, mock.Anything, mock.Anything).RunAndReturn(func(_ context.Context, _ int, r io.Reader, size int64) error {
	96	+	if failPart {
	97	+	return errors.New("part failed")
	98	+	}
	99	+	part, err := io.ReadAll(r)
	100	+	data = append(data, part...)
	101	+	return err
	102	+	}).Maybe()
	103	+	u.EXPECT().Complete(mock.Anything).RunAndReturn(func(context.Context) (ports.BlobInfo, error) {
	104	+	objects[key] = data
	105	+	delete(active, key)
	106	+	return ports.BlobInfo{Key: key, Size: int64(len(data))}, nil
	107	+	}).Maybe()
	108	+	u.EXPECT().Abort(mock.Anything).RunAndReturn(func(context.Context) error { delete(active, key); return nil }).Maybe()
	109	+	return u, nil
	110	+	}).Maybe()
	111	+	return b, puts, multiparts
	112	+	}
	113	+
	114	+	func TestArtifactBlobAssembly(t *testing.T) {
	115	+	for _, size := range []int{0, 1, ports.BlobPartSize, ports.BlobPartSize + 1, 2*ports.BlobPartSize + 17} {
	116	+	t.Run(strconv.Itoa(size), func(t *testing.T) {
	117	+	ctx := t.Context()
	118	+	b, puts, multiparts := artifactBlobMock(t, false)
	119	+	c := &CI{cfg: CIConfig{Blobs: b}}
	120	+	a := domain.CIArtifact{RunID: 1, ID: 2, Store: domain.CIArtifactStoreBlob}
	121	+	data := bytes.Repeat([]byte("x"), size)
	122	+	// Parts straddle block boundaries, with an empty block in the list.
	123	+	cut := min(size, 3<<20)
	124	+	_, err := b.Put(ctx, a.BlockKey("a"), bytes.NewReader(data[:cut]), int64(cut))
	125	+	require.NoError(t, err)
	126	+	_, err = b.Put(ctx, a.BlockKey("b"), bytes.NewReader(data[cut:]), int64(size-cut))
	127	+	require.NoError(t, err)
	128	+	_, err = b.Put(ctx, a.BlockKey("empty"), bytes.NewReader(nil), 0)
	129	+	require.NoError(t, err)
	130	+	*puts = 0
	131	+	n, digest, err := c.assembleBlobs(ctx, a, []string{"a", "empty", "b"}, int64(size))
	132	+	require.NoError(t, err)
	133	+	assert.EqualValues(t, size, n)
	134	+	sum := sha256.Sum256(data)
	135	+	assert.Equal(t, "sha256:"+hex.EncodeToString(sum[:]), digest)
	136	+	if size <= ports.BlobPartSize {
	137	+	assert.Equal(t, 1, *puts)
	138	+	assert.Zero(t, *multiparts)
	139	+	} else {
	140	+	assert.Zero(t, *puts)
	141	+	assert.Equal(t, 1, *multiparts)
	142	+	}
	143	+	if size > 0 {
	144	+	r, _, err := b.Open(ctx, a.BlobKey(), 0, -1)
	145	+	require.NoError(t, err)
	146	+	got, err := io.ReadAll(r)
	147	+	require.NoError(t, err)
	148	+	require.NoError(t, r.Close())
	149	+	assert.Equal(t, data, got)
	150	+	} else {
	151	+	info, err := b.Stat(ctx, a.BlobKey())
	152	+	require.NoError(t, err)
	153	+	assert.Zero(t, info.Size)
	154	+	}
	155	+	// No legacy reader is needed for a blob download, including EOF resumes.
	156	+	a.Size = int64(size)
	157	+	var got []byte
	158	+	require.NoError(t, c.streamArtifact(ctx, a, 0, func(ch ports.ArtifactChunk) error { got = append(got, ch.Data...); return nil }))
	159	+	assert.Equal(t, data, append([]byte{}, got...))
	160	+	require.NoError(t, c.streamArtifact(ctx, a, a.Size, func(ports.ArtifactChunk) error { return nil }))
	161	+	c.removeArtifactFiles(ctx, []domain.CIArtifact{a})
	162	+	_, err = b.Stat(ctx, a.BlobKey())
	163	+	assert.ErrorIs(t, err, domain.ErrNotFound)
	164	+	require.NoError(t, b.List(ctx, a.BlobKey()+".blocks/", func(ports.BlobInfo) error { t.Error("block survived deletion"); return nil }))
	165	+	})
	166	+	}
	167	+	}
	168	+
	169	+	func TestArtifactBlobAssemblyFailureAborts(t *testing.T) {
	170	+	for _, failPart := range []bool{false, true} {
	171	+	t.Run(map[bool]string{false: "missing block", true: "failed part"}[failPart], func(t *testing.T) {
	172	+	b, _, multiparts := artifactBlobMock(t, failPart)
	173	+	c := &CI{cfg: CIConfig{Blobs: b}}
	174	+	a := domain.CIArtifact{RunID: 1, ID: 2}
	175	+	data := bytes.Repeat([]byte("a"), ports.BlobPartSize)
	176	+	_, err := b.Put(t.Context(), a.BlockKey("a"), bytes.NewReader(data), int64(len(data)))
	177	+	require.NoError(t, err)
	178	+	_, _, err = c.assembleBlobs(t.Context(), a, []string{"a", "missing"}, ports.BlobPartSize+1)
	179	+	require.Error(t, err)
	180	+	assert.Equal(t, 1, *multiparts)
	181	+	_, err = b.Stat(t.Context(), a.BlobKey())
	182	+	assert.ErrorIs(t, err, domain.ErrNotFound)
	183	+	require.NoError(t, b.ListMultipart(t.Context(), "ci/", func(ports.MultipartInfo) error { t.Error("multipart survived failure"); return nil }))
	184	+	})
	185	+	}
	186	+	}
	187	+
	188	+	func TestArtifactPutUnsizedBound(t *testing.T) {
	189	+	b, _, _ := artifactBlobMock(t, false)
	190	+	c := &CI{cfg: CIConfig{Blobs: b}}
	191	+	for _, size := range []int{0, 1, domain.CIArtifactMaxBlockSize, domain.CIArtifactMaxBlockSize + 1} {
	192	+	n, err := c.putUnsized(t.Context(), "ci/1/2", bytes.NewReader(make([]byte, size)))
	193	+	if size > domain.CIArtifactMaxBlockSize {
	194	+	assert.ErrorIs(t, err, domain.ErrInvalid)
	195	+	assert.Contains(t, err.Error(), "a block has at most 8 MiB")
	196	+	} else {
	197	+	require.NoError(t, err)
	198	+	assert.EqualValues(t, size, n)
	199	+	}
	200	+	}
	201	+	ctx, cancel := context.WithCancel(t.Context())
	202	+	cancel()
	203	+	_, err := c.putUnsized(ctx, "ci/1/cancelled", bytes.NewReader([]byte("x")))
	204	+	assert.ErrorIs(t, err, context.Canceled)
	205	+	_, err = b.Stat(t.Context(), "ci/1/cancelled")
	206	+	assert.ErrorIs(t, err, domain.ErrNotFound)
	207	+	}
	208	+
	209	+	func TestArtifactLegacyStream(t *testing.T) {
	210	+	a := domain.CIArtifact{RunID: 1, ID: 2, Size: 6}
	211	+	legacy := mocks.NewCIArtifactFiles(t)
	212	+	legacy.EXPECT().Open(mock.Anything, artifactRef(a), int64(2)).RunAndReturn(func(context.Context, ports.CIArtifactRef, int64) (io.ReadCloser, error) {
	213	+	return io.NopCloser(bytes.NewReader([]byte("gacy"))), nil
	214	+	}).Twice()
	215	+	c := &CI{cfg: CIConfig{ArtifactFiles: legacy}}
	216	+	for _, store := range []domain.CIArtifactStore{domain.CIArtifactStoreLegacy, "disk"} {
	217	+	a.Store = store
	218	+	var got []byte
	219	+	require.NoError(t, c.streamArtifact(t.Context(), a, 2, func(ch ports.ArtifactChunk) error { got = append(got, ch.Data...); return nil }))
	220	+	assert.Equal(t, "gacy", string(got))
	221	+	}
	222	+	c.cfg.ArtifactFiles = nil
	223	+	assert.ErrorIs(t, c.streamArtifact(t.Context(), a, 0, func(ports.ArtifactChunk) error { return nil }), domain.ErrNotFound)
	224	+	}
	225	+
	226	+	func TestArtifactBlobOrphanSweep(t *testing.T) {
	227	+	ctx := t.Context()
	228	+	b := mocks.NewBlobStore(t)
	229	+	s := mocks.NewCIArtifactStore(t)
	230	+	c := &CI{cfg: CIConfig{Blobs: b, Artifacts: s, Clock: stillClock{}}}
	231	+	now := c.now()
	232	+	old := now.Add(-25 * time.Hour)
	233	+	pending := domain.CIArtifact{RunID: 1, ID: 2, Store: domain.CIArtifactStoreBlob, State: domain.CIArtifactPending}
	234	+	committed := pending
	235	+	committed.ID = 3
	236	+	committed.State = domain.CIArtifactCommitted
	237	+	missing := pending
	238	+	missing.ID = 4
	239	+	b.EXPECT().List(mock.Anything, "ci/", mock.Anything).RunAndReturn(func(_ context.Context, _ string, fn func(ports.BlobInfo) error) error {
	240	+	for _, info := range []ports.BlobInfo{
	241	+	{Key: missing.BlobKey(), ModTime: now}, // young: no lookup, no deletion
	242	+	{Key: missing.BlobKey(), ModTime: old},
	243	+	{Key: pending.BlockKey("a"), ModTime: old},
	244	+	{Key: committed.BlockKey("a"), ModTime: old},
	245	+	{Key: committed.BlobKey(), ModTime: old},
	246	+	} {
	247	+	require.NoError(t, fn(info))
	248	+	}
	249	+	return nil
	250	+	})
	251	+	s.EXPECT().GetArtifact(mock.Anything, missing.ID).Return(domain.CIArtifact{}, domain.ErrNotFound).Once()
	252	+	s.EXPECT().GetArtifact(mock.Anything, pending.ID).Return(pending, nil).Once()
	253	+	s.EXPECT().GetArtifact(mock.Anything, committed.ID).Return(committed, nil).Twice()
	254	+	b.EXPECT().Delete(mock.Anything, missing.BlobKey()).Return(nil).Once()
	255	+	b.EXPECT().Delete(mock.Anything, committed.BlockKey("a")).Return(nil).Once()
	256	+	b.EXPECT().ListMultipart(mock.Anything, "ci/", mock.Anything).RunAndReturn(func(_ context.Context, _ string, fn func(ports.MultipartInfo) error) error {
	257	+	require.NoError(t, fn(ports.MultipartInfo{Key: missing.BlobKey(), ID: "old", Started: old}))
	258	+	require.NoError(t, fn(ports.MultipartInfo{Key: missing.BlobKey(), ID: "young", Started: now}))
	259	+	return nil
	260	+	})
	261	+	b.EXPECT().AbortMultipart(mock.Anything, missing.BlobKey(), "old").Return(nil).Once()
	262	+	c.sweepArtifactBlobOrphans(ctx)
	263	+	}
	264	+
	265	+	func TestArtifactRemoveRunsWithoutLegacyReader(t *testing.T) {
	266	+	ctx := t.Context()
	267	+	b, _, _ := artifactBlobMock(t, false)
	268	+	c := &CI{cfg: CIConfig{Blobs: b}}
	269	+	a := domain.CIArtifact{RunID: 1, ID: 2}
	270	+	other := a
	271	+	other.RunID = 2
	272	+	for _, key := range []string{a.BlobKey(), a.BlockKey("a"), other.BlobKey()} {
	273	+	_, err := b.Put(ctx, key, bytes.NewReader([]byte("x")), 1)
	274	+	require.NoError(t, err)
	275	+	}
	276	+	_, err := b.NewMultipart(ctx, a.BlobKey())
	277	+	require.NoError(t, err)
	278	+	require.NoError(t, c.removeArtifactRuns(ctx, []int64{1}))
	279	+	_, err = b.Stat(ctx, a.BlobKey())
	280	+	assert.ErrorIs(t, err, domain.ErrNotFound)
	281	+	_, err = b.Stat(ctx, a.BlockKey("a"))
	282	+	assert.ErrorIs(t, err, domain.ErrNotFound)
	283	+	_, err = b.Stat(ctx, other.BlobKey())
	284	+	require.NoError(t, err)
	285	+	require.NoError(t, b.ListMultipart(ctx, "ci/", func(ports.MultipartInfo) error { t.Error("upload survived run deletion"); return nil }))
	286	+	}
	287	+
	288	+	func TestArtifactBlobIDs(t *testing.T) {
	289	+	a := domain.CIArtifact{RunID: 7, ID: 42}
	290	+	for _, key := range []string{a.BlobKey(), a.BlockKey("abcdef.0123")} {
	291	+	run, id, block, ok := artifactBlobIDs(key)
	292	+	assert.True(t, ok)
	293	+	assert.EqualValues(t, 7, run)
	294	+	assert.EqualValues(t, 42, id)
	295	+	assert.Equal(t, strings.Contains(key, ".blocks/"), block)
	296	+	}
	297	+	for _, key := range []string{"release/1", "ci/1/2", a.BlobKey() + ".blocks/", a.BlobKey() + "/wrong"} {
	298	+	_, _, _, ok := artifactBlobIDs(key)
	299	+	assert.False(t, ok, key)
	300	+	}
	301	+	}

M internal/usecase/ci_watch.go +1 -3


@@ -326,8 +326,6 @@ func (c *CI) RemoveLogs(ctx context.Context, runIDs []int64) error {
326	326		c.locks.Delete(id)
327	327		}
328	328		err := errors.Join(c.cfg.Logs.RemoveRuns(ctx, runIDs), c.cfg.Summaries.RemoveRuns(ctx, runIDs))
329		-	if c.artifactsOn() {
330		-	err = errors.Join(err, c.cfg.ArtifactFiles.RemoveRuns(ctx, runIDs))
331		-	}
	329	+	err = errors.Join(err, c.removeArtifactRuns(ctx, runIDs))
332	330		return err
333	331		}

 		c.locks.Delete(id)
 	}
 	err := errors.Join(c.cfg.Logs.RemoveRuns(ctx, runIDs), c.cfg.Summaries.RemoveRuns(ctx, runIDs))
-	if c.artifactsOn() {
+	err = errors.Join(err, c.removeArtifactRuns(ctx, runIDs))
-		err = errors.Join(err, c.cfg.ArtifactFiles.RemoveRuns(ctx, runIDs))
-	}
 	return err
 }

M proto/ermite/storage/ci_artifact.proto +4 -2

@@ -1,8 +1,8 @@
1	1		// SPDX-License-Identifier: AGPL-3.0-or-later
2	2
3	3		// Internal, unversioned storage schema of the CI artifacts (artifacts v4,
4		-	// CI-4 D3). The contents are not stored here: they are files under
5		-	// $ERMITE_DATA_DIR/ci/artifacts.
	4	+	// CI-4 D3). Contents are stored separately, in the blob store or (for
	5	+	// legacy records) under $ERMITE_DATA_DIR/ci/artifacts.
6	6
7	7		syntax = "proto3";
8	8

@@ -49,6 +49,8 @@ message CIArtifact {
49	49		// list_digest: the SHA-256 of the block list of the commit, to recognize a
50	50		// repeated commit.
51	51		bytes list_digest = 15;
	52	+	// "" or "disk": legacy files under ci/artifacts; "blob": the blob store.
	53	+	string store = 16;
52	54		}
53	55
54	56		// CIArtifactBlock is stored under "artifactblk/<artifact:020d>/<hex(block id)>":

 // SPDX-License-Identifier: AGPL-3.0-or-later
 // Internal, unversioned storage schema of the CI artifacts (artifacts v4,
-// CI-4 D3). The contents are not stored here: they are files under
+// CI-4 D3). Contents are stored separately, in the blob store or (for
-// $ERMITE_DATA_DIR/ci/artifacts.
+// legacy records) under $ERMITE_DATA_DIR/ci/artifacts.
 syntax = "proto3";
   // list_digest: the SHA-256 of the block list of the commit, to recognize a
   // repeated commit.
   bytes list_digest = 15;
+  // "" or "disk": legacy files under ci/artifacts; "blob": the blob store.
+  string store = 16;
 }
 // CIArtifactBlock is stored under "artifactblk/<artifact:020d>/<hex(block id)>":