feat(faultfs): inject failures in dir and file operations

FailOp fails the n-th SyncDir, Rename, Remove, Truncate or OpenFile call.
DropOnSyncFailure models pages dropped after a failed fsync.

commit: b799b920f9aee8e319e77501d91c6234e0be2669
author: brice · 12d ago · 2026-10-10 02:40 UTC
committer: brice · 12d ago · 2026-10-10 02:52 UTC
parent: 19d4f5e
signature: GPG VerifiedCD88F9E564E0102646FB6077A1A07C7C7C3E349D · arch-gpg Verified for the committer email

2 files · +211 -2

M internal/fsys/faultfs/faultfs.go +106 -2

@@ -1,8 +1,9 @@
1	1		// SPDX-License-Identifier: MIT
2	2
3	3		// Package faultfs is an in-memory fsys.FS for tests. It models a page cache in
4		-	// front of durable storage and can inject write and sync failures, delay
5		-	// syncs, count syncs and simulate a power loss.
	4	+	// front of durable storage and can inject failures in writes, syncs and
	5	+	// directory and file operations, delay syncs, count syncs and simulate a power
	6	+	// loss.
6	7		//
7	8		// Durability model: file bytes become durable on Sync or SyncAll; directory
8	9		// operations (create, rename, remove, mkdir) become durable on SyncDir of the

@@ -35,6 +36,42 @@ var ErrInjected = errors.New("faultfs: injected failure")
35	36
36	37		const defaultPageSize = 4096
37	38
	39	+	// OpKind identifies a file-system operation other than Write and Sync, whose
	40	+	// failures are injected with FailOp.
	41	+	type OpKind uint8
	42	+
	43	+	// Operations that FailOp can fail.
	44	+	const (
	45	+	OpSyncDir OpKind = iota
	46	+	OpRename
	47	+	OpRemove
	48	+	OpTruncate
	49	+	OpOpenFile
	50	+	numOps
	51	+	)
	52	+
	53	+	func (k OpKind) String() string {
	54	+	switch k {
	55	+	case OpSyncDir:
	56	+	return "SyncDir"
	57	+	case OpRename:
	58	+	return "Rename"
	59	+	case OpRemove:
	60	+	return "Remove"
	61	+	case OpTruncate:
	62	+	return "Truncate"
	63	+	case OpOpenFile:
	64	+	return "OpenFile"
	65	+	}
	66	+	return "unknown"
	67	+	}
	68	+
	69	+	// opCall designates the n-th call of an operation kind.
	70	+	type opCall struct {
	71	+	kind OpKind
	72	+	n int
	73	+	}
	74	+
38	75		type node struct {
39	76		data []byte // current content (page cache)
40	77		durable []byte // content as of the last successful sync

@@ -64,6 +101,10 @@ type FS struct {
64	101		failSync map[int]error // by absolute sync number (Sync+SyncAll)
65	102		syncDelay time.Duration
66	103		nSyncCalls int
	104	+	dropOnSyncFail bool
	105	+
	106	+	opCalls [numOps]int // calls so far, by kind
	107	+	failOps map[opCall]error // by kind and absolute call number
67	108		}
68	109
69	110		type writeFault struct {

@@ -81,6 +122,7 @@ func New() *FS {
81	122		locks: map[string]bool{},
82	123		failWrite: map[int]writeFault{},
83	124		failSync: map[int]error{},
	125	+	failOps: map[opCall]error{},
84	126		}
85	127		}
86	128

@@ -115,6 +157,49 @@ func (f *FS) FailSync(n int, err error) {
115	157		f.failSync[f.nSyncCalls+n] = err
116	158		}
117	159
	160	+	// FailOp makes the n-th call from now (1 = the next one) of the operation
	161	+	// kind fail with err (ErrInjected if nil). A failed call has no effect. Every
	162	+	// call counts, whether or not it would have failed anyway.
	163	+	func (f *FS) FailOp(kind OpKind, n int, err error) {
	164	+	if err == nil {
	165	+	err = ErrInjected
	166	+	}
	167	+	f.mu.Lock()
	168	+	defer f.mu.Unlock()
	169	+	f.failOps[opCall{kind, f.opCalls[kind] + n}] = err
	170	+	}
	171	+
	172	+	// OpCalls returns the number of calls of the operation kind so far.
	173	+	func (f *FS) OpCalls(kind OpKind) int {
	174	+	f.mu.Lock()
	175	+	defer f.mu.Unlock()
	176	+	return f.opCalls[kind]
	177	+	}
	178	+
	179	+	// injected counts a call of kind and returns the error to fail it with, if
	180	+	// any. It must be called with f.mu held.
	181	+	func (f *FS) injected(kind OpKind) error {
	182	+	f.opCalls[kind]++
	183	+	c := opCall{kind, f.opCalls[kind]}
	184	+	err, ok := f.failOps[c]
	185	+	if ok {
	186	+	delete(f.failOps, c)
	187	+	}
	188	+	return err
	189	+	}
	190	+
	191	+	// DropOnSyncFailure selects what a failed Sync or SyncAll does to the data
	192	+	// that was not yet durable. By default it stays in the page cache and a later
	193	+	// successful sync makes it durable. When enabled, it is dropped, as on
	194	+	// systems that mark the pages clean after a write-back error ("fsyncgate"):
	195	+	// the file reverts to its last durable content, and the data is gone even if
	196	+	// a later sync succeeds.
	197	+	func (f *FS) DropOnSyncFailure(drop bool) {
	198	+	f.mu.Lock()
	199	+	f.dropOnSyncFail = drop
	200	+	f.mu.Unlock()
	201	+	}
	202	+
118	203		// SetSyncDelay makes every Sync and SyncAll block for d before taking effect.
119	204		func (f *FS) SetSyncDelay(d time.Duration) {
120	205		f.mu.Lock()

@@ -294,6 +379,9 @@ func (f *FS) OpenFile(name string, flag int, perm os.FileMode) (fsys.File, error
294	379		name = filepath.Clean(name)
295	380		f.mu.Lock()
296	381		defer f.mu.Unlock()
	382	+	if err := f.injected(OpOpenFile); err != nil {
	383	+	return nil, &fs.PathError{Op: "open", Path: name, Err: err}
	384	+	}
297	385		n, ok := f.files[name]
298	386		switch {
299	387		case f.dirs[name]:

@@ -322,6 +410,9 @@ func (f *FS) Rename(oldpath, newpath string) error {
322	410		oldpath, newpath = filepath.Clean(oldpath), filepath.Clean(newpath)
323	411		f.mu.Lock()
324	412		defer f.mu.Unlock()
	413	+	if err := f.injected(OpRename); err != nil {
	414	+	return &os.LinkError{Op: "rename", Old: oldpath, New: newpath, Err: err}
	415	+	}
325	416		n, ok := f.files[oldpath]
326	417		if !ok {
327	418		return &os.LinkError{Op: "rename", Old: oldpath, New: newpath, Err: fs.ErrNotExist}

@@ -345,6 +436,9 @@ func (f *FS) Remove(name string) error {
345	436		name = filepath.Clean(name)
346	437		f.mu.Lock()
347	438		defer f.mu.Unlock()
	439	+	if err := f.injected(OpRemove); err != nil {
	440	+	return &fs.PathError{Op: "remove", Path: name, Err: err}
	441	+	}
348	442		n, ok := f.files[name]
349	443		if !ok {
350	444		return &fs.PathError{Op: "remove", Path: name, Err: fs.ErrNotExist}

@@ -394,6 +488,9 @@ func (f *FS) SyncDir(dir string) error {
394	488		dir = filepath.Clean(dir)
395	489		f.mu.Lock()
396	490		defer f.mu.Unlock()
	491	+	if err := f.injected(OpSyncDir); err != nil {
	492	+	return &fs.PathError{Op: "sync", Path: dir, Err: err}
	493	+	}
397	494		if !f.dirs[dir] {
398	495		return &fs.PathError{Op: "sync", Path: dir, Err: fs.ErrNotExist}
399	496		}

@@ -540,6 +637,10 @@ func (h *file) sync(all bool) error {
540	637		}
541	638		if err, ok := f.failSync[f.nSyncCalls]; ok {
542	639		delete(f.failSync, f.nSyncCalls)
	640	+	if f.dropOnSyncFail {
	641	+	h.n.data = append([]byte(nil), h.n.durable...)
	642	+	h.n.truncated = false
	643	+	}
543	644		f.mu.Unlock()
544	645		return err
545	646		}

@@ -569,6 +670,9 @@ func (h *file) Truncate(size int64) error {
569	670		if err := h.check(); err != nil {
570	671		return err
571	672		}
	673	+	if err := h.fs.injected(OpTruncate); err != nil {
	674	+	return &fs.PathError{Op: "truncate", Err: err}
	675	+	}
572	676		if !h.write \|\| size < 0 {
573	677		return &fs.PathError{Op: "truncate", Err: fs.ErrInvalid}
574	678		}

@@ -1,8 +1,9 @@
1		// SPDX-License-Identifier: MIT	1		// SPDX-License-Identifier: MIT
2			2
3		// Package faultfs is an in-memory fsys.FS for tests. It models a page cache in	3		// Package faultfs is an in-memory fsys.FS for tests. It models a page cache in
4	-	// front of durable storage and can inject write and sync failures, delay	4	+	// front of durable storage and can inject failures in writes, syncs and
5	-	// syncs, count syncs and simulate a power loss.	5	+	// directory and file operations, delay syncs, count syncs and simulate a power
			6	+	// loss.
6		//	7		//
7		// Durability model: file bytes become durable on Sync or SyncAll; directory	8		// Durability model: file bytes become durable on Sync or SyncAll; directory
8		// operations (create, rename, remove, mkdir) become durable on SyncDir of the	9		// operations (create, rename, remove, mkdir) become durable on SyncDir of the

@@ -35,6 +36,42 @@ var ErrInjected = errors.New("faultfs: injected failure")
35			36
36		const defaultPageSize = 4096	37		const defaultPageSize = 4096
37			38
			39	+	// OpKind identifies a file-system operation other than Write and Sync, whose
			40	+	// failures are injected with FailOp.
			41	+	type OpKind uint8
			42	+
			43	+	// Operations that FailOp can fail.
			44	+	const (
			45	+	OpSyncDir OpKind = iota
			46	+	OpRename
			47	+	OpRemove
			48	+	OpTruncate
			49	+	OpOpenFile
			50	+	numOps
			51	+	)
			52	+
			53	+	func (k OpKind) String() string {
			54	+	switch k {
			55	+	case OpSyncDir:
			56	+	return "SyncDir"
			57	+	case OpRename:
			58	+	return "Rename"
			59	+	case OpRemove:
			60	+	return "Remove"
			61	+	case OpTruncate:
			62	+	return "Truncate"
			63	+	case OpOpenFile:
			64	+	return "OpenFile"
			65	+	}
			66	+	return "unknown"
			67	+	}
			68	+
			69	+	// opCall designates the n-th call of an operation kind.
			70	+	type opCall struct {
			71	+	kind OpKind
			72	+	n int
			73	+	}
			74	+
38		type node struct {	75		type node struct {
39		data []byte // current content (page cache)	76		data []byte // current content (page cache)
40		durable []byte // content as of the last successful sync	77		durable []byte // content as of the last successful sync

@@ -64,6 +101,10 @@ type FS struct {
64		failSync map[int]error // by absolute sync number (Sync+SyncAll)	101		failSync map[int]error // by absolute sync number (Sync+SyncAll)
65		syncDelay time.Duration	102		syncDelay time.Duration
66		nSyncCalls int	103		nSyncCalls int
			104	+	dropOnSyncFail bool
			105	+
			106	+	opCalls [numOps]int // calls so far, by kind
			107	+	failOps map[opCall]error // by kind and absolute call number
67		}	108		}
68			109
69		type writeFault struct {	110		type writeFault struct {

@@ -81,6 +122,7 @@ func New() *FS {
81		locks: map[string]bool{},	122		locks: map[string]bool{},
82		failWrite: map[int]writeFault{},	123		failWrite: map[int]writeFault{},
83		failSync: map[int]error{},	124		failSync: map[int]error{},
			125	+	failOps: map[opCall]error{},
84		}	126		}
85		}	127		}
86			128

@@ -115,6 +157,49 @@ func (f *FS) FailSync(n int, err error) {
115		f.failSync[f.nSyncCalls+n] = err	157		f.failSync[f.nSyncCalls+n] = err
116		}	158		}
117			159
			160	+	// FailOp makes the n-th call from now (1 = the next one) of the operation
			161	+	// kind fail with err (ErrInjected if nil). A failed call has no effect. Every
			162	+	// call counts, whether or not it would have failed anyway.
			163	+	func (f *FS) FailOp(kind OpKind, n int, err error) {
			164	+	if err == nil {
			165	+	err = ErrInjected
			166	+	}
			167	+	f.mu.Lock()
			168	+	defer f.mu.Unlock()
			169	+	f.failOps[opCall{kind, f.opCalls[kind] + n}] = err
			170	+	}
			171	+
			172	+	// OpCalls returns the number of calls of the operation kind so far.
			173	+	func (f *FS) OpCalls(kind OpKind) int {
			174	+	f.mu.Lock()
			175	+	defer f.mu.Unlock()
			176	+	return f.opCalls[kind]
			177	+	}
			178	+
			179	+	// injected counts a call of kind and returns the error to fail it with, if
			180	+	// any. It must be called with f.mu held.
			181	+	func (f *FS) injected(kind OpKind) error {
			182	+	f.opCalls[kind]++
			183	+	c := opCall{kind, f.opCalls[kind]}
			184	+	err, ok := f.failOps[c]
			185	+	if ok {
			186	+	delete(f.failOps, c)
			187	+	}
			188	+	return err
			189	+	}
			190	+
			191	+	// DropOnSyncFailure selects what a failed Sync or SyncAll does to the data
			192	+	// that was not yet durable. By default it stays in the page cache and a later
			193	+	// successful sync makes it durable. When enabled, it is dropped, as on
			194	+	// systems that mark the pages clean after a write-back error ("fsyncgate"):
			195	+	// the file reverts to its last durable content, and the data is gone even if
			196	+	// a later sync succeeds.
			197	+	func (f *FS) DropOnSyncFailure(drop bool) {
			198	+	f.mu.Lock()
			199	+	f.dropOnSyncFail = drop
			200	+	f.mu.Unlock()
			201	+	}
			202	+
118		// SetSyncDelay makes every Sync and SyncAll block for d before taking effect.	203		// SetSyncDelay makes every Sync and SyncAll block for d before taking effect.
119		func (f *FS) SetSyncDelay(d time.Duration) {	204		func (f *FS) SetSyncDelay(d time.Duration) {
120		f.mu.Lock()	205		f.mu.Lock()

@@ -294,6 +379,9 @@ func (f *FS) OpenFile(name string, flag int, perm os.FileMode) (fsys.File, error
294		name = filepath.Clean(name)	379		name = filepath.Clean(name)
295		f.mu.Lock()	380		f.mu.Lock()
296		defer f.mu.Unlock()	381		defer f.mu.Unlock()
			382	+	if err := f.injected(OpOpenFile); err != nil {
			383	+	return nil, &fs.PathError{Op: "open", Path: name, Err: err}
			384	+	}
297		n, ok := f.files[name]	385		n, ok := f.files[name]
298		switch {	386		switch {
299		case f.dirs[name]:	387		case f.dirs[name]:

@@ -322,6 +410,9 @@ func (f *FS) Rename(oldpath, newpath string) error {
322		oldpath, newpath = filepath.Clean(oldpath), filepath.Clean(newpath)	410		oldpath, newpath = filepath.Clean(oldpath), filepath.Clean(newpath)
323		f.mu.Lock()	411		f.mu.Lock()
324		defer f.mu.Unlock()	412		defer f.mu.Unlock()
			413	+	if err := f.injected(OpRename); err != nil {
			414	+	return &os.LinkError{Op: "rename", Old: oldpath, New: newpath, Err: err}
			415	+	}
325		n, ok := f.files[oldpath]	416		n, ok := f.files[oldpath]
326		if !ok {	417		if !ok {
327		return &os.LinkError{Op: "rename", Old: oldpath, New: newpath, Err: fs.ErrNotExist}	418		return &os.LinkError{Op: "rename", Old: oldpath, New: newpath, Err: fs.ErrNotExist}

@@ -345,6 +436,9 @@ func (f *FS) Remove(name string) error {
345		name = filepath.Clean(name)	436		name = filepath.Clean(name)
346		f.mu.Lock()	437		f.mu.Lock()
347		defer f.mu.Unlock()	438		defer f.mu.Unlock()
			439	+	if err := f.injected(OpRemove); err != nil {
			440	+	return &fs.PathError{Op: "remove", Path: name, Err: err}
			441	+	}
348		n, ok := f.files[name]	442		n, ok := f.files[name]
349		if !ok {	443		if !ok {
350		return &fs.PathError{Op: "remove", Path: name, Err: fs.ErrNotExist}	444		return &fs.PathError{Op: "remove", Path: name, Err: fs.ErrNotExist}

@@ -394,6 +488,9 @@ func (f *FS) SyncDir(dir string) error {
394		dir = filepath.Clean(dir)	488		dir = filepath.Clean(dir)
395		f.mu.Lock()	489		f.mu.Lock()
396		defer f.mu.Unlock()	490		defer f.mu.Unlock()
			491	+	if err := f.injected(OpSyncDir); err != nil {
			492	+	return &fs.PathError{Op: "sync", Path: dir, Err: err}
			493	+	}
397		if !f.dirs[dir] {	494		if !f.dirs[dir] {
398		return &fs.PathError{Op: "sync", Path: dir, Err: fs.ErrNotExist}	495		return &fs.PathError{Op: "sync", Path: dir, Err: fs.ErrNotExist}
399		}	496		}

@@ -540,6 +637,10 @@ func (h *file) sync(all bool) error {
540		}	637		}
541		if err, ok := f.failSync[f.nSyncCalls]; ok {	638		if err, ok := f.failSync[f.nSyncCalls]; ok {
542		delete(f.failSync, f.nSyncCalls)	639		delete(f.failSync, f.nSyncCalls)
			640	+	if f.dropOnSyncFail {
			641	+	h.n.data = append([]byte(nil), h.n.durable...)
			642	+	h.n.truncated = false
			643	+	}
543		f.mu.Unlock()	644		f.mu.Unlock()
544		return err	645		return err
545		}	646		}

@@ -569,6 +670,9 @@ func (h *file) Truncate(size int64) error {
569		if err := h.check(); err != nil {	670		if err := h.check(); err != nil {
570		return err	671		return err
571		}	672		}
			673	+	if err := h.fs.injected(OpTruncate); err != nil {
			674	+	return &fs.PathError{Op: "truncate", Err: err}
			675	+	}
572		if !h.write \|\| size < 0 {	676		if !h.write \|\| size < 0 {
573		return &fs.PathError{Op: "truncate", Err: fs.ErrInvalid}	677		return &fs.PathError{Op: "truncate", Err: fs.ErrInvalid}
574		}	678		}

M internal/fsys/faultfs/faultfs_test.go +105 -0


@@ -382,3 +382,108 @@ func TestConcurrentUse(t *testing.T) {
382	382		t.Fatalf("Syncs=%d Writes=%d", f.Syncs(), f.Writes())
383	383		}
384	384		}
	385	+
	386	+	func TestFailOp(t *testing.T) {
	387	+	f := New()
	388	+	f.WriteFile("/d/a", []byte("x"))
	389	+	f.WriteFile("/d/b", nil)
	390	+
	391	+	f.FailOp(OpOpenFile, 2, nil)
	392	+	h, err := f.OpenFile("/d/a", os.O_RDWR, 0)
	393	+	if err != nil {
	394	+	t.Fatal(err)
	395	+	}
	396	+	if _, err := f.OpenFile("/d/a", os.O_RDWR, 0); !errors.Is(err, ErrInjected) {
	397	+	t.Fatalf("OpenFile = %v", err)
	398	+	}
	399	+	if _, err := f.OpenFile("/d/a", os.O_RDWR, 0); err != nil {
	400	+	t.Fatalf("faults fire once: %v", err)
	401	+	}
	402	+
	403	+	boom := errors.New("boom")
	404	+	f.FailOp(OpTruncate, 1, boom)
	405	+	if err := h.Truncate(0); !errors.Is(err, boom) {
	406	+	t.Fatalf("Truncate = %v", err)
	407	+	}
	408	+	if sz, _ := h.Size(); sz != 1 {
	409	+	t.Fatalf("failed Truncate changed the file: size %d", sz)
	410	+	}
	411	+	if err := h.Truncate(0); err != nil {
	412	+	t.Fatal(err)
	413	+	}
	414	+
	415	+	f.FailOp(OpRename, 1, nil)
	416	+	if err := f.Rename("/d/b", "/d/c"); !errors.Is(err, ErrInjected) {
	417	+	t.Fatalf("Rename = %v", err)
	418	+	}
	419	+	if _, ok := f.ReadFile("/d/b"); !ok {
	420	+	t.Fatal("failed Rename moved the file")
	421	+	}
	422	+	if err := f.Rename("/d/b", "/d/c"); err != nil {
	423	+	t.Fatal(err)
	424	+	}
	425	+
	426	+	f.FailOp(OpRemove, 1, nil)
	427	+	if err := f.Remove("/d/c"); !errors.Is(err, ErrInjected) {
	428	+	t.Fatalf("Remove = %v", err)
	429	+	}
	430	+	if _, ok := f.ReadFile("/d/c"); !ok {
	431	+	t.Fatal("failed Remove deleted the file")
	432	+	}
	433	+
	434	+	f.FailOp(OpSyncDir, 1, nil)
	435	+	if err := f.SyncDir("/d"); !errors.Is(err, ErrInjected) {
	436	+	t.Fatalf("SyncDir = %v", err)
	437	+	}
	438	+	if err := f.SyncDir("/d"); err != nil {
	439	+	t.Fatal(err)
	440	+	}
	441	+	if got := f.OpCalls(OpSyncDir); got != 2 {
	442	+	t.Fatalf("OpCalls(SyncDir) = %d", got)
	443	+	}
	444	+	}
	445	+
	446	+	func TestFailedSyncDirKeepsOpsVolatile(t *testing.T) {
	447	+	f := New()
	448	+	_ = f.MkdirAll("/d", 0o700)
	449	+	_ = f.SyncDir("/d")
	450	+	create(t, f, "/d/a")
	451	+	f.FailOp(OpSyncDir, 1, nil)
	452	+	if err := f.SyncDir("/d"); err == nil {
	453	+	t.Fatal("SyncDir succeeded")
	454	+	}
	455	+	f.PowerLoss(PowerLossOptions{})
	456	+	if _, ok := f.ReadFile("/d/a"); ok {
	457	+	t.Fatal("create survived a failed SyncDir")
	458	+	}
	459	+	}
	460	+
	461	+	func TestDropOnSyncFailure(t *testing.T) {
	462	+	for _, drop := range []bool{false, true} {
	463	+	f := New()
	464	+	f.DropOnSyncFailure(drop)
	465	+	h := create(t, f, "/a")
	466	+	write(t, h, "kept")
	467	+	if err := h.Sync(); err != nil {
	468	+	t.Fatal(err)
	469	+	}
	470	+	write(t, h, "-lost?")
	471	+	f.FailSync(1, nil)
	472	+	if err := h.Sync(); !errors.Is(err, ErrInjected) {
	473	+	t.Fatalf("Sync = %v", err)
	474	+	}
	475	+	want := "kept-lost?"
	476	+	if drop {
	477	+	want = "kept"
	478	+	}
	479	+	if got := mustRead(t, f, "/a"); got != want {
	480	+	t.Fatalf("drop=%v: content %q, want %q", drop, got, want)
	481	+	}
	482	+	if err := h.Sync(); err != nil {
	483	+	t.Fatal(err)
	484	+	}
	485	+	if d, _ := f.DurableFile("/a"); string(d) != want {
	486	+	t.Fatalf("drop=%v: durable %q, want %q", drop, d, want)
	487	+	}
	488	+	}
	489	+	}


@@ -382,3 +382,108 @@ func TestConcurrentUse(t *testing.T) {
382	t.Fatalf("Syncs=%d Writes=%d", f.Syncs(), f.Writes())	382		t.Fatalf("Syncs=%d Writes=%d", f.Syncs(), f.Writes())
383	}	383		}
384	}	384		}
		385	+
		386	+	func TestFailOp(t *testing.T) {
		387	+	f := New()
		388	+	f.WriteFile("/d/a", []byte("x"))
		389	+	f.WriteFile("/d/b", nil)
		390	+
		391	+	f.FailOp(OpOpenFile, 2, nil)
		392	+	h, err := f.OpenFile("/d/a", os.O_RDWR, 0)
		393	+	if err != nil {
		394	+	t.Fatal(err)
		395	+	}
		396	+	if _, err := f.OpenFile("/d/a", os.O_RDWR, 0); !errors.Is(err, ErrInjected) {
		397	+	t.Fatalf("OpenFile = %v", err)
		398	+	}
		399	+	if _, err := f.OpenFile("/d/a", os.O_RDWR, 0); err != nil {
		400	+	t.Fatalf("faults fire once: %v", err)
		401	+	}
		402	+
		403	+	boom := errors.New("boom")
		404	+	f.FailOp(OpTruncate, 1, boom)
		405	+	if err := h.Truncate(0); !errors.Is(err, boom) {
		406	+	t.Fatalf("Truncate = %v", err)
		407	+	}
		408	+	if sz, _ := h.Size(); sz != 1 {
		409	+	t.Fatalf("failed Truncate changed the file: size %d", sz)
		410	+	}
		411	+	if err := h.Truncate(0); err != nil {
		412	+	t.Fatal(err)
		413	+	}
		414	+
		415	+	f.FailOp(OpRename, 1, nil)
		416	+	if err := f.Rename("/d/b", "/d/c"); !errors.Is(err, ErrInjected) {
		417	+	t.Fatalf("Rename = %v", err)
		418	+	}
		419	+	if _, ok := f.ReadFile("/d/b"); !ok {
		420	+	t.Fatal("failed Rename moved the file")
		421	+	}
		422	+	if err := f.Rename("/d/b", "/d/c"); err != nil {
		423	+	t.Fatal(err)
		424	+	}
		425	+
		426	+	f.FailOp(OpRemove, 1, nil)
		427	+	if err := f.Remove("/d/c"); !errors.Is(err, ErrInjected) {
		428	+	t.Fatalf("Remove = %v", err)
		429	+	}
		430	+	if _, ok := f.ReadFile("/d/c"); !ok {
		431	+	t.Fatal("failed Remove deleted the file")
		432	+	}
		433	+
		434	+	f.FailOp(OpSyncDir, 1, nil)
		435	+	if err := f.SyncDir("/d"); !errors.Is(err, ErrInjected) {
		436	+	t.Fatalf("SyncDir = %v", err)
		437	+	}
		438	+	if err := f.SyncDir("/d"); err != nil {
		439	+	t.Fatal(err)
		440	+	}
		441	+	if got := f.OpCalls(OpSyncDir); got != 2 {
		442	+	t.Fatalf("OpCalls(SyncDir) = %d", got)
		443	+	}
		444	+	}
		445	+
		446	+	func TestFailedSyncDirKeepsOpsVolatile(t *testing.T) {
		447	+	f := New()
		448	+	_ = f.MkdirAll("/d", 0o700)
		449	+	_ = f.SyncDir("/d")
		450	+	create(t, f, "/d/a")
		451	+	f.FailOp(OpSyncDir, 1, nil)
		452	+	if err := f.SyncDir("/d"); err == nil {
		453	+	t.Fatal("SyncDir succeeded")
		454	+	}
		455	+	f.PowerLoss(PowerLossOptions{})
		456	+	if _, ok := f.ReadFile("/d/a"); ok {
		457	+	t.Fatal("create survived a failed SyncDir")
		458	+	}
		459	+	}
		460	+
		461	+	func TestDropOnSyncFailure(t *testing.T) {
		462	+	for _, drop := range []bool{false, true} {
		463	+	f := New()
		464	+	f.DropOnSyncFailure(drop)
		465	+	h := create(t, f, "/a")
		466	+	write(t, h, "kept")
		467	+	if err := h.Sync(); err != nil {
		468	+	t.Fatal(err)
		469	+	}
		470	+	write(t, h, "-lost?")
		471	+	f.FailSync(1, nil)
		472	+	if err := h.Sync(); !errors.Is(err, ErrInjected) {
		473	+	t.Fatalf("Sync = %v", err)
		474	+	}
		475	+	want := "kept-lost?"
		476	+	if drop {
		477	+	want = "kept"
		478	+	}
		479	+	if got := mustRead(t, f, "/a"); got != want {
		480	+	t.Fatalf("drop=%v: content %q, want %q", drop, got, want)
		481	+	}
		482	+	if err := h.Sync(); err != nil {
		483	+	t.Fatal(err)
		484	+	}
		485	+	if d, _ := f.DurableFile("/a"); string(d) != want {
		486	+	t.Fatalf("drop=%v: durable %q, want %q", drop, d, want)
		487	+	}
		488	+	}
		489	+	}