bug: durable: a failed checkpoint after a landed Append leaves the Handle stuck on ErrStepConflict #241

type failOnceCheckpoint struct {
	durable.Store
	failed bool
}

func (s *failOnceCheckpoint) Checkpoint(ctx context.Context, id durable.InstanceID, snap []byte, through int, opts ...durable.CheckpointOption) error {
	if through >= 0 && !s.failed {
		s.failed = true
		return errors.New("transient checkpoint I/O")
	}
	return s.Store.Checkpoint(ctx, id, snap, through, opts...)
}

func TestTickCheckpointFailure_DoesNotWedgeHandle(t *testing.T) {
	ctx := context.Background()
	st := &failOnceCheckpoint{Store: durable.NewMemStore()}
	clk := state.NewFakeClock(epoch)
	r := durable.NewRunner(chainedTimerMachine(), st,
		durable.WithRunnerClock[string, string, *timerCtx](clk),
		durable.WithCheckpointEvery[string, string, *timerCtx](2))
	h, _ := r.Start(ctx, "wedge", &timerCtx{}, state.WithInitialState("idle"))
	if _, err := h.Fire(ctx, "arm"); err != nil { // step 0
		t.Fatal(err)
	}
	clk.Advance(4 * time.Second)
	if _, err := h.Tick(ctx); err == nil { // step 1: Append lands, Checkpoint fails
		t.Fatal("expected injected checkpoint failure")
	}
	clk.Advance(8 * time.Second)
	if _, err := h.Tick(ctx); err != nil { // fails: ErrStepConflict on step 1
		t.Fatalf("Tick after a transient checkpoint failure: %v", err)
	}
}