1
0
mirror of https://github.com/redis/go-redis.git synced 2026-10-22 19:49:00 +03:00
Files
go-redis/autopipeline_test.go
ofekshenawa 48d65e5641 Metadata driven cluster routing (#4093)
* feat(command): add metadata-driven routing

* fix(cluster): preserve routing compatibility

* fix spellcheck ci

* fix(cluster): harden metadata routing

- Preserve legacy routing, hook order and typed results.
- Handle topology health and invalid metadata safely.
- Reject fan-out from ordered async pipelines before dispatch.
- Reduce allocations and duplicate routing work.
- Add regression coverage and document remaining limitations.

* fix(cluster): correct routing and cursor ownership

- Handle unavailable nodes and malformed topology replies
- Bound metadata verification and retry refreshes in the background
- Bind client-local cursor tokens to their originating nodes
- Validate routing policies and refresh topology after redirects
- Reduce routing allocations and extend tests and documentation

* fix spellcheck ci

* fix ci

* fix ci

* refactor(cluster): split cursor ownership

* fix cursor bug

* fix ci
2026-10-21 17:34:41 +03:00

6133 lines
186 KiB
Go

// Code consolidated from per-topic autopipeline test files.
package redis_test
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"math/rand"
"net"
"os"
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
. "github.com/bsm/ginkgo/v2"
. "github.com/bsm/gomega"
"github.com/redis/go-redis/v9"
"github.com/redis/go-redis/v9/internal/hashtag"
"github.com/redis/go-redis/v9/internal/routing"
)
// apTestAddr is the redis address for the plain go tests in this file:
// REDIS_PORT if set, else the stack default :6379. The suite-level redisAddr
// variable is NOT safe here — it defaults to :6380 and is corrected only by
// the ginkgo BeforeSuite, which plain go tests may run without.
func apTestAddr() string {
if p := os.Getenv("REDIS_PORT"); p != "" {
return ":" + p
}
return ":6379"
}
// apRELoadOnce guards a lazy, one-time load of the Redis Enterprise connection
// for the plain go tests in this file. redisOptions() resolves reConn in the
// ginkgo BeforeSuite, but plain TestXxx funcs may run before it, so
// apTestOptions() loads it on first use instead.
var apRELoadOnce sync.Once
// apTestOptions returns the base options for the plain go tests: the resolved
// Redis Enterprise database (address, credentials, TLS) when running against an
// external RE cluster (RE_CLUSTER=true and REDIS_ENDPOINTS_CONFIG_PATH set),
// otherwise Addr: apTestAddr(). Unlike redisOptions() it does not depend on
// BeforeSuite, so it is safe to call from plain TestXxx funcs. Callers that need
// extra options mutate the returned struct before NewClient.
func apTestOptions() *redis.Options {
apRELoadOnce.Do(func() {
if reConn != nil {
return
}
if re, _ := strconv.ParseBool(os.Getenv("RE_CLUSTER")); re && os.Getenv("REDIS_ENDPOINTS_CONFIG_PATH") != "" {
reConn, _ = loadREConnection()
}
})
opt := &redis.Options{Addr: apTestAddr()}
applyREConnection(opt) // no-op when reConn is nil (non-RE runs)
return opt
}
// apBaselineRTT returns a conservative baseline round-trip time to the test
// server so latency-bound assertions can scale to the endpoint (localhost vs a
// remote Redis Enterprise database) instead of assuming a fixed local bound. It
// warms the connection, then returns the minimum of a few PINGs (least noisy);
// on error it returns 0 and callers fall back to their fixed floor.
func apBaselineRTT(ctx context.Context, c *redis.Client) time.Duration {
_ = c.Ping(ctx).Err() // warm dial + handshake
best := time.Duration(0)
for i := 0; i < 5; i++ {
start := time.Now()
if err := c.Ping(ctx).Err(); err != nil {
return 0
}
if d := time.Since(start); best == 0 || d < best {
best = d
}
}
return best
}
// apLatencyBudget scales a per-round-trip time budget to the measured RTT, with
// a fixed floor so localhost (near-zero RTT) still enforces a tight ceiling.
// roundTrips is the number of sequential server round trips the timed section
// makes; the 2x factor plus 20ms leaves headroom for scheduling/jitter without
// masking a real per-command coalescing regression.
func apLatencyBudget(rtt time.Duration, roundTrips int, floor time.Duration) time.Duration {
scaled := time.Duration(roundTrips)*rtt*2 + 20*time.Millisecond
if scaled < floor {
return floor
}
return scaled
}
// ===== from autopipeline_accessor_test.go =====
// TestAutoPipelineSecondaryAccessors guards a regression where the deferred
// await() was injected only on the canonical Val()/Result() of each command
// type, leaving secondary value-accessors (Int, Bytes, Bool, Float, the generic
// Cmd type, SliceCmd.Val, map getters, ...) reading cmd.val before the batch
// executed. Each accessor below must block until its command runs and return the
// real value, not the zero value. Run under -race to catch the missing barrier.
func TestAutoPipelineSecondaryAccessors(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
c.FlushDB(ctx)
// The deferred face is where secondary accessors must block until the
// batch executes (the blocking face waits inside the call itself, so it
// could never catch a missing await barrier).
ap, err := c.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const G = 50
var wg sync.WaitGroup
var bad int64
fail := func(cond bool) {
if cond {
atomic.AddInt64(&bad, 1)
}
}
wg.Add(G)
for g := 0; g < G; g++ {
go func(id int) {
defer wg.Done()
ik := fmt.Sprintf("acc:int:%d", id)
sk := fmt.Sprintf("acc:str:%d", id)
hk := fmt.Sprintf("acc:hash:%d", id)
// Seed values via the autopipeline (these return *StatusCmd/*IntCmd).
ap.Set(ctx, sk, "12345", 0)
ap.Set(ctx, ik, "0", 0)
ap.HSet(ctx, hk, "f1", "v1", "f2", "v2")
// StringCmd secondary converters — previously missing await().
n, err := ap.Get(ctx, sk).Int()
fail(err != nil || n != 12345)
i64, err := ap.Get(ctx, sk).Int64()
fail(err != nil || i64 != 12345)
b, err := ap.Get(ctx, sk).Bytes()
fail(err != nil || string(b) != "12345")
// IntCmd path + IntCmd.Uint64 (previously missing await()).
incr := ap.Incr(ctx, ik)
u, err := incr.Uint64()
fail(err != nil || u != 1)
// Generic Cmd via Do() — the whole *Cmd type was missing await().
doCmd := ap.Do(ctx, "GET", sk)
v, err := doCmd.Int()
fail(err != nil || v != 12345)
txt, err := ap.Do(ctx, "GET", sk).Text()
fail(err != nil || txt != "12345")
// SliceCmd.Val()/Result() — previously missing await().
mget := ap.MGet(ctx, sk, ik)
vals := mget.Val()
fail(len(vals) != 2 || vals[0] != "12345")
// MapStringStringCmd via HGetAll (Val()/Result() are covered, but
// exercise it as the typical map path).
m, err := ap.HGetAll(ctx, hk).Result()
fail(err != nil || m["f1"] != "v1" || m["f2"] != "v2")
}(g)
}
wg.Wait()
if bad > 0 {
t.Fatalf("secondary accessors: %d wrong/zero reads (await regression)", bad)
}
}
// ===== from autopipeline_backpressure_test.go =====
// TestAutoPipelineWindowedNoDeadlock submits a large window of commands without
// waiting between them (the async producer pattern), then reads them all. It must
// not deadlock and every future must resolve with its correct result — exercises
// the enqueue/permit/flush path under a producer that outruns per-command waits.
func TestAutoPipelineWindowedNoDeadlock(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.PoolSize = 20
client := redis.NewClient(opt)
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 300, MaxConcurrentBatches: 8, Unordered: true,
})
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const n = 5000
futs := make([]redis.AutoFuture, n)
for i := 0; i < n; i++ {
futs[i] = ap.Submit(ctx, redis.NewCmd(ctx, "set", fmt.Sprintf("bp:%d", i), i))
}
for i := range futs {
if err := futs[i].Wait(); err != nil {
t.Fatalf("future %d: %v", i, err)
}
}
// spot-check a few landed correctly
for _, i := range []int{0, n / 2, n - 1} {
if v, _ := client.Get(ctx, fmt.Sprintf("bp:%d", i)).Result(); v != fmt.Sprintf("%d", i) {
t.Fatalf("bp:%d = %q, want %d", i, v, i)
}
}
for i := 0; i < n; i++ {
client.Del(ctx, fmt.Sprintf("bp:%d", i))
}
}
// TestAutoPipelineSoak drives continuous autopipelined traffic from many
// goroutines for a few seconds and checks that goroutine and connection counts
// return to a stable baseline afterward (no slow leak) and results stay correct.
// Skipped under -short; bump `dur` for a longer soak.
// settleGoroutines GCs and polls runtime.NumGoroutine until it drops to
// limit or a deadline passes, returning the last observed count. Replaces
// fixed post-GC sleeps, which false-positive on slow CI.
func settleGoroutines(limit int) int {
deadline := time.Now().Add(5 * time.Second)
got := 0
for {
runtime.GC()
got = runtime.NumGoroutine()
if got <= limit || time.Now().After(deadline) {
return got
}
time.Sleep(50 * time.Millisecond)
}
}
func TestAutoPipelineSoak(t *testing.T) {
if testing.Short() {
t.Skip("soak skipped in -short")
}
ctx := context.Background()
opt := apTestOptions()
opt.PoolSize = 20
client := redis.NewClient(opt)
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 200, MaxConcurrentBatches: 16, Unordered: true,
})
if err != nil {
t.Fatal(err)
}
defer ap.Close()
runtime.GC()
time.Sleep(100 * time.Millisecond)
baseGoroutines := runtime.NumGoroutine()
const dur = 3 * time.Second
deadline := time.Now().Add(dur)
var wg sync.WaitGroup
var mismatch int64
var mu sync.Mutex
for g := 0; g < 32; g++ {
wg.Add(1)
go func(id int) {
defer wg.Done()
i := 0
for time.Now().Before(deadline) {
key := fmt.Sprintf("soak:%d:%d", id, i%16)
val := fmt.Sprintf("%d", i)
setF := ap.Set(ctx, key, val, 0)
if err := setF.Err(); err != nil {
mu.Lock()
mismatch++
mu.Unlock()
}
i++
}
}(g)
}
wg.Wait()
if mismatch != 0 {
t.Fatalf("%d errors during soak", mismatch)
}
got := settleGoroutines(baseGoroutines + 16)
if got > baseGoroutines+16 {
t.Fatalf("goroutine leak: baseline=%d after soak=%d", baseGoroutines, got)
}
}
// ===== from autopipeline_blocking_test.go =====
var _ = Describe("AutoPipeline Blocking Commands", func() {
ctx := context.Background()
var client *redis.Client
var ap *redis.AutoPipeliner
BeforeEach(func() {
client = redis.NewClient(redisOptions())
Expect(client.FlushDB(ctx).Err()).NotTo(HaveOccurred())
var err error
ap, err = client.AutoPipeline()
Expect(err).NotTo(HaveOccurred())
})
AfterEach(func() {
if ap != nil {
Expect(ap.Close()).NotTo(HaveOccurred())
}
Expect(client.Close()).NotTo(HaveOccurred())
})
It("should not autopipeline blocking commands", func() {
// Push a value to the list
Expect(client.RPush(ctx, "list", "value").Err()).NotTo(HaveOccurred())
// BLPOP should execute immediately without autopipelining.
// ap.Do returns a generic *redis.Cmd; use its typed accessors.
rtt := apBaselineRTT(ctx, client)
start := time.Now()
result := ap.Do(ctx, "BLPOP", "list", "1")
val, err := result.StringSlice()
elapsed := time.Since(start)
Expect(err).NotTo(HaveOccurred())
Expect(val).To(Equal([]string{"list", "value"}))
// Value is available, so this is ~one round trip; bound scales to the
// measured RTT (100ms floor for localhost).
Expect(elapsed).To(BeNumerically("<", apLatencyBudget(rtt, 1, 100*time.Millisecond)))
})
It("should mix blocking and non-blocking commands", func() {
// Push values
Expect(client.RPush(ctx, "list3", "a", "b", "c").Err()).NotTo(HaveOccurred())
Expect(client.Set(ctx, "key1", "value1", 0).Err()).NotTo(HaveOccurred())
// Mix blocking and non-blocking commands
blpopCmd := ap.Do(ctx, "BLPOP", "list3", "1")
getCmd := ap.Do(ctx, "GET", "key1")
brpopCmd := ap.Do(ctx, "BRPOP", "list3", "1")
// Get results — ap.Do returns generic *redis.Cmd; use typed accessors.
blpopVal, err := blpopCmd.StringSlice()
Expect(err).NotTo(HaveOccurred())
Expect(blpopVal).To(Equal([]string{"list3", "a"}))
getVal, err := getCmd.Text()
Expect(err).NotTo(HaveOccurred())
Expect(getVal).To(Equal("value1"))
brpopVal, err := brpopCmd.StringSlice()
Expect(err).NotTo(HaveOccurred())
Expect(brpopVal).To(Equal([]string{"list3", "c"}))
})
})
// ===== from autopipeline_buffer_test.go =====
// Autopipeline + zero-copy GetToBuffer/SetFromBuffer, standalone, concurrent.
func TestAPZeroCopyBuffer(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.AutoPipelineOptions = &redis.AutoPipelineOptions{MaxBatchSize: 100, MaxConcurrentBatches: 30, Unordered: true}
c := redis.NewClient(opt)
defer c.Close()
c.FlushDB(ctx)
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const g = 100
const perG = 30
var wg sync.WaitGroup
var bad int64
var mu sync.Mutex
var errs []string
wg.Add(g)
for x := 0; x < g; x++ {
go func(id int) {
defer wg.Done()
for i := 0; i < perG; i++ {
key := fmt.Sprintf("buf:%d:%d", id, i)
val := fmt.Sprintf("payload-%d-%d", id, i)
// write from buffer via autopipeline
if err := ap.SetFromBuffer(ctx, key, []byte(val)).Err(); err != nil {
atomic.AddInt64(&bad, 1)
mu.Lock()
if len(errs) < 10 {
errs = append(errs, "set:"+err.Error())
}
mu.Unlock()
continue
}
// read into caller buffer via autopipeline
rbuf := make([]byte, len(val))
cmd := ap.GetToBuffer(ctx, key, rbuf)
n, err := cmd.Result()
if err != nil {
atomic.AddInt64(&bad, 1)
mu.Lock()
if len(errs) < 10 {
errs = append(errs, "get:"+err.Error())
}
mu.Unlock()
continue
}
got := string(cmd.Bytes())
if n != len(val) || got != val {
atomic.AddInt64(&bad, 1)
mu.Lock()
if len(errs) < 10 {
errs = append(errs, fmt.Sprintf("key %s n=%d got=%q want=%q", key, n, got, val))
}
mu.Unlock()
}
}
}(x)
}
wg.Wait()
if bad > 0 {
t.Fatalf("zero-copy buffer via autopipeline failed: %d cases; sample=%v", bad, errs)
}
}
// ===== from autopipeline_close_race_test.go =====
// TestAutoPipelineCloseRace stresses concurrent command submission racing with
// Close(). The hazard: a submit that reads closed==false, then stalls, then
// enqueues onto a shard whose flusher has already drained and exited during
// shutdown — leaving the command's completion signal never fired, so the
// caller's blocking accessor hangs forever. Every submitted command must
// resolve (success or ErrClosed), never block past the deadline.
//
// Best-effort: this is a stress test, not a deterministic reproducer. The exact
// interleaving (a submit preempted between the closed-check and the enqueue for
// the whole duration of Close) is narrow and may not trigger every run; the fix
// (re-checking closed under the shard lock) is justified by code analysis. The
// test guards against regressions of the obvious form and runs many iterations
// to widen the window.
func TestAutoPipelineCloseRace(t *testing.T) {
for iter := 0; iter < 50; iter++ {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
const G = 64
var wg sync.WaitGroup
wg.Add(G)
start := make(chan struct{})
for g := 0; g < G; g++ {
go func() {
defer wg.Done()
<-start
for i := 0; i < 20; i++ {
// .Err() blocks until the command resolves. If the command
// is lost (never signalled), this never returns and the
// watchdog below fires.
_ = ap.Set(ctx, "closerace", i, 0).Err()
}
}()
}
close(start)
// Close concurrently with in-flight submits — the race window.
time.Sleep(time.Millisecond)
_ = ap.Close()
// Watchdog: all submitters must finish promptly. A hung await() means a
// lost command (the bug this guards).
done := make(chan struct{})
go func() { wg.Wait(); close(done) }()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatalf("iter %d: submitters hung after Close — a command was lost", iter)
}
c.Close()
}
}
// ===== from autopipeline_close_wiring_test.go =====
// TestClientCloseClosesAutoPipeliner verifies that closing the client also
// closes the shared AutoPipeliner, so its background flusher goroutines don't
// outlive the client. IsClosed() is the direct signal — checking it (rather than
// a downstream command error, which could just reflect the closed pool) proves
// Client.Close stopped the pipeliner itself. Without the wiring in Client.Close
// the autopipeliner would report open after the client is gone.
func TestClientCloseClosesAutoPipeliner(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
c.FlushDB(ctx)
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
if err := ap.Set(ctx, "cw", "1", 0).Err(); err != nil {
t.Fatalf("set via autopipeline before close: %v", err)
}
if ap.IsClosed() {
t.Fatal("autopipeliner reported closed before client.Close")
}
if err := c.Close(); err != nil {
t.Fatalf("client close: %v", err)
}
if !ap.IsClosed() {
t.Fatal("client.Close did not close the shared autopipeliner")
}
// A second client Close must not panic; like the base client it reports the
// already-closed error (the autopipeliner part is idempotent and nil here).
if err := c.Close(); err != nil && err != redis.ErrClosed {
t.Fatalf("second client close: unexpected error %v", err)
}
}
// ===== from autopipeline_cmdable_test.go =====
var _ = Describe("AutoPipeline Cmdable Interface", func() {
ctx := context.Background()
var client *redis.Client
var ap *redis.AutoPipeliner
BeforeEach(func() {
client = redis.NewClient(redisOptions())
Expect(client.FlushDB(ctx).Err()).NotTo(HaveOccurred())
var err error
ap, err = client.AutoPipeline()
Expect(err).NotTo(HaveOccurred())
})
AfterEach(func() {
if ap != nil {
Expect(ap.Close()).NotTo(HaveOccurred())
}
Expect(client.Close()).NotTo(HaveOccurred())
})
It("should support string commands", func() {
// Use autopipeline like a regular client
setCmd := ap.Set(ctx, "key1", "value1", 0)
getCmd := ap.Get(ctx, "key1")
incrCmd := ap.Incr(ctx, "counter")
decrCmd := ap.Decr(ctx, "counter")
// Get results
Expect(setCmd.Err()).NotTo(HaveOccurred())
Expect(setCmd.Val()).To(Equal("OK"))
val, err := getCmd.Result()
Expect(err).NotTo(HaveOccurred())
Expect(val).To(Equal("value1"))
Expect(incrCmd.Val()).To(Equal(int64(1)))
Expect(decrCmd.Val()).To(Equal(int64(0)))
})
It("should support hash commands", func() {
// Use hash commands
hsetCmd := ap.HSet(ctx, "hash1", "field1", "value1", "field2", "value2")
hgetCmd := ap.HGet(ctx, "hash1", "field1")
hgetallCmd := ap.HGetAll(ctx, "hash1")
// Get results
Expect(hsetCmd.Val()).To(Equal(int64(2)))
Expect(hgetCmd.Val()).To(Equal("value1"))
Expect(hgetallCmd.Val()).To(Equal(map[string]string{
"field1": "value1",
"field2": "value2",
}))
})
It("should support list commands", func() {
// Use list commands
rpushCmd := ap.RPush(ctx, "list1", "a", "b", "c")
lrangeCmd := ap.LRange(ctx, "list1", 0, -1)
lpopCmd := ap.LPop(ctx, "list1")
// Get results
Expect(rpushCmd.Val()).To(Equal(int64(3)))
Expect(lrangeCmd.Val()).To(Equal([]string{"a", "b", "c"}))
Expect(lpopCmd.Val()).To(Equal("a"))
})
It("should support set commands", func() {
// Use set commands
saddCmd := ap.SAdd(ctx, "set1", "member1", "member2", "member3")
smembersCmd := ap.SMembers(ctx, "set1")
sismemberCmd := ap.SIsMember(ctx, "set1", "member1")
// Get results
Expect(saddCmd.Val()).To(Equal(int64(3)))
Expect(smembersCmd.Val()).To(ConsistOf("member1", "member2", "member3"))
Expect(sismemberCmd.Val()).To(BeTrue())
})
It("should support sorted set commands", func() {
// Use sorted set commands
zaddCmd := ap.ZAdd(
ctx, "zset1",
redis.Z{Score: 1, Member: "one"},
redis.Z{Score: 2, Member: "two"},
redis.Z{Score: 3, Member: "three"},
)
zrangeCmd := ap.ZRange(ctx, "zset1", 0, -1)
zscoreCmd := ap.ZScore(ctx, "zset1", "two")
// Get results
Expect(zaddCmd.Val()).To(Equal(int64(3)))
Expect(zrangeCmd.Val()).To(Equal([]string{"one", "two", "three"}))
Expect(zscoreCmd.Val()).To(Equal(float64(2)))
})
It("should support generic commands", func() {
// Set some keys
ap.Set(ctx, "key1", "value1", 0)
ap.Set(ctx, "key2", "value2", 0)
ap.Set(ctx, "key3", "value3", 0)
// Use generic commands
existsCmd := ap.Exists(ctx, "key1", "key2", "key3")
delCmd := ap.Del(ctx, "key1")
ttlCmd := ap.TTL(ctx, "key2")
// Get results
Expect(existsCmd.Val()).To(Equal(int64(3)))
Expect(delCmd.Val()).To(Equal(int64(1)))
Expect(ttlCmd.Val()).To(Equal(time.Duration(-1))) // No expiration
})
It("should support Do method for custom commands", func() {
// Use Do for custom commands
setCmd := ap.Do(ctx, "SET", "custom_key", "custom_value")
getCmd := ap.Do(ctx, "GET", "custom_key")
// Get results
setVal, err := setCmd.Result()
Expect(err).NotTo(HaveOccurred())
Expect(setVal).To(Equal("OK"))
getVal, err := getCmd.Result()
Expect(err).NotTo(HaveOccurred())
Expect(getVal).To(Equal("custom_value"))
})
It("should support Pipeline method", func() {
// Get a traditional pipeline from autopipeliner
pipe := ap.Pipeline()
Expect(pipe).NotTo(BeNil())
// Use the pipeline
pipe.Set(ctx, "pipe_key", "pipe_value", 0)
pipe.Get(ctx, "pipe_key")
cmds, err := pipe.Exec(ctx)
Expect(err).NotTo(HaveOccurred())
Expect(cmds).To(HaveLen(2))
})
It("should support Pipelined method", func() {
// Use Pipelined for convenience
cmds, err := ap.Pipelined(ctx, func(pipe redis.Pipeliner) error {
pipe.Set(ctx, "pipelined_key", "pipelined_value", 0)
pipe.Get(ctx, "pipelined_key")
return nil
})
Expect(err).NotTo(HaveOccurred())
Expect(cmds).To(HaveLen(2))
Expect(cmds[0].(*redis.StatusCmd).Val()).To(Equal("OK"))
Expect(cmds[1].(*redis.StringCmd).Val()).To(Equal("pipelined_value"))
})
It("should mix autopipelined and direct commands", func() {
// Use autopipeline commands
ap.Set(ctx, "ap_key1", "ap_value1", 0)
ap.Set(ctx, "ap_key2", "ap_value2", 0)
// Use traditional pipeline
pipe := ap.Pipeline()
pipe.Set(ctx, "pipe_key1", "pipe_value1", 0)
pipe.Set(ctx, "pipe_key2", "pipe_value2", 0)
_, err := pipe.Exec(ctx)
Expect(err).NotTo(HaveOccurred())
// Verify all keys exist
val1, _ := ap.Get(ctx, "ap_key1").Result()
val2, _ := ap.Get(ctx, "ap_key2").Result()
val3, _ := ap.Get(ctx, "pipe_key1").Result()
val4, _ := ap.Get(ctx, "pipe_key2").Result()
Expect(val1).To(Equal("ap_value1"))
Expect(val2).To(Equal("ap_value2"))
Expect(val3).To(Equal("pipe_value1"))
Expect(val4).To(Equal("pipe_value2"))
})
})
// ===== from autopipeline_config_test.go =====
// TestAutoPipelineOptionsNotMutated guards a regression where the autopipeliner
// filled zero-value defaults (MaxBatchSize, MaxConcurrentBatches) directly into
// the caller's *AutoPipelineOptions. A config shared across clients (or inspected
// later by the caller) must not be mutated by creating an AutoPipeliner.
func TestAutoPipelineOptionsNotMutated(t *testing.T) {
cfg := &redis.AutoPipelineOptions{} // all zero -> defaults applied internally
c := redis.NewClient(&redis.Options{Addr: apTestAddr(), AutoPipelineOptions: cfg})
defer c.Close()
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
if cfg.MaxBatchSize != 0 {
t.Fatalf("caller config MaxBatchSize mutated to %d (want 0)", cfg.MaxBatchSize)
}
if cfg.MaxConcurrentBatches != 0 {
t.Fatalf("caller config MaxConcurrentBatches mutated to %d (want 0)", cfg.MaxConcurrentBatches)
}
// The same config reused for a second client must still be pristine and
// usable (defaults applied to an internal copy, not the shared struct).
c2 := redis.NewClient(&redis.Options{Addr: apTestAddr(), AutoPipelineOptions: cfg})
defer c2.Close()
ap2, err := c2.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap2.Close()
if cfg.MaxBatchSize != 0 || cfg.MaxConcurrentBatches != 0 {
t.Fatal("shared config mutated after second client created an AutoPipeliner")
}
}
// ===== from autopipeline_conn_state_test.go =====
// TestAutoPipelineBlockingCommandIsolation verifies a blocking command (detected
// via readTimeout at the submit gate, autopipeline.go:679) runs outside the
// pipeline and does NOT corrupt batched commands interleaved around it: the
// batched SET/GET before and after the BLPOP all return their correct replies.
func TestAutoPipelineBlockingCommandIsolation(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
client.Del(ctx, "cs:list", "cs:a", "cs:b")
if err := client.RPush(ctx, "cs:list", "item").Err(); err != nil {
t.Fatal(err)
}
ap, err := client.AutoPipeline() // blocking face
if err != nil {
t.Fatal(err)
}
defer ap.Close()
if v := ap.Set(ctx, "cs:a", "1", 0).Val(); v != "OK" {
t.Fatalf("set a before blocking: %q", v)
}
// Blocking command interleaved with batched work.
blp, err := ap.BLPop(ctx, time.Second, "cs:list").Result()
if err != nil {
t.Fatalf("blpop: %v", err)
}
if len(blp) != 2 || blp[1] != "item" {
t.Fatalf("blpop = %v, want [cs:list item]", blp)
}
// Batched work after a blocking command still works and stays uncorrupted.
if v := ap.Set(ctx, "cs:b", "2", 0).Val(); v != "OK" {
t.Fatalf("set b after blocking: %q", v)
}
if v := ap.Get(ctx, "cs:a").Val(); v != "1" {
t.Fatalf("get a after blocking: %q, want 1", v)
}
if v := ap.Get(ctx, "cs:b").Val(); v != "2" {
t.Fatalf("get b after blocking: %q, want 2", v)
}
client.Del(ctx, "cs:a", "cs:b")
}
// TestAutoPipelineDoubleClose verifies Close is idempotent — a second Close does
// not panic (it may return an error, but must not crash or hang).
func TestAutoPipelineDoubleClose(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
ap, err := client.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
if err := ap.Close(); err != nil {
t.Fatalf("first close: %v", err)
}
// Second close must not panic; ErrClosed or nil are both acceptable.
_ = ap.Close()
}
// ===== from autopipeline_context_test.go =====
// AutoPipeline context contract (previously untested; WaitContext had zero
// coverage). The documented contract:
// - a caller's context is NOT honored once a command is enqueued (execution
// runs on context.Background); it only gates the wait via WaitContext.
// - WaitContext abandons the wait on ctx done, but the command still executes
// and its result becomes readable via a later Wait.
// - blocking commands (readTimeout != nil) run directly with the caller ctx
// and DO honor it.
// AP-C1: a caller ctx cancelled after enqueue does not cancel the command; the
// result is still produced.
func TestAPContextCancelDoesNotCancelExecution(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
ap, err := client.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
cctx, cancel := context.WithCancel(ctx)
f := ap.Set(cctx, "apctx:k", "v", 0)
cancel() // cancel after submit, before the batch flushes/completes
if err := f.Err(); err != nil {
t.Fatalf("command should still execute after caller ctx cancel, got %v", err)
}
if f.Val() != "OK" {
t.Fatalf("val = %q, want OK", f.Val())
}
if v, _ := client.Get(ctx, "apctx:k").Result(); v != "v" {
t.Fatalf("value not persisted: %q", v)
}
client.Del(ctx, "apctx:k")
}
// AP-C2: WaitContext returns ctx.Err while the batch is in flight, and a later
// Wait returns the real result.
func TestAPWaitContextAbandonsWaitOnly(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
// Hold the batch open long enough that the WaitContext deadline fires first.
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{MaxBatchSize: 300, MaxFlushDelay: 500 * time.Millisecond})
if err != nil {
t.Fatal(err)
}
defer ap.Close()
f := ap.Submit(ctx, redis.NewCmd(ctx, "set", "apctx:wc", "v"))
wctx, cancel := context.WithTimeout(ctx, 50*time.Millisecond)
defer cancel()
if err := f.WaitContext(wctx); err != context.DeadlineExceeded {
t.Fatalf("WaitContext err = %v, want DeadlineExceeded (batch held open)", err)
}
// The command still completes; a plain Wait returns the real result.
if err := f.Wait(); err != nil {
t.Fatalf("Wait after WaitContext timeout: %v", err)
}
client.Del(ctx, "apctx:wc")
}
// AP-C3: WaitContext returns the real result when the batch completes first.
func TestAPWaitContextReturnsResult(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
ap, err := client.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
f := ap.Submit(ctx, redis.NewCmd(ctx, "set", "apctx:ok", "v"))
if err := f.WaitContext(ctx); err != nil {
t.Fatalf("WaitContext: %v", err)
}
client.Del(ctx, "apctx:ok")
}
// AP-C4: a blocking command runs directly with the caller ctx (not batched on
// Background). With ContextTimeoutEnabled the ctx deadline reaches the read, so a
// BLPOP is cut short by the ctx — proving the caller ctx flows to the blocking
// path (unlike batched commands, which execute on Background).
func TestAPBlockingCommandHonorsContext(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.ContextTimeoutEnabled = true
client := redis.NewClient(opt)
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
ap, err := client.AutoPipeline() // blocking face
if err != nil {
t.Fatal(err)
}
defer ap.Close()
client.Del(ctx, "apctx:blk")
bctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
defer cancel()
start := time.Now()
// BLPOP on an empty list blocks; the ctx deadline must cut it short.
err = ap.BLPop(bctx, 5*time.Second, "apctx:blk").Err()
elapsed := time.Since(start)
if err == nil {
t.Fatal("expected BLPOP to fail on ctx deadline")
}
if elapsed > 2*time.Second {
t.Fatalf("BLPOP did not honor ctx deadline (took %v)", elapsed)
}
}
// AP-C5: submitting after Close returns ErrClosed.
func TestAPSubmitAfterCloseErrClosed(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
ap, err := client.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
if err := ap.Close(); err != nil {
t.Fatalf("close: %v", err)
}
f := ap.Set(ctx, "apctx:closed", "v", 0)
if err := f.Err(); err != redis.ErrClosed {
t.Fatalf("submit after close err = %v, want ErrClosed", err)
}
}
// AP-C6: WaitContext on a zero AutoFuture returns an error, not a panic.
func TestAPZeroFutureWaitContext(t *testing.T) {
var f redis.AutoFuture
if err := f.WaitContext(context.Background()); err == nil {
t.Fatal("zero AutoFuture WaitContext should return an error")
}
}
// ===== from autopipeline_correctness_test.go =====
// Validates: each command gets its OWN result (no cross-talk), under heavy
// concurrency. Each goroutine writes a unique value, reads it back, asserts it
// got exactly what it wrote.
func TestAPNoCrossTalk(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.AutoPipelineOptions = &redis.AutoPipelineOptions{MaxBatchSize: 200, MaxConcurrentBatches: 50, Unordered: true}
c := redis.NewClient(opt)
defer c.Close()
c.FlushDB(ctx)
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const goroutines = 200
const perG = 50
var wg sync.WaitGroup
var mismatches int64
var mu sync.Mutex
errs := []string{}
wg.Add(goroutines)
for g := 0; g < goroutines; g++ {
go func(id int) {
defer wg.Done()
for i := 0; i < perG; i++ {
key := fmt.Sprintf("ct:%d:%d", id, i)
want := fmt.Sprintf("v-%d-%d", id, i)
// SET then GET via autopipeline; assert GET returns our value.
if err := ap.Set(ctx, key, want, 0).Err(); err != nil {
mu.Lock()
errs = append(errs, fmt.Sprintf("set %s: %v", key, err))
mu.Unlock()
continue
}
got, err := ap.Get(ctx, key).Result()
if err != nil {
mu.Lock()
errs = append(errs, fmt.Sprintf("get %s: %v", key, err))
mu.Unlock()
continue
}
if got != want {
mu.Lock()
mismatches++
if len(errs) < 20 {
errs = append(errs, fmt.Sprintf("key %s: got %q want %q", key, got, want))
}
mu.Unlock()
}
}
}(g)
}
wg.Wait()
if mismatches > 0 || len(errs) > 0 {
t.Fatalf("cross-talk detected: %d mismatches, errs=%v", mismatches, errs)
}
}
// Validates: per-goroutine ordering. Each goroutine does INCR on its own key N
// times; final value must equal N and intermediate results must be strictly
// 1,2,3,... in call order (blocking Do guarantees this only if results map to
// the right command).
func TestAPPerGoroutineOrder(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.AutoPipelineOptions = &redis.AutoPipelineOptions{MaxBatchSize: 100, MaxConcurrentBatches: 30, Unordered: true}
c := redis.NewClient(opt)
defer c.Close()
c.FlushDB(ctx)
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const goroutines = 100
const incrs = 100
var wg sync.WaitGroup
var bad int64
wg.Add(goroutines)
for g := 0; g < goroutines; g++ {
go func(id int) {
defer wg.Done()
key := fmt.Sprintf("ord:%d", id)
for i := 1; i <= incrs; i++ {
v, err := ap.Incr(ctx, key).Result()
if err != nil || v != int64(i) {
atomicAdd(&bad)
}
}
}(g)
}
wg.Wait()
if bad > 0 {
t.Fatalf("ordering violated on %d INCR results (expected strict 1..N per goroutine)", bad)
}
}
// Validates: no lost commands. Issue exactly N writes across goroutines, then
// count keys; must be exactly N.
func TestAPNoLostCommands(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.AutoPipelineOptions = &redis.AutoPipelineOptions{MaxBatchSize: 256, MaxConcurrentBatches: 64, Unordered: true}
c := redis.NewClient(opt)
defer c.Close()
c.FlushDB(ctx)
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const goroutines = 300
const perG = 100
var wg sync.WaitGroup
wg.Add(goroutines)
for g := 0; g < goroutines; g++ {
go func(id int) {
defer wg.Done()
for i := 0; i < perG; i++ {
ap.Set(ctx, fmt.Sprintf("lost:%d:%d", id, i), "1", 0)
}
}(g)
}
wg.Wait()
n, err := c.DBSize(ctx).Result()
if err != nil {
t.Fatal(err)
}
want := int64(goroutines * perG)
if n != want {
t.Fatalf("lost commands: DBSize=%d want=%d (%d missing)", n, want, want-n)
}
}
// Validates: error isolation. A WRONGTYPE error on one command must not
// corrupt sibling commands sharing the same batch.
func TestAPErrorIsolation(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.AutoPipelineOptions = &redis.AutoPipelineOptions{MaxBatchSize: 64, MaxConcurrentBatches: 16, Unordered: true}
c := redis.NewClient(opt)
defer c.Close()
c.FlushDB(ctx)
// Seed a list key so INCR on it errors with WRONGTYPE.
c.RPush(ctx, "alist", "x")
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const goroutines = 50
var wg sync.WaitGroup
var goodFail, badNeighbor int64
wg.Add(goroutines)
for g := 0; g < goroutines; g++ {
go func(id int) {
defer wg.Done()
// Good neighbor command (own key)
ok := fmt.Sprintf("iso:%d", id)
// Interleave a guaranteed-error command (INCR on a list) with a good one.
errCmd := ap.Incr(ctx, "alist")
goodCmd := ap.Set(ctx, ok, strconv.Itoa(id), 0)
if errCmd.Err() == nil {
atomicAdd(&badNeighbor) // the bad command should have errored
} else {
atomicAdd(&goodFail)
}
if goodCmd.Err() != nil {
atomicAdd(&badNeighbor) // the good command must NOT inherit the error
}
}(g)
}
wg.Wait()
if badNeighbor > 0 {
t.Fatalf("error isolation broken: %d cases (good cmd errored or bad cmd succeeded)", badNeighbor)
}
if goodFail != goroutines {
t.Fatalf("expected all %d INCR-on-list to error, got %d", goroutines, goodFail)
}
}
func atomicAdd(p *int64) {
atomic.AddInt64(p, 1)
}
// ===== from autopipeline_faces_split_test.go =====
// TestBlockingFaceExecutesOnCall: AutoPipeline() command blocks until executed —
// the returned cmd already holds its result without an explicit further wait.
func TestBlockingFaceExecutesOnCall(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
c.FlushDB(ctx)
ap, err := c.AutoPipeline() // blocking face
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const G, N = 50, 100
var wg sync.WaitGroup
var bad int
var mu sync.Mutex
wg.Add(G)
for g := 0; g < G; g++ {
go func(id int) {
defer wg.Done()
key := func() string { return "blk:" + string(rune('a'+id%26)) }()
_ = key
for i := 1; i <= N; i++ {
// Incr blocks until executed; per-goroutine order must hold.
cmd := ap.Incr(ctx, "blkkey:"+string(rune(id)))
if v, err := cmd.Result(); err != nil || v != int64(i) {
mu.Lock()
bad++
mu.Unlock()
}
}
}(g)
}
wg.Wait()
if bad > 0 {
t.Fatalf("blocking face: %d ordering/exec failures", bad)
}
}
// TestAsyncFaceDeferred: AsyncAutoPipeline() command returns immediately; result
// is read later. Windowed submit then read works.
func TestAsyncFaceDeferred(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
c.FlushDB(ctx)
ap, err := c.AsyncAutoPipeline() // deferred face, ordered default
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const N = 500
cmds := make([]*redis.IntCmd, 0, N)
for i := 0; i < N; i++ {
cmds = append(cmds, ap.Incr(ctx, "asynckey")) // does not block
}
for i, cmd := range cmds {
if v, err := cmd.Result(); err != nil || v != int64(i+1) {
t.Fatalf("async windowed: cmd %d got v=%d err=%v", i, v, err)
}
}
}
// ===== from autopipeline_faces_test.go =====
// TestFutureFaceTyped verifies the future face exposes the typed command
// surface returning the usual *XxxCmd (same shape as the blocking client),
// with the result deferred until read. No cross-talk across goroutines.
func TestFutureFaceTyped(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
c.FlushDB(ctx)
fap, err := c.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
defer fap.Close()
const G, N = 100, 50
var wg sync.WaitGroup
var bad int64
wg.Add(G)
for g := 0; g < G; g++ {
go func(id int) {
defer wg.Done()
sets := make([]*redis.StatusCmd, N)
for i := 0; i < N; i++ {
sets[i] = fap.Set(ctx, fmt.Sprintf("ff:%d:%d", id, i), fmt.Sprintf("v%d-%d", id, i), 0)
}
for _, s := range sets {
if s.Err() != nil {
atomic.AddInt64(&bad, 1)
}
}
for i := 0; i < N; i++ {
want := fmt.Sprintf("v%d-%d", id, i)
if v, _ := fap.Get(ctx, fmt.Sprintf("ff:%d:%d", id, i)).Result(); v != want {
atomic.AddInt64(&bad, 1)
}
}
}(g)
}
wg.Wait()
if bad > 0 {
t.Fatalf("future face: %d failures", bad)
}
}
// TestOrderedModeWindowed verifies that MaxConcurrentBatches=1 gives a single
// ordered command stream: a windowed caller (submit many, read later) sees
// strict per-key ordering even though it never blocks between submits.
func TestOrderedModeWindowed(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
c.FlushDB(ctx)
// Deferred face: submits genuinely don't block, so the windowed claim is
// real (on the blocking face every Incr would wait, proving nothing).
fap, err := c.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{MaxBatchSize: 500, MaxConcurrentBatches: 1})
if err != nil {
t.Fatal(err)
}
defer fap.Close()
const G, N = 80, 150
var wg sync.WaitGroup
var bad int64
wg.Add(G)
for g := 0; g < G; g++ {
go func(id int) {
defer wg.Done()
key := fmt.Sprintf("om:%d", id)
cs := make([]*redis.IntCmd, 0, N)
for i := 0; i < N; i++ {
cs = append(cs, fap.Incr(ctx, key)) // windowed: no read between submits
}
for i, cc := range cs {
if v, err := cc.Result(); err != nil || v != int64(i+1) {
atomic.AddInt64(&bad, 1)
}
}
}(g)
}
wg.Wait()
if bad > 0 {
t.Fatalf("ordered windowed: %d ordering violations", bad)
}
}
// TestAutoPipelineOptionsValidate verifies the ordering-safety rule: parallel
// batches (MaxConcurrentBatches>1) require an explicit Unordered:true opt-out.
func TestAutoPipelineOptionsValidate(t *testing.T) {
// conc>1 without Unordered -> error.
if err := (&redis.AutoPipelineOptions{MaxConcurrentBatches: 8}).Validate(); err == nil {
t.Fatal("expected error for MaxConcurrentBatches>1 without Unordered")
}
// conc>1 with Unordered -> ok.
if err := (&redis.AutoPipelineOptions{MaxConcurrentBatches: 8, Unordered: true}).Validate(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// ordered single stream -> ok.
if err := (&redis.AutoPipelineOptions{MaxConcurrentBatches: 1}).Validate(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// default is ordered and valid.
if err := redis.DefaultAutoPipelineOptions().Validate(); err != nil {
t.Fatalf("default config invalid: %v", err)
}
// negative values are rejected so a typo surfaces at construction.
if err := (&redis.AutoPipelineOptions{MaxBatchSize: -1}).Validate(); err == nil {
t.Fatal("expected error for negative MaxBatchSize")
}
if err := (&redis.AutoPipelineOptions{MaxConcurrentBatches: -1}).Validate(); err == nil {
t.Fatal("expected error for negative MaxConcurrentBatches")
}
if err := (&redis.AutoPipelineOptions{MaxFlushDelay: -1}).Validate(); err == nil {
t.Fatal("expected error for negative MaxFlushDelay")
}
// zero values are allowed (mean "use default" / "no delay").
if err := (&redis.AutoPipelineOptions{}).Validate(); err != nil {
t.Fatalf("zero-value config should be valid: %v", err)
}
}
// TestAutoPipelineErrorsOnUnsafeConfig verifies the unsafe config is rejected
// with an error (not a panic): AsyncAutoPipeline runs from a post-init call, so a
// bad config surfaces as a returned error the caller can handle.
func TestAutoPipelineErrorsOnUnsafeConfig(t *testing.T) {
c := redis.NewClient(apTestOptions())
defer c.Close()
ap, err := c.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{MaxConcurrentBatches: 8})
if err == nil {
t.Fatal("expected error for MaxConcurrentBatches>1 without Unordered")
}
if ap != nil {
t.Fatal("expected nil AutoPipeliner on config error")
}
}
// ===== from autopipeline_fuzz_test.go =====
// TestAutoPipelineDifferentialFuzz runs the same randomized, type-diverse command
// stream through a plain client and diffs every result against two autopipeline
// runs: the blocking face called sequentially (which exercises the lone-command
// fast path — each call flushes solo), and the async face driven windowed —
// every command submitted before any result is read — so the stream coalesces
// into real multi-command batches and the diff covers cross-talk, ordering, and
// reply-shape demux where they can actually break. A counting hook asserts the
// async phase really formed >=2-command batches (anti-vacuousness guard).
func TestAutoPipelineDifferentialFuzz(t *testing.T) {
ctx := context.Background()
// Dedicated DB so FlushDB between runs can't touch anything else.
client := redis.NewClient(&redis.Options{Addr: apTestAddr(), DB: 15})
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
for _, seed := range []int64{1, 7, 42, 1337} {
ops := genOps(rand.New(rand.NewSource(seed)), 400)
if err := client.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
plainRes := runOps(ctx, client, ops)
if err := client.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
ap, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
apRes := runOps(ctx, ap, ops)
ap.Close()
for i := range ops {
if plainRes[i] != apRes[i] {
t.Fatalf("seed=%d op#%d %s(%s): plain=%q autopipeline=%q",
seed, i, ops[i].kind, ops[i].key, plainRes[i], apRes[i])
}
}
// Async windowed phase: submit everything, then read. The ordered
// default preserves submit order, so results must equal the
// sequential plain run — but now via real multi-command batches.
if err := client.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
batchClient := redis.NewClient(&redis.Options{Addr: apTestAddr(), DB: 15})
var realBatches atomic.Int32
batchClient.AddHook(batchCountingHook{batches: &realBatches})
asyncAP, err := batchClient.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 100,
MaxFlushDelay: 2 * time.Millisecond,
})
if err != nil {
t.Fatal(err)
}
asyncRes := runOpsWindowed(ctx, asyncAP, ops)
asyncAP.Close()
batchClient.Close()
for i := range ops {
if plainRes[i] != asyncRes[i] {
t.Fatalf("seed=%d op#%d %s(%s): plain=%q async-windowed=%q",
seed, i, ops[i].kind, ops[i].key, plainRes[i], asyncRes[i])
}
}
if realBatches.Load() == 0 {
t.Fatalf("seed=%d: async windowed run formed no >=2-command batches — differential coverage is vacuous", seed)
}
}
client.FlushDB(ctx)
}
// batchCountingHook counts pipeline dispatches of >=2 commands: proof the
// windowed fuzz phase exercised real batches rather than the solo fast path.
type batchCountingHook struct{ batches *atomic.Int32 }
func (batchCountingHook) DialHook(next redis.DialHook) redis.DialHook { return next }
func (batchCountingHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
return next
}
func (h batchCountingHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return func(ctx context.Context, cmds []redis.Cmder) error {
if len(cmds) >= 2 {
h.batches.Add(1)
}
return next(ctx, cmds)
}
}
type fuzzOp struct {
kind string
key string
val string
n int64
}
func genOps(rng *rand.Rand, n int) []fuzzOp {
kinds := []string{
"set", "get", "getdel", "incr", "incrby", "append", "strlen",
"exists", "del", "hset", "hget", "hgetall", "lpush", "lrange", "llen",
"expire", "ttl", "type", "setbig", "setbin",
}
keys := make([]string, 8)
for i := range keys {
keys[i] = fmt.Sprintf("fz:%d", i)
}
ops := make([]fuzzOp, n)
for i := range ops {
k := keys[rng.Intn(len(keys))]
ops[i] = fuzzOp{
kind: kinds[rng.Intn(len(kinds))], key: k,
val: fmt.Sprintf("v%d", rng.Intn(1000)), n: int64(rng.Intn(50)),
}
}
return ops
}
// runOps executes the op sequence via the Cmdable interface and returns a
// normalized "value|error" string per op for cross-run comparison.
func runOps(ctx context.Context, c redis.Cmdable, ops []fuzzOp) []string {
out := make([]string, len(ops))
res := func(v interface{}, err error) string {
if err != nil {
return "ERR:" + err.Error()
}
return fmt.Sprintf("%v", v)
}
for i, op := range ops {
switch op.kind {
case "set":
out[i] = res(c.Set(ctx, op.key, op.val, 0).Result())
case "get":
out[i] = res(c.Get(ctx, op.key).Result())
case "getdel":
out[i] = res(c.GetDel(ctx, op.key).Result())
case "incr":
out[i] = res(c.Incr(ctx, op.key).Result())
case "incrby":
out[i] = res(c.IncrBy(ctx, op.key, op.n).Result())
case "append":
out[i] = res(c.Append(ctx, op.key, op.val).Result())
case "strlen":
out[i] = res(c.StrLen(ctx, op.key).Result())
case "exists":
out[i] = res(c.Exists(ctx, op.key).Result())
case "del":
out[i] = res(c.Del(ctx, op.key).Result())
case "hset":
out[i] = res(c.HSet(ctx, op.key, op.val, op.n).Result())
case "hget":
out[i] = res(c.HGet(ctx, op.key, op.val).Result())
case "hgetall":
out[i] = res(c.HGetAll(ctx, op.key).Result())
case "lpush":
out[i] = res(c.LPush(ctx, op.key, op.val).Result())
case "lrange":
out[i] = res(c.LRange(ctx, op.key, 0, -1).Result())
case "llen":
out[i] = res(c.LLen(ctx, op.key).Result())
case "expire":
out[i] = res(c.Expire(ctx, op.key, 100*time.Second).Result())
case "ttl":
// TTL varies with wall clock; normalize to "set"/"unset".
d, err := c.TTL(ctx, op.key).Result()
if err != nil {
out[i] = "ERR:" + err.Error()
} else if d > 0 {
out[i] = "ttl>0"
} else {
out[i] = fmt.Sprintf("%v", d)
}
case "type":
out[i] = res(c.Type(ctx, op.key).Result())
case "setbig":
out[i] = res(c.Set(ctx, op.key, strings.Repeat("Z", 70000), 0).Result())
case "setbin":
out[i] = res(c.Set(ctx, op.key, "b\x00\r\n\x01\xffX", 0).Result())
}
}
return out
}
// runOpsWindowed issues the whole op sequence first and materializes results
// afterwards, in order — on the async face this coalesces the stream into
// real multi-command batches instead of lone-command flushes. Result strings
// are normalized identically to runOps.
func runOpsWindowed(ctx context.Context, c redis.Cmdable, ops []fuzzOp) []string {
out := make([]string, len(ops))
res := func(v interface{}, err error) string {
if err != nil {
return "ERR:" + err.Error()
}
return fmt.Sprintf("%v", v)
}
thunks := make([]func() string, len(ops))
for i, op := range ops {
switch op.kind {
case "set":
cmd := c.Set(ctx, op.key, op.val, 0)
thunks[i] = func() string { return res(cmd.Result()) }
case "get":
cmd := c.Get(ctx, op.key)
thunks[i] = func() string { return res(cmd.Result()) }
case "getdel":
cmd := c.GetDel(ctx, op.key)
thunks[i] = func() string { return res(cmd.Result()) }
case "incr":
cmd := c.Incr(ctx, op.key)
thunks[i] = func() string { return res(cmd.Result()) }
case "incrby":
cmd := c.IncrBy(ctx, op.key, op.n)
thunks[i] = func() string { return res(cmd.Result()) }
case "append":
cmd := c.Append(ctx, op.key, op.val)
thunks[i] = func() string { return res(cmd.Result()) }
case "strlen":
cmd := c.StrLen(ctx, op.key)
thunks[i] = func() string { return res(cmd.Result()) }
case "exists":
cmd := c.Exists(ctx, op.key)
thunks[i] = func() string { return res(cmd.Result()) }
case "del":
cmd := c.Del(ctx, op.key)
thunks[i] = func() string { return res(cmd.Result()) }
case "hset":
cmd := c.HSet(ctx, op.key, op.val, op.n)
thunks[i] = func() string { return res(cmd.Result()) }
case "hget":
cmd := c.HGet(ctx, op.key, op.val)
thunks[i] = func() string { return res(cmd.Result()) }
case "hgetall":
cmd := c.HGetAll(ctx, op.key)
thunks[i] = func() string { return res(cmd.Result()) }
case "lpush":
cmd := c.LPush(ctx, op.key, op.val)
thunks[i] = func() string { return res(cmd.Result()) }
case "lrange":
cmd := c.LRange(ctx, op.key, 0, -1)
thunks[i] = func() string { return res(cmd.Result()) }
case "llen":
cmd := c.LLen(ctx, op.key)
thunks[i] = func() string { return res(cmd.Result()) }
case "expire":
cmd := c.Expire(ctx, op.key, 100*time.Second)
thunks[i] = func() string { return res(cmd.Result()) }
case "ttl":
cmd := c.TTL(ctx, op.key)
thunks[i] = func() string {
d, err := cmd.Result()
if err != nil {
return "ERR:" + err.Error()
}
if d > 0 {
return "ttl>0"
}
return fmt.Sprintf("%v", d)
}
case "type":
cmd := c.Type(ctx, op.key)
thunks[i] = func() string { return res(cmd.Result()) }
case "setbig":
cmd := c.Set(ctx, op.key, strings.Repeat("Z", 70000), 0)
thunks[i] = func() string { return res(cmd.Result()) }
case "setbin":
cmd := c.Set(ctx, op.key, "b\x00\r\n\x01\xffX", 0)
thunks[i] = func() string { return res(cmd.Result()) }
}
}
for i, th := range thunks {
out[i] = th()
}
return out
}
// ===== from autopipeline_hook_deadlock_test.go =====
// resultPeekHook reads every command's result inside the hook chain — the
// redisotel span-status pattern (after next) and the stricter before-next
// variant from the #3867 deadlock report. Reading a deferred command's result
// on the dispatch goroutine used to block on the batch's done channel, which
// that same goroutine closes only after the hook chain returns: a
// deterministic deadlock on the async face.
type resultPeekHook struct{ before bool }
func (h resultPeekHook) DialHook(next redis.DialHook) redis.DialHook { return next }
func (h resultPeekHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
return func(ctx context.Context, cmd redis.Cmder) error {
if h.before {
_ = cmd.Err()
_ = cmd.String()
}
err := next(ctx, cmd)
if !h.before {
_ = cmd.Err()
_ = cmd.String()
}
return err
}
}
func (h resultPeekHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return func(ctx context.Context, cmds []redis.Cmder) error {
if h.before {
for _, c := range cmds {
_ = c.Err()
_ = c.String()
}
}
err := next(ctx, cmds)
if !h.before {
for _, c := range cmds {
_ = c.Err()
_ = c.String()
}
}
return err
}
}
// runWithWatchdog fails the test instead of hanging the suite if fn deadlocks.
func runWithWatchdog(t *testing.T, timeout time.Duration, fn func()) {
t.Helper()
done := make(chan struct{})
go func() { defer close(done); fn() }()
select {
case <-done:
case <-time.After(timeout):
t.Fatal("deadlock: hook reading deferred results blocked the dispatch")
}
}
func testAsyncHookPeek(t *testing.T, before bool) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
c.AddHook(resultPeekHook{before: before})
c.FlushDB(ctx)
ap, err := c.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
runWithWatchdog(t, 30*time.Second, func() {
const N = 200
sets := make([]*redis.StatusCmd, N)
for i := 0; i < N; i++ {
sets[i] = ap.Set(ctx, fmt.Sprintf("hookpeek:%d", i), i, 0)
}
for i, s := range sets {
if s.Err() != nil {
t.Errorf("set %d: %v", i, s.Err())
}
}
for i := 0; i < N; i++ {
v, err := ap.Get(ctx, fmt.Sprintf("hookpeek:%d", i)).Result()
if err != nil || v != fmt.Sprint(i) {
t.Errorf("get %d: v=%q err=%v", i, v, err)
}
}
})
}
// TestAsyncAutoPipelineHookReadsAfterNext: redisotel-style hook (reads results
// after next()). Deadlocked before the innermost-close fix.
func TestAsyncAutoPipelineHookReadsAfterNext(t *testing.T) { testAsyncHookPeek(t, false) }
// TestAsyncAutoPipelineHookReadsBeforeNext: hook reads results before next().
// Deadlocked before the dispatcher-goroutine guard in await().
func TestAsyncAutoPipelineHookReadsBeforeNext(t *testing.T) { testAsyncHookPeek(t, true) }
// TestAsyncAutoPipelineHookSoloCommand drives the lone-command fast path
// (Process chain, not the pipeline chain) with both hook variants.
func TestAsyncAutoPipelineHookSoloCommand(t *testing.T) {
for _, before := range []bool{false, true} {
t.Run(fmt.Sprintf("before=%v", before), func(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
c.AddHook(resultPeekHook{before: before})
c.FlushDB(ctx)
ap, err := c.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
runWithWatchdog(t, 30*time.Second, func() {
// One command, then an immediate read: the lone-caller flush
// dispatches it through the solo Process path.
cmd := ap.Set(ctx, "hookpeek:solo", "v", 0)
if err := cmd.Err(); err != nil {
t.Errorf("solo set: %v", err)
}
if v, err := ap.Get(ctx, "hookpeek:solo").Result(); err != nil || v != "v" {
t.Errorf("solo get: v=%q err=%v", v, err)
}
})
})
}
}
// TestBlockingAutoPipelineHookReads: the blocking face was never affected;
// pin that with the same hooks.
func TestBlockingAutoPipelineHookReads(t *testing.T) {
for _, before := range []bool{false, true} {
t.Run(fmt.Sprintf("before=%v", before), func(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
c.AddHook(resultPeekHook{before: before})
c.FlushDB(ctx)
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
runWithWatchdog(t, 30*time.Second, func() {
for i := 0; i < 50; i++ {
if err := ap.Set(ctx, fmt.Sprintf("hookpeek:b:%d", i), i, 0).Err(); err != nil {
t.Errorf("set %d: %v", i, err)
}
}
})
})
}
}
// TestAsyncAutoPipelineDoHookReads drives Do's background goroutine (its own
// close path) under a result-reading ProcessHook.
func TestAsyncAutoPipelineDoHookReads(t *testing.T) {
for _, before := range []bool{false, true} {
t.Run(fmt.Sprintf("before=%v", before), func(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
c.AddHook(resultPeekHook{before: before})
ap, err := c.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
runWithWatchdog(t, 30*time.Second, func() {
cmd := ap.Do(ctx, "PING")
if v, err := cmd.Result(); err != nil || v != "PONG" {
t.Errorf("do ping: v=%v err=%v", v, err)
}
})
})
}
}
// ===== from autopipeline_leak_test.go =====
// TestAutoPipelineNoGoroutineLeak opens and closes many autopipelines (both
// faces, ordered + unordered) and asserts goroutines return to baseline — i.e.
// flushers AND dispatcher workers all exit on Close, nothing leaks.
func TestAutoPipelineNoGoroutineLeak(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.PoolSize = 50
client := redis.NewClient(opt)
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
cfgs := []*redis.AutoPipelineOptions{
{MaxBatchSize: 300}, // ordered, 1 permit, 1 stripe
{MaxBatchSize: 300, MaxConcurrentBatches: 8, Unordered: true}, // unordered, 8 permits, 8 stripes
{MaxBatchSize: 300, MaxConcurrentBatches: 50, Unordered: true}, // many permits -> many workers
}
cycle := func() {
for _, cfg := range cfgs {
for _, async := range []bool{false, true} {
var ap *redis.AutoPipeliner
var err error
if async {
ap, err = client.AsyncAutoPipelineWithOptions(cfg)
} else {
ap, err = client.AutoPipelineWithOptions(cfg)
}
if err != nil {
t.Fatal(err)
}
wg := make([]redis.AutoFuture, 0, 200)
for i := 0; i < 200; i++ {
if async {
wg = append(wg, ap.Submit(ctx, redis.NewCmd(ctx, "set", "leak:k", i)))
} else {
_ = ap.Set(ctx, "leak:k", i, 0).Err()
}
}
for i := range wg {
_ = wg[i].Wait()
}
if err := ap.Close(); err != nil {
t.Fatalf("close: %v", err)
}
}
}
}
cycle() // warmup so lazy client/pool goroutines exist before we snapshot
runtime.GC()
time.Sleep(150 * time.Millisecond)
base := runtime.NumGoroutine()
const rounds = 40
for r := 0; r < rounds; r++ {
cycle()
}
got := settleGoroutines(base + 8) // poll: exiting goroutines wind down at their own pace
// Allow small slack for pool/runtime background goroutines; a leak of
// flushers/workers would be rounds*(shards+workers) ~ hundreds.
if got > base+8 {
t.Fatalf("goroutine leak: baseline=%d after %d open/close rounds=%d (delta %d)",
base, rounds, got, got-base)
}
t.Logf("no leak: baseline=%d final=%d (delta %d) over %d rounds", base, got, got-base, rounds)
}
// ===== from autopipeline_retry_test.go =====
// TestAutoPipelineRetriesOnNetworkError verifies AutoPipeline inherits the
// pipeline retry/re-drive loop (it dispatches through the same execer as manual
// Pipeline). The pooled conn's next write is armed to fail at the wire (a
// seeded broken idle conn would be filtered by the pool health check and never
// carry an attempt); the batch's first attempt genuinely dies, the retry
// redials, and the futures must read correct results from the re-drive.
//
// MaxFlushDelay forces the two commands into one real (>=2) batch so they run
// through the pipeline execer; a lone command would take the single-command
// fast path (Process) instead.
func TestAutoPipelineRetriesOnNetworkError(t *testing.T) {
ctx := context.Background()
var failNextWrite atomic.Bool
var dials atomic.Int32
client := redis.NewClient(&redis.Options{
Addr: apTestAddr(), MaxRetries: 2, PoolSize: 1,
Dialer: func(ctx context.Context, network, addr string) (net.Conn, error) {
dials.Add(1)
cn, err := (&net.Dialer{}).DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
return &flakyWriteConn{Conn: cn, fail: &failNextWrite}, nil
},
})
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
if err := client.Set(ctx, "apr:n", 0, 0).Err(); err != nil {
t.Fatal(err)
}
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 300,
MaxFlushDelay: 50 * time.Millisecond,
})
if err != nil {
t.Fatal(err)
}
defer ap.Close()
// Warm the DEDICATED PIPELINE POOL connection first: creating the
// autopipeliner created the pipeline pool (lazily), and BATCHES run there,
// not on the main-pool conn the Ping dialed. Without this warm-up the
// batch's first dispatch dials a fresh pipeline conn and the dial count
// reads one high for a reason unrelated to the retry. The warm-up must be
// a real multi-command batch: a lone command takes the solo fast path
// (Process on the main pool) and would not touch the pipeline pool.
w1 := ap.Set(ctx, "apr:warm", 1, 0)
w2 := ap.Incr(ctx, "apr:warm2")
if err := w1.Err(); err != nil {
t.Fatal(err)
}
if err := w2.Err(); err != nil {
t.Fatal(err)
}
// Arm after the handshake: the pipeline-pool conn is healthy, so it passes
// the pool health check, and the batch's first write dies on the wire.
dialsBefore := dials.Load()
failNextWrite.Store(true)
s := ap.Set(ctx, "apr:k", "v", 0)
n := ap.Incr(ctx, "apr:n")
if err := s.Err(); err != nil {
t.Fatalf("set after retry: %v", err)
}
if err := n.Err(); err != nil {
t.Fatalf("incr after retry: %v", err)
}
if s.Val() != "OK" {
t.Fatalf("set = %q, want OK", s.Val())
}
if n.Val() != 1 {
t.Fatalf("incr = %d, want 1", n.Val())
}
if failNextWrite.Load() {
t.Fatal("armed write failure never consumed: the first attempt did not reach the wire")
}
if got := dials.Load(); got != dialsBefore+1 {
t.Fatalf("dials after failure = %d, want %d (exactly one redial by the retry)", got, dialsBefore+1)
}
}
// TestAutoPipelineRetryExhaustionSurfacesError checks that when the pipeline
// retry loop is EXHAUSTED (every attempt fails on a retryable error), the
// AutoPipeline commands surface the error to their futures.
//
// This matters more for AutoPipeline than for manual Pipeline: manual Pipeline
// returns the error from Exec, but AutoPipeline discards the pipeline error
// (dispatchCmds: `_ = ...processPipelineHook(...)`) and results are observable
// ONLY via each command's future. If generalProcessPipeline does not call
// setCmdsErr on exhaustion (it only does so on the early-exit branch), a caller
// sees a nil error and a zero value — silent loss.
func TestAutoPipelineRetryExhaustionSurfacesError(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(&redis.Options{
Addr: apTestAddr(),
MaxRetries: 2,
PoolSize: 1,
Dialer: func(ctx context.Context, network, addr string) (net.Conn, error) {
return &badConn{writeErr: io.EOF}, nil // every conn fails all I/O
},
})
defer client.Close()
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 300,
MaxFlushDelay: 50 * time.Millisecond,
})
if err != nil {
t.Fatal(err)
}
defer ap.Close()
s := ap.Set(ctx, "k", "v", 0)
n := ap.Incr(ctx, "n")
if s.Err() == nil || n.Err() == nil {
t.Fatalf("SILENT LOSS: AutoPipeline commands returned nil error on retry exhaustion (set err=%v, incr err=%v) — every attempt failed but the futures report success",
s.Err(), n.Err())
}
}
// ===== from autopipeline_sequential_test.go =====
func TestAutoPipelineSequential(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.AutoPipelineOptions = &redis.AutoPipelineOptions{
MaxBatchSize: 10,
MaxFlushDelay: 50 * time.Millisecond,
MaxConcurrentBatches: 5,
Unordered: true,
}
client := redis.NewClient(opt)
defer client.Close()
if err := client.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
ap, err := client.AsyncAutoPipeline() // deferred face: submit-then-read (Do bypasses the engine)
if err != nil {
t.Fatal(err)
}
defer ap.Close()
// Sequential usage - no goroutines needed: typed calls on the deferred
// face queue without blocking and are batched automatically
cmds := make([]redis.Cmder, 20)
for i := 0; i < 20; i++ {
key := fmt.Sprintf("key%d", i)
cmds[i] = ap.Set(ctx, key, i, 0)
}
// Now access results - this will block until commands execute
for i, cmd := range cmds {
if err := cmd.Err(); err != nil {
t.Fatalf("Command %d failed: %v", i, err)
}
}
// Verify all keys were set
for i := 0; i < 20; i++ {
key := fmt.Sprintf("key%d", i)
val, err := client.Get(ctx, key).Int()
if err != nil {
t.Fatalf("Failed to get %s: %v", key, err)
}
if val != i {
t.Fatalf("Expected %d, got %d", i, val)
}
}
}
func TestAutoPipelineSequentialSmallBatches(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.AutoPipelineOptions = &redis.AutoPipelineOptions{
MaxBatchSize: 1000, // Large batch size
MaxFlushDelay: 20 * time.Millisecond, // Rely on timer
MaxConcurrentBatches: 5,
Unordered: true,
}
client := redis.NewClient(opt)
defer client.Close()
if err := client.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
ap, err := client.AsyncAutoPipeline() // deferred face: submit-then-read (Do bypasses the engine)
if err != nil {
t.Fatal(err)
}
defer ap.Close()
// Queue commands sequentially with small delays
// They should be flushed by timer, not batch size
for i := 0; i < 5; i++ {
key := fmt.Sprintf("key%d", i)
cmd := ap.Set(ctx, key, i, 0)
// Access result immediately - should block until flush
if err := cmd.Err(); err != nil {
t.Fatalf("Command %d failed: %v", i, err)
}
time.Sleep(5 * time.Millisecond)
}
// Verify all keys were set
for i := 0; i < 5; i++ {
key := fmt.Sprintf("key%d", i)
val, err := client.Get(ctx, key).Int()
if err != nil {
t.Fatalf("Failed to get %s: %v", key, err)
}
if val != i {
t.Fatalf("Expected %d, got %d", i, val)
}
}
}
func TestAutoPipelineSequentialMixed(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.AutoPipelineOptions = &redis.AutoPipelineOptions{
MaxBatchSize: 5,
MaxFlushDelay: 50 * time.Millisecond,
MaxConcurrentBatches: 5,
Unordered: true,
}
client := redis.NewClient(opt)
defer client.Close()
if err := client.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
ap, err := client.AsyncAutoPipeline() // deferred face: submit-then-read (Do bypasses the engine)
if err != nil {
t.Fatal(err)
}
defer ap.Close()
// Queue some commands
cmd1 := ap.Set(ctx, "key1", "value1", 0)
cmd2 := ap.Set(ctx, "key2", "value2", 0)
cmd3 := ap.Set(ctx, "key3", "value3", 0)
// Access first result - should trigger batch flush
if err := cmd1.Err(); err != nil {
t.Fatalf("cmd1 failed: %v", err)
}
// Other commands in same batch should also be done
if err := cmd2.Err(); err != nil {
t.Fatalf("cmd2 failed: %v", err)
}
if err := cmd3.Err(); err != nil {
t.Fatalf("cmd3 failed: %v", err)
}
// Verify
val, err := client.Get(ctx, "key1").Result()
if err != nil || val != "value1" {
t.Fatalf("key1: expected value1, got %v, err %v", val, err)
}
}
// ===== from autopipeline_singleton_test.go =====
// TestAutoPipelineConcurrentFirstCall guards a data race where AutoPipeline()
// did its cache check-and-set on the client's autopipeliner field without a
// lock. Concurrent first calls could each build a separate AutoPipeliner —
// racing the field (caught by -race) and leaking the loser's flusher goroutines
// and pipeline pool. Every concurrent caller must observe the SAME instance.
func TestAutoPipelineConcurrentFirstCall(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
c.FlushDB(ctx)
const G = 64
results := make([]*redis.AutoPipeliner, G)
errs := make([]error, G)
var wg sync.WaitGroup
start := make(chan struct{})
wg.Add(G)
for g := 0; g < G; g++ {
go func(i int) {
defer wg.Done()
<-start // maximize the race on the first call
results[i], errs[i] = c.AutoPipeline()
}(g)
}
close(start)
wg.Wait()
for i, err := range errs {
if err != nil {
t.Fatalf("call %d: AutoPipeline() returned error: %v", i, err)
}
}
first := results[0]
if first == nil {
t.Fatal("AutoPipeline() returned nil")
}
for i, ap := range results {
if ap != first {
t.Fatalf("call %d returned a different AutoPipeliner instance; concurrent first calls must share one", i)
}
}
defer first.Close()
// The shared instance must still work.
if err := first.Set(ctx, "singleton", "1", 0).Err(); err != nil {
t.Fatalf("shared autopipeliner unusable: %v", err)
}
}
// ===== from autopipeline_test.go =====
var _ = Describe("AutoPipeline", func() {
ctx := context.Background()
var client *redis.Client
var ap *redis.AutoPipeliner
BeforeEach(func() {
opt := redisOptions()
opt.AutoPipelineOptions = &redis.AutoPipelineOptions{
MaxBatchSize: 10,
MaxFlushDelay: 50 * time.Millisecond,
MaxConcurrentBatches: 5,
Unordered: true,
}
client = redis.NewClient(opt)
Expect(client.FlushDB(ctx).Err()).NotTo(HaveOccurred())
var err error
ap, err = client.AutoPipeline()
Expect(err).NotTo(HaveOccurred())
})
AfterEach(func() {
if ap != nil {
Expect(ap.Close()).NotTo(HaveOccurred())
}
Expect(client.Close()).NotTo(HaveOccurred())
})
It("should batch commands automatically", func() {
// Issue multiple commands concurrently to enable batching. A typed call
// without blocking, so each goroutine must await its own command's result
// (cmd.Err) before reporting done — wg.Wait alone only waits for the
// queueing, not the batch flush+execute, which would race the reads below.
var wg sync.WaitGroup
for i := 0; i < 5; i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
key := fmt.Sprintf("key%d", idx)
Expect(ap.Set(ctx, key, idx, 0).Err()).NotTo(HaveOccurred())
}(i)
}
// Wait for all commands to execute (each goroutine awaited its result).
wg.Wait()
// Verify all commands were executed
for i := 0; i < 5; i++ {
key := fmt.Sprintf("key%d", i)
val, err := client.Get(ctx, key).Int()
Expect(err).NotTo(HaveOccurred())
Expect(val).To(Equal(i))
}
})
It("should flush when batch size is reached", func() {
// Queue exactly MaxBatchSize commands concurrently. Each goroutine awaits
// its command's result so wg.Wait reflects execution, not just queueing.
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
key := fmt.Sprintf("key%d", idx)
Expect(ap.Set(ctx, key, idx, 0).Err()).NotTo(HaveOccurred())
}(i)
}
// Wait for all commands to execute (each goroutine awaited its result).
wg.Wait()
// Verify all commands were executed
for i := 0; i < 10; i++ {
key := fmt.Sprintf("key%d", i)
val, err := client.Get(ctx, key).Int()
Expect(err).NotTo(HaveOccurred())
Expect(val).To(Equal(i))
}
})
It("should flush on timer expiry", func() {
// Queue a single command from another goroutine; the blocking call
// returns only after its batch flushed, so signal completion instead
// of sleeping a fixed window.
done := make(chan error, 1)
go func() {
done <- ap.Set(ctx, "key1", "value1", 0).Err()
}()
Eventually(done, "5s").Should(Receive(BeNil()))
val, err := client.Get(ctx, "key1").Result()
Expect(err).NotTo(HaveOccurred())
Expect(val).To(Equal("value1"))
})
It("should handle concurrent commands", func() {
const numGoroutines = 10
const commandsPerGoroutine = 100
var wg sync.WaitGroup
wg.Add(numGoroutines)
for g := 0; g < numGoroutines; g++ {
go func(goroutineID int) {
defer wg.Done()
for i := 0; i < commandsPerGoroutine; i++ {
key := fmt.Sprintf("g%d:key%d", goroutineID, i)
ap.Set(ctx, key, i, 0)
}
}(g)
}
wg.Wait()
// All commands are now complete (Do() blocks until executed)
// Verify all commands were executed
for g := 0; g < numGoroutines; g++ {
for i := 0; i < commandsPerGoroutine; i++ {
key := fmt.Sprintf("g%d:key%d", g, i)
val, err := client.Get(ctx, key).Int()
Expect(err).NotTo(HaveOccurred())
Expect(val).To(Equal(i))
}
}
})
It("should support manual flush", func() {
// Queue commands concurrently
var wg sync.WaitGroup
for i := 0; i < 5; i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
key := fmt.Sprintf("key%d", idx)
ap.Set(ctx, key, idx, 0)
}(i)
}
// Wait for all commands to complete
wg.Wait()
// Verify all commands were executed
for i := 0; i < 5; i++ {
key := fmt.Sprintf("key%d", i)
val, err := client.Get(ctx, key).Int()
Expect(err).NotTo(HaveOccurred())
Expect(val).To(Equal(i))
}
})
It("should handle errors in commands", func() {
// Queue a mix of valid and invalid commands concurrently
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
ap.Set(ctx, "key1", "value1", 0)
}()
wg.Add(1)
go func() {
defer wg.Done()
// A command failing with a redis error (WRONGTYPE) inside a batch
// must not affect its batch neighbours.
client.Set(ctx, "not-a-counter", "str", 0)
_ = ap.Incr(ctx, "not-a-counter").Err()
}()
wg.Add(1)
go func() {
defer wg.Done()
ap.Set(ctx, "key2", "value2", 0)
}()
// Wait for all commands to complete
wg.Wait()
// Valid commands should still execute
val, err := client.Get(ctx, "key1").Result()
Expect(err).NotTo(HaveOccurred())
Expect(val).To(Equal("value1"))
val, err = client.Get(ctx, "key2").Result()
Expect(err).NotTo(HaveOccurred())
Expect(val).To(Equal("value2"))
})
It("should close gracefully", func() {
// Queue commands concurrently
var wg sync.WaitGroup
for i := 0; i < 5; i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
key := fmt.Sprintf("key%d", idx)
ap.Set(ctx, key, idx, 0)
}(i)
}
// Wait for all commands to complete
wg.Wait()
// Close should flush pending commands
Expect(ap.Close()).NotTo(HaveOccurred())
ap = nil // Prevent double close in AfterEach
// Verify all commands were executed
for i := 0; i < 5; i++ {
key := fmt.Sprintf("key%d", i)
val, err := client.Get(ctx, key).Int()
Expect(err).NotTo(HaveOccurred())
Expect(val).To(Equal(i))
}
})
It("should return error after close", func() {
Expect(ap.Close()).NotTo(HaveOccurred())
// Commands after close should fail
cmd := ap.Set(ctx, "key", "value", 0)
Expect(cmd.Err()).To(Equal(redis.ErrClosed))
})
It("should respect MaxConcurrentBatches", func() {
// Create autopipeliner with low concurrency limit
opt := redisOptions()
opt.AutoPipelineOptions = &redis.AutoPipelineOptions{
MaxBatchSize: 5,
MaxFlushDelay: 10 * time.Millisecond,
MaxConcurrentBatches: 2,
Unordered: true,
}
client2 := redis.NewClient(opt)
defer client2.Close()
ap2, err := client2.AutoPipeline()
Expect(err).NotTo(HaveOccurred())
defer ap2.Close()
// Queue many commands to trigger multiple batches concurrently
var wg sync.WaitGroup
for i := 0; i < 50; i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
key := fmt.Sprintf("key%d", idx)
ap2.Set(ctx, key, idx, 0)
}(i)
}
// Wait for all commands to complete
wg.Wait()
// Verify all commands were executed
for i := 0; i < 50; i++ {
key := fmt.Sprintf("key%d", i)
val, err := client2.Get(ctx, key).Int()
Expect(err).NotTo(HaveOccurred())
Expect(val).To(Equal(i))
}
})
It("should report queue length", func() {
// Deferred face with a wide explicit window: queued commands stay
// queued until the window elapses, so Len is deterministic.
aap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 100,
MaxFlushDelay: time.Second,
})
Expect(err).NotTo(HaveOccurred())
defer aap.Close()
Expect(aap.Len()).To(Equal(0))
cmds := make([]*redis.StatusCmd, 0, 5)
for i := 0; i < 5; i++ {
cmds = append(cmds, aap.Set(ctx, fmt.Sprintf("key%d", i), i, 0)) // queued, non-blocking
}
Expect(aap.Len()).To(Equal(5))
for _, c := range cmds {
Expect(c.Err()).NotTo(HaveOccurred()) // blocks until the window flushes
}
Expect(aap.Len()).To(Equal(0))
})
})
func TestAutoPipelineBasic(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.AutoPipelineOptions = &redis.AutoPipelineOptions{
MaxBatchSize: 10,
MaxFlushDelay: 50 * time.Millisecond,
MaxConcurrentBatches: 5,
Unordered: true,
}
client := redis.NewClient(opt)
defer client.Close()
if err := client.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
ap, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
// Queue commands concurrently
var wg sync.WaitGroup
for i := 0; i < 100; i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
key := fmt.Sprintf("key%d", idx)
cmd := ap.Set(ctx, key, idx, 0)
// Wait for command to complete
_ = cmd.Err()
}(i)
}
// Wait for all commands to complete
wg.Wait()
// Verify
for i := 0; i < 100; i++ {
key := fmt.Sprintf("key%d", i)
val, err := client.Get(ctx, key).Int()
if err != nil {
t.Fatalf("Failed to get key%d: %v", i, err)
}
if val != i {
t.Fatalf("Expected %d, got %d", i, val)
}
}
}
func TestAutoPipelineMaxFlushDelay(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.AutoPipelineOptions = &redis.AutoPipelineOptions{
MaxBatchSize: 1000, // Large batch size so only timer triggers flush
MaxFlushDelay: 50 * time.Millisecond,
MaxConcurrentBatches: 5,
Unordered: true,
}
client := redis.NewClient(opt)
defer client.Close()
if err := client.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
ap, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
rtt := apBaselineRTT(ctx, client)
// Send commands slowly, one at a time
// They should be flushed by the timer, not by batch size
start := time.Now()
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
key := fmt.Sprintf("key%d", idx)
cmd := ap.Set(ctx, key, idx, 0)
// Wait for command to complete
_ = cmd.Err()
}(i)
// Wait a bit between commands to ensure they don't batch by size
time.Sleep(10 * time.Millisecond)
}
wg.Wait()
elapsed := time.Since(start)
// Verify all keys were set
for i := 0; i < 10; i++ {
key := fmt.Sprintf("key%d", i)
val, err := client.Get(ctx, key).Int()
if err != nil {
t.Fatalf("Failed to get %s: %v", key, err)
}
if val != i {
t.Fatalf("Expected %d, got %d", i, val)
}
}
// Roughly 10*10ms sends + a few flush intervals; if the timer doesn't work,
// commands would wait until Close(). Bound scales to RTT (500ms floor).
bound := apLatencyBudget(rtt, 10, 500*time.Millisecond)
if elapsed > bound {
t.Fatalf("Commands took too long (%v, bound %v, rtt %v), flush interval may not be working", elapsed, bound, rtt)
}
t.Logf("Completed in %v (flush interval working correctly)", elapsed)
}
func TestAutoPipelineConcurrency(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.AutoPipelineOptions = redis.DefaultAutoPipelineOptions()
client := redis.NewClient(opt)
defer client.Close()
if err := client.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
ap, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const numGoroutines = 100
const commandsPerGoroutine = 100
var wg sync.WaitGroup
var failures atomic.Int64
wg.Add(numGoroutines)
for g := 0; g < numGoroutines; g++ {
go func(goroutineID int) {
defer wg.Done()
for i := 0; i < commandsPerGoroutine; i++ {
key := fmt.Sprintf("g%d:key%d", goroutineID, i)
if err := ap.Set(ctx, key, i, 0).Err(); err != nil {
failures.Add(1)
}
}
}(g)
}
wg.Wait()
if n := failures.Load(); n != 0 {
t.Fatalf("%d of %d Sets failed", n, numGoroutines*commandsPerGoroutine)
}
// Blocking-face Sets have executed by the time they return — verify the
// server actually holds every key.
size, err := client.DBSize(ctx).Result()
if err != nil {
t.Fatal(err)
}
if expected := int64(numGoroutines * commandsPerGoroutine); size != expected {
t.Fatalf("DBSize = %d, want %d", size, expected)
}
}
// TestAutoPipelineSingleCommandNoBlock verifies that single commands don't block
func TestAutoPipelineSingleCommandNoBlock(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.AutoPipelineOptions = redis.DefaultAutoPipelineOptions()
client := redis.NewClient(opt)
defer client.Close()
ap, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
// Warm up: the first-ever command pays dial + protocol handshake, which
// is not what this test bounds.
if err := ap.Ping(ctx).Err(); err != nil {
t.Fatal(err)
}
rtt := apBaselineRTT(ctx, client)
start := time.Now()
cmd := ap.Ping(ctx)
err = cmd.Err()
elapsed := time.Since(start)
if err != nil {
t.Fatalf("Command failed: %v", err)
}
if cmd.Val() != "PONG" {
t.Fatalf("Ping = %q, want PONG", cmd.Val())
}
// A warmed lone caller flushes immediately (no coalescing wait): one round
// trip. The bound scales to the measured RTT (50ms floor for localhost).
bound := apLatencyBudget(rtt, 1, 50*time.Millisecond)
if elapsed > bound {
t.Errorf("Single command took too long: %v (bound %v, rtt %v)", elapsed, bound, rtt)
}
t.Logf("Single command completed in %v", elapsed)
}
// TestAutoPipelineSequentialSingleThread verifies sequential single-threaded execution
func TestAutoPipelineSequentialSingleThread(t *testing.T) {
ctx := context.Background()
opt := apTestOptions()
opt.AutoPipelineOptions = redis.DefaultAutoPipelineOptions()
client := redis.NewClient(opt)
defer client.Close()
ap, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
// Warm up dial + handshake so the loop times steady-state round trips.
if err := ap.Ping(ctx).Err(); err != nil {
t.Fatal(err)
}
rtt := apBaselineRTT(ctx, client)
// Execute 10 commands sequentially in a single thread
start := time.Now()
for i := 0; i < 10; i++ {
key := fmt.Sprintf("test:key:%d", i)
t.Logf("Sending command %d", i)
cmd := ap.Set(ctx, key, i, 0)
t.Logf("Waiting for command %d to complete", i)
err := cmd.Err()
if err != nil {
t.Fatalf("Command %d failed: %v", i, err)
}
t.Logf("Command %d completed", i)
}
elapsed := time.Since(start)
// Ten sequential round trips: the bound scales to the measured RTT so this
// holds against a remote endpoint, with a 100ms floor to keep localhost tight.
bound := apLatencyBudget(rtt, 10, 100*time.Millisecond)
if elapsed > bound {
t.Errorf("10 sequential commands took too long: %v (bound %v, rtt %v)", elapsed, bound, rtt)
}
t.Logf("10 sequential commands completed in %v (avg: %v per command)", elapsed, elapsed/10)
}
// ===== from autopipeline_typed_test.go =====
// TestAutoPipelineTypedCommands tests that typed commands like Get() and Set()
// work correctly with autopipelining and block until execution.
func TestAutoPipelineTypedCommands(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
// Skip if Redis is not available
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("Redis not available: %v", err)
}
ap, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
// Test Set and Get
setResult, setErr := ap.Set(ctx, "test_key", "test_value", 0).Result()
if setErr != nil {
t.Fatalf("Set failed: %v", setErr)
}
if setResult != "OK" {
t.Errorf("Expected 'OK', got '%s'", setResult)
}
getResult, getErr := ap.Get(ctx, "test_key").Result()
if getErr != nil {
t.Fatalf("Get failed: %v", getErr)
}
if getResult != "test_value" {
t.Errorf("Expected 'test_value', got '%s'", getResult)
}
// Clean up
client.Del(ctx, "test_key")
}
// TestAutoPipelineTypedCommandsMultiple tests multiple typed commands
func TestAutoPipelineTypedCommandsMultiple(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
// Skip if Redis is not available
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("Redis not available: %v", err)
}
ap, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
// Queue multiple Set commands
ap.Set(ctx, "key1", "value1", 0)
ap.Set(ctx, "key2", "value2", 0)
ap.Set(ctx, "key3", "value3", 0)
// Get the values - should block until execution
val1, err1 := ap.Get(ctx, "key1").Result()
if err1 != nil {
t.Fatalf("Get key1 failed: %v", err1)
}
if val1 != "value1" {
t.Errorf("Expected 'value1', got '%s'", val1)
}
val2, err2 := ap.Get(ctx, "key2").Result()
if err2 != nil {
t.Fatalf("Get key2 failed: %v", err2)
}
if val2 != "value2" {
t.Errorf("Expected 'value2', got '%s'", val2)
}
val3, err3 := ap.Get(ctx, "key3").Result()
if err3 != nil {
t.Fatalf("Get key3 failed: %v", err3)
}
if val3 != "value3" {
t.Errorf("Expected 'value3', got '%s'", val3)
}
// Clean up
client.Del(ctx, "key1", "key2", "key3")
}
// TestAutoPipelineTypedCommandsVal tests the Val() method
func TestAutoPipelineTypedCommandsVal(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
// Skip if Redis is not available
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("Redis not available: %v", err)
}
ap, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
// Set a value
ap.Set(ctx, "test_val_key", "test_val_value", 0)
// Get using Val() - should block until execution
val := ap.Get(ctx, "test_val_key").Val()
if val != "test_val_value" {
t.Errorf("Expected 'test_val_value', got '%s'", val)
}
// Clean up
client.Del(ctx, "test_val_key")
}
// ===== from autopipeline_universal_test.go =====
// TestUniversalClientAutoPipeline verifies AutoPipeline is reachable through the
// UniversalClient interface: a standalone/cluster-backed UniversalClient returns
// a working AutoPipeliner, while Ring (also a UniversalClient) returns an error
// rather than being silently absent.
func TestUniversalClientAutoPipeline(t *testing.T) {
ctx := context.Background()
var uc redis.UniversalClient = redis.NewUniversalClient(&redis.UniversalOptions{
Addrs: []string{apTestAddr()},
})
defer uc.Close()
if err := uc.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
ap, err := uc.AutoPipeline()
if err != nil {
t.Fatalf("UniversalClient.AutoPipeline: %v", err)
}
defer ap.Close()
if v := ap.Set(ctx, "uc:k", "v", 0).Val(); v != "OK" {
t.Fatalf("set via UniversalClient autopipeline = %q, want OK", v)
}
if v := ap.Get(ctx, "uc:k").Val(); v != "v" {
t.Fatalf("get via UniversalClient autopipeline = %q, want v", v)
}
uc.Del(ctx, "uc:k")
// Ring: AutoPipeline is unsupported and must return an error (directly and
// through the UniversalClient interface).
ring := redis.NewRing(&redis.RingOptions{Addrs: map[string]string{"s1": apTestAddr()}})
defer ring.Close()
if _, err := ring.AutoPipeline(); err == nil {
t.Fatal("Ring.AutoPipeline should return an error (unsupported)")
}
if _, err := ring.AsyncAutoPipeline(); err == nil {
t.Fatal("Ring.AsyncAutoPipeline should return an error (unsupported)")
}
var ru redis.UniversalClient = ring
if _, err := ru.AutoPipeline(); err == nil {
t.Fatal("Ring via UniversalClient.AutoPipeline should return an error")
}
}
// TestAutoPipelinerAsUniversalClient verifies an *AutoPipeliner is a drop-in
// UniversalClient: data commands are batched through it, while the non-batched
// surface (PoolStats, Do, ...) delegates to the underlying client.
func TestAutoPipelinerAsUniversalClient(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
apc, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer apc.Close()
// Use the autopipeliner strictly through the UniversalClient interface.
var uc redis.UniversalClient = apc
// Data commands: batched by the autopipeliner.
if v := uc.Set(ctx, "ucap:k", "v", 0).Val(); v != "OK" {
t.Fatalf("Set = %q, want OK", v)
}
if v := uc.Get(ctx, "ucap:k").Val(); v != "v" {
t.Fatalf("Get = %q, want v", v)
}
// Non-batched surface delegates to the underlying client.
if uc.PoolStats() == nil {
t.Fatal("PoolStats() returned nil via UniversalClient")
}
if err := uc.Do(ctx, "PING").Err(); err != nil {
t.Fatalf("Do(PING): %v", err)
}
uc.Del(ctx, "ucap:k")
}
// ===== from osscluster_autopipeline_buffer_test.go =====
// Autopipeline + zero-copy buffer on CLUSTER (cross-slot, concurrent).
func TestAPZeroCopyBufferCluster(t *testing.T) {
ctx := context.Background()
c := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: []string{":16600", ":16601", ":16602", ":16603", ":16604", ":16605"},
AutoPipelineOptions: &redis.AutoPipelineOptions{MaxBatchSize: 100, MaxConcurrentBatches: 30, Unordered: true},
})
defer c.Close()
skipIfClusterUnhealthy(t, c)
_ = c.ForEachMaster(ctx, func(ctx context.Context, m *redis.Client) error { return m.FlushAll(ctx).Err() })
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const g = 80
const perG = 30
var wg sync.WaitGroup
var bad int64
var mu sync.Mutex
var errs []string
wg.Add(g)
for x := 0; x < g; x++ {
go func(id int) {
defer wg.Done()
for i := 0; i < perG; i++ {
key := fmt.Sprintf("clbuf:%d:%d:%d", id, i, id*7919+i) // spread slots
val := fmt.Sprintf("p-%d-%d", id, i)
if err := ap.SetFromBuffer(ctx, key, []byte(val)).Err(); err != nil {
atomic.AddInt64(&bad, 1)
mu.Lock()
if len(errs) < 10 {
errs = append(errs, "set:"+err.Error())
}
mu.Unlock()
continue
}
rbuf := make([]byte, len(val))
cmd := ap.GetToBuffer(ctx, key, rbuf)
n, err := cmd.Result()
if err != nil || n != len(val) || string(cmd.Bytes()) != val {
atomic.AddInt64(&bad, 1)
mu.Lock()
if len(errs) < 10 {
errs = append(errs, fmt.Sprintf("key %s n=%d got=%q want=%q err=%v", key, n, string(cmd.Bytes()), val, err))
}
mu.Unlock()
}
}
}(x)
}
wg.Wait()
if bad > 0 {
t.Fatalf("cluster zero-copy buffer via autopipeline failed: %d; sample=%v", bad, errs)
}
}
// ===== from osscluster_autopipeline_correctness_test.go =====
func newTestCluster() *redis.ClusterClient {
return redis.NewClusterClient(&redis.ClusterOptions{
Addrs: []string{":16600", ":16601", ":16602", ":16603", ":16604", ":16605"},
AutoPipelineOptions: &redis.AutoPipelineOptions{
MaxBatchSize: 200, MaxConcurrentBatches: 50, Unordered: true,
},
})
}
// skipIfClusterUnhealthy skips the test unless a real, formed cluster answers.
// Ping alone is insufficient: in CI's standalone test job a Redis may answer on
// the cluster ports while the cluster is not formed (CLUSTER INFO reports
// cluster_state:fail / CLUSTERDOWN, and commands return MOVED), which would turn
// "no cluster available" into spurious failures. Gate on cluster_state:ok.
func skipIfClusterUnhealthy(t *testing.T, c *redis.ClusterClient) {
t.Helper()
ctx := context.Background()
if err := c.Ping(ctx).Err(); err != nil {
t.Skipf("cluster not reachable: %v", err)
}
info, err := c.ClusterInfo(ctx).Result()
if err != nil {
t.Skipf("cluster not reachable (CLUSTER INFO): %v", err)
}
if !strings.Contains(info, "cluster_state:ok") {
t.Skipf("cluster not healthy (no cluster_state:ok)")
}
}
// Validates cluster routing: a single autopipeline batch contains keys that
// hash to MANY different slots/shards. Each command must route to the correct
// node and return its own correct value. If routing were wrong, GETs would
// miss or return another key's value.
func TestAPClusterCrossSlotRouting(t *testing.T) {
ctx := context.Background()
c := newTestCluster()
defer c.Close()
skipIfClusterUnhealthy(t, c)
_ = c.ForEachMaster(ctx, func(ctx context.Context, m *redis.Client) error {
return m.FlushAll(ctx).Err()
})
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const goroutines = 100
const perG = 50
var wg sync.WaitGroup
var mismatch, slotsSeen sync.Map
var bad int64
var mu sync.Mutex
wg.Add(goroutines)
for g := 0; g < goroutines; g++ {
go func(id int) {
defer wg.Done()
for i := 0; i < perG; i++ {
// Keys with no hashtag => spread across all 16384 slots.
key := fmt.Sprintf("cl:%d:%d:%d", id, i, id*7919+i)
want := fmt.Sprintf("V-%d-%d", id, i)
slotsSeen.Store(hashtag.Slot(key), true)
if err := ap.Set(ctx, key, want, 0).Err(); err != nil {
mu.Lock()
bad++
mismatch.Store(key, "set:"+err.Error())
mu.Unlock()
continue
}
got, err := ap.Get(ctx, key).Result()
if err != nil || got != want {
mu.Lock()
bad++
mismatch.Store(key, fmt.Sprintf("got=%q want=%q err=%v", got, want, err))
mu.Unlock()
}
}
}(g)
}
wg.Wait()
if bad > 0 {
shown := 0
mismatch.Range(func(k, v any) bool {
t.Logf("MISMATCH %v: %v", k, v)
shown++
return shown < 15
})
t.Fatalf("cluster cross-slot routing failures: %d", bad)
}
// Sanity: we actually exercised many distinct slots (not all one shard).
n := 0
slotsSeen.Range(func(_, _ any) bool { n++; return true })
if n < 100 {
t.Fatalf("test too weak: only %d distinct slots exercised", n)
}
t.Logf("validated cross-slot routing across %d distinct slots", n)
}
// Validates per-key correctness on cluster with INCR ordering per goroutine,
// each goroutine on its own (randomly-slotted) key.
func TestAPClusterPerGoroutineOrder(t *testing.T) {
ctx := context.Background()
c := newTestCluster()
defer c.Close()
skipIfClusterUnhealthy(t, c)
// Start from a clean slate so INCR counts are deterministic across runs.
_ = c.ForEachMaster(ctx, func(ctx context.Context, m *redis.Client) error {
return m.FlushAll(ctx).Err()
})
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const goroutines = 80
const incrs = 80
var wg sync.WaitGroup
var bad int64
var mu sync.Mutex
wg.Add(goroutines)
for g := 0; g < goroutines; g++ {
go func(id int) {
defer wg.Done()
key := fmt.Sprintf("clord:%d:%d", id, id*104729)
for i := 1; i <= incrs; i++ {
v, err := ap.Incr(ctx, key).Result()
if err != nil || v != int64(i) {
mu.Lock()
bad++
mu.Unlock()
}
}
}(g)
}
wg.Wait()
if bad > 0 {
t.Fatalf("cluster ordering/correctness failures: %d", bad)
}
}
// ===== from osscluster_autopipeline_moved_test.go =====
// TestAPClusterMovedRedirectFollowed verifies AutoPipeline inherits the cluster
// MOVED-redirect handling (it dispatches through ClusterClient.processPipeline,
// which runs the MaxRedirects loop + checkMovedErr). SwapNodes forces the client
// to believe a replica is the master for a slot; a write there is answered with
// MOVED and the client must follow it, so the value is still read back correctly.
func TestAPClusterMovedRedirectFollowed(t *testing.T) {
ctx := context.Background()
c := newTestCluster()
defer c.Close()
skipIfClusterUnhealthy(t, c)
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const key, val = "moved:key", "moved:val"
if err := ap.Set(ctx, key, val, 0).Err(); err != nil {
t.Fatalf("initial set: %v", err)
}
// Force the client's in-memory topology to point the slot at the wrong node,
// so the next access is answered with MOVED and must be followed.
if err := c.SwapNodes(ctx, key); err != nil {
t.Fatalf("SwapNodes: %v", err)
}
got, err := ap.Get(ctx, key).Result()
if err != nil {
t.Fatalf("get after MOVED: %v", err)
}
if got != val {
t.Fatalf("get after MOVED = %q, want %q (redirect not followed correctly)", got, val)
}
}
// TestAPClusterMovedSurfacedWithoutRedirects anchors the positive test: with
// MaxRedirects disabled (-1 -> 0, one attempt), the forced MOVED is NOT followed
// and must surface as an error on the command's future — proving SwapNodes really
// induced a redirect (so TestAPClusterMovedRedirectFollowed exercised the
// redirect path rather than passing trivially).
func TestAPClusterMovedSurfacedWithoutRedirects(t *testing.T) {
ctx := context.Background()
c := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: []string{":16600", ":16601", ":16602", ":16603", ":16604", ":16605"},
MaxRedirects: -1, // normalized to 0: a single attempt, no follow-through
AutoPipelineOptions: &redis.AutoPipelineOptions{
MaxBatchSize: 200, MaxConcurrentBatches: 50, Unordered: true,
},
})
defer c.Close()
skipIfClusterUnhealthy(t, c)
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const key = "moved:noredir:key"
if err := ap.Set(ctx, key, "v", 0).Err(); err != nil {
t.Fatalf("initial set: %v", err)
}
// SwapNodes is racy against the periodic LazyReload, so re-swap and retry
// until we observe the MOVED (or give up and skip — the swap never stuck).
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
if err := c.SwapNodes(ctx, key); err != nil {
t.Fatalf("SwapNodes: %v", err)
}
err := ap.Get(ctx, key).Err()
if err != nil && strings.Contains(err.Error(), "MOVED") {
return // proven: the redirect was needed and, with no redirects, surfaced
}
time.Sleep(100 * time.Millisecond)
}
t.Skip("could not observe a MOVED (swap kept being reverted by topology reload)")
}
// ===== from osscluster_autopipeline_test.go =====
func TestClusterAutoPipelineBasic(t *testing.T) {
ctx := context.Background()
client := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: []string{":16600", ":16601", ":16602"},
AutoPipelineOptions: &redis.AutoPipelineOptions{
MaxBatchSize: 10,
MaxFlushDelay: 50 * time.Millisecond,
MaxConcurrentBatches: 5,
Unordered: true,
},
})
defer client.Close()
skipIfClusterUnhealthy(t, client)
if err := client.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
ap, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
// Queue commands concurrently across different keys (different slots)
var wg sync.WaitGroup
for i := 0; i < 100; i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
key := fmt.Sprintf("key%d", idx)
cmd := ap.Set(ctx, key, idx, 0)
// Wait for command to complete
_ = cmd.Err()
}(i)
}
// Wait for all commands to complete
wg.Wait()
// Verify all keys were set correctly
for i := 0; i < 100; i++ {
key := fmt.Sprintf("key%d", i)
val, err := client.Get(ctx, key).Int()
if err != nil {
t.Fatalf("Failed to get %s: %v", key, err)
}
if val != i {
t.Fatalf("Expected %d, got %d", i, val)
}
}
}
func TestClusterAutoPipelineConcurrency(t *testing.T) {
ctx := context.Background()
client := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: []string{":16600", ":16601", ":16602"},
AutoPipelineOptions: &redis.AutoPipelineOptions{
MaxBatchSize: 50,
MaxFlushDelay: 10 * time.Millisecond,
MaxConcurrentBatches: 10,
Unordered: true,
},
})
defer client.Close()
skipIfClusterUnhealthy(t, client)
if err := client.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
ap, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const numGoroutines = 10
const commandsPerGoroutine = 100
var wg sync.WaitGroup
for g := 0; g < numGoroutines; g++ {
wg.Add(1)
go func(goroutineID int) {
defer wg.Done()
for i := 0; i < commandsPerGoroutine; i++ {
key := fmt.Sprintf("g%d:key%d", goroutineID, i)
cmd := ap.Set(ctx, key, i, 0)
// Wait for command to complete
_ = cmd.Err()
}
}(g)
}
// Wait for all commands to complete
wg.Wait()
// Verify all commands were executed
for g := 0; g < numGoroutines; g++ {
for i := 0; i < commandsPerGoroutine; i++ {
key := fmt.Sprintf("g%d:key%d", g, i)
val, err := client.Get(ctx, key).Int()
if err != nil {
t.Fatalf("Failed to get %s: %v", key, err)
}
if val != i {
t.Fatalf("Expected %d, got %d for key %s", i, val, key)
}
}
}
}
func TestClusterAutoPipelineCrossSlot(t *testing.T) {
ctx := context.Background()
client := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: []string{":16600", ":16601", ":16602"},
AutoPipelineOptions: &redis.AutoPipelineOptions{
MaxBatchSize: 20,
MaxFlushDelay: 10 * time.Millisecond,
MaxConcurrentBatches: 5,
Unordered: true,
},
})
defer client.Close()
skipIfClusterUnhealthy(t, client)
if err := client.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
ap, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
// Use keys that will hash to different slots
keys := []string{
"user:1000",
"user:2000",
"user:3000",
"session:abc",
"session:def",
"cache:foo",
"cache:bar",
}
var wg sync.WaitGroup
for i, key := range keys {
wg.Add(1)
go func(k string, val int) {
defer wg.Done()
cmd := ap.Set(ctx, k, val, 0)
// Wait for command to complete
_ = cmd.Err()
}(key, i)
}
wg.Wait()
// Verify all keys were set
for i, key := range keys {
val, err := client.Get(ctx, key).Int()
if err != nil {
t.Fatalf("Failed to get %s: %v", key, err)
}
if val != i {
t.Fatalf("Expected %d, got %d for key %s", i, val, key)
}
}
}
// ===== review-fix coverage additions (deep review 2026-07-13) =====
// shortCircuitHook returns from the chain WITHOUT calling next — the
// circuit-breaker hook shape the Hook contract explicitly allows. The engine
// must surface its error on every command instead of waking waiters with
// silent zero values.
type shortCircuitHook struct{ err error }
func (h shortCircuitHook) DialHook(next redis.DialHook) redis.DialHook { return next }
func (h shortCircuitHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
return func(ctx context.Context, cmd redis.Cmder) error { return h.err }
}
func (h shortCircuitHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return func(ctx context.Context, cmds []redis.Cmder) error { return h.err }
}
// pipelineBreakerHook short-circuits the pipeline chain once armed. It stays
// open until then because connection init itself runs a handshake pipeline
// through the hook chain (initConn -> Pipeline.Exec), and plain commands
// (Ping skip-gates, topology discovery) always pass through.
type pipelineBreakerHook struct {
err error
armed *atomic.Bool
}
func (h pipelineBreakerHook) DialHook(next redis.DialHook) redis.DialHook { return next }
func (h pipelineBreakerHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
return func(ctx context.Context, cmd redis.Cmder) error { return next(ctx, cmd) }
}
func (h pipelineBreakerHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return func(ctx context.Context, cmds []redis.Cmder) error {
if !h.armed.Load() {
return next(ctx, cmds)
}
return h.err
}
}
func TestAutoPipelineHookShortCircuit(t *testing.T) {
errBreaker := errors.New("breaker open")
for _, tc := range []struct {
name string
async bool
}{{"blocking", false}, {"async", true}} {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
if err := c.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
c.AddHook(shortCircuitHook{err: errBreaker})
var ap *redis.AutoPipeliner
var err error
if tc.async {
ap, err = c.AsyncAutoPipeline()
} else {
ap, err = c.AutoPipeline()
}
if err != nil {
t.Fatal(err)
}
defer ap.Close()
runWithWatchdog(t, 30*time.Second, func() {
// Batch path: several concurrent commands so they flush as a
// pipeline batch through the short-circuiting chain.
const N = 20
var wg sync.WaitGroup
errs := make([]error, N)
wg.Add(N)
for i := 0; i < N; i++ {
go func(i int) {
defer wg.Done()
errs[i] = ap.Set(ctx, fmt.Sprintf("sc:%d", i), i, 0).Err()
}(i)
}
wg.Wait()
for i, err := range errs {
if !errors.Is(err, errBreaker) {
t.Errorf("cmd %d: err = %v, want the hook's error", i, err)
}
}
// Solo path: a lone command takes the Process fast path.
if err := ap.Set(ctx, "sc:solo", 1, 0).Err(); !errors.Is(err, errBreaker) {
t.Errorf("solo: err = %v, want the hook's error", err)
}
})
})
}
}
// TestAutoPipelineHookAddedAfterCreate pins that hooks installed AFTER the
// autopipeliner exists (directly on the client and via the AutoPipeliner's
// delegating AddHook) take effect and may read results without deadlocking.
func TestAutoPipelineHookAddedAfterCreate(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
if err := c.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
ap, err := c.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
runWithWatchdog(t, 30*time.Second, func() {
// Round 1: no hooks yet.
if err := ap.Set(ctx, "late:0", "v0", 0).Err(); err != nil {
t.Fatal(err)
}
// Round 2: result-reading hook added on the client after creation.
c.AddHook(resultPeekHook{before: false})
if v, err := ap.Get(ctx, "late:0").Result(); err != nil || v != "v0" {
t.Fatalf("after client AddHook: v=%q err=%v", v, err)
}
// Round 3: another added through the AutoPipeliner's delegate.
ap.AddHook(resultPeekHook{before: true})
cmds := make([]*redis.StatusCmd, 0, 30)
for i := 0; i < 30; i++ {
cmds = append(cmds, ap.Set(ctx, fmt.Sprintf("late:%d", i), i, 0))
}
for i, cmd := range cmds {
if err := cmd.Err(); err != nil {
t.Fatalf("cmd %d after ap.AddHook: %v", i, err)
}
}
})
}
// TestAutoPipelineCloseFlushesPendingFutures pins Close's contract: deferred
// commands submitted but not yet flushed are executed (not dropped, never
// hung) by the shutdown drain.
func TestAutoPipelineCloseFlushesPendingFutures(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
if err := c.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
if err := c.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
// Wide window + deep batch: nothing flushes on its own before Close.
ap, err := c.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 1000,
MaxFlushDelay: 500 * time.Millisecond,
})
if err != nil {
t.Fatal(err)
}
runWithWatchdog(t, 30*time.Second, func() {
const N = 50
cmds := make([]*redis.StatusCmd, 0, N)
for i := 0; i < N; i++ {
cmds = append(cmds, ap.Set(ctx, fmt.Sprintf("pend:%d", i), i, 0))
}
if err := ap.Close(); err != nil {
t.Fatal(err)
}
for i, cmd := range cmds {
if err := cmd.Err(); err != nil {
t.Fatalf("pending cmd %d after Close: %v", i, err)
}
}
n, err := c.DBSize(ctx).Result()
if err != nil {
t.Fatal(err)
}
if n != N {
t.Fatalf("DBSize = %d, want %d (Close must flush pending commands)", n, N)
}
})
}
// TestAutoPipelineClientLevelOptionsApply pins that Options.AutoPipelineOptions
// actually configures both faces (nothing previously failed if it were ignored).
func TestAutoPipelineClientLevelOptionsApply(t *testing.T) {
c := redis.NewClient(&redis.Options{
Addr: apTestAddr(),
AutoPipelineOptions: &redis.AutoPipelineOptions{MaxBatchSize: 123},
})
defer c.Close()
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
if got := ap.Config().MaxBatchSize; got != 123 {
t.Fatalf("blocking face MaxBatchSize = %d, want 123 (client-level options ignored)", got)
}
aap, err := c.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
if got := aap.Config().MaxBatchSize; got != 123 {
t.Fatalf("async face MaxBatchSize = %d, want 123 (client-level options ignored)", got)
}
}
// TestAutoPipelineMaxBatchBytesDefaultsToGuardrail pins that an unset (or
// explicitly zero) MaxBatchBytes resolves to the 128 KiB full-duplex deadlock
// guardrail (see the field's GoDoc), not to "unbounded" — the previous
// behavior. Checked via both preset constructors and the actual resolved
// config on a constructed autopipeliner, since MaxBatchSize's own default is
// applied redundantly in three places and MaxBatchBytes now mirrors that.
func TestAutoPipelineMaxBatchBytesDefaultsToGuardrail(t *testing.T) {
const want = 128 * 1024
if got := redis.DefaultAutoPipelineOptions().MaxBatchBytes; got != want {
t.Errorf("DefaultAutoPipelineOptions().MaxBatchBytes = %d, want %d", got, want)
}
if got := redis.DefaultBlockingAutoPipelineOptions().MaxBatchBytes; got != want {
t.Errorf("DefaultBlockingAutoPipelineOptions().MaxBatchBytes = %d, want %d", got, want)
}
c := redis.NewClient(&redis.Options{
Addr: apTestAddr(),
AutoPipelineOptions: &redis.AutoPipelineOptions{
MaxBatchSize: 50, // leave MaxBatchBytes at its zero value
},
})
defer c.Close()
ap, err := c.AutoPipeline()
if err != nil {
t.Fatal(err)
}
if got := ap.Config().MaxBatchBytes; got != want {
t.Fatalf("resolved MaxBatchBytes = %d, want %d (zero must default to the guardrail, not stay unbounded)", got, want)
}
}
// TestAsyncAutoPipelineBlockingCommand covers the readTimeout diversion on the
// DEFERRED face: BLPOP must bypass the batch (it would stall a whole pipeline)
// and still deliver its result through the usual accessors.
func TestAsyncAutoPipelineBlockingCommand(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
if err := c.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
c.Del(ctx, "blk:list")
ap, err := c.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
runWithWatchdog(t, 30*time.Second, func() {
// Value present: BLPOP returns it.
if err := c.LPush(ctx, "blk:list", "v1").Err(); err != nil {
t.Fatal(err)
}
vals, err := ap.BLPop(ctx, time.Second, "blk:list").Result()
if err != nil || len(vals) != 2 || vals[1] != "v1" {
t.Fatalf("BLPop = %v, %v; want [blk:list v1]", vals, err)
}
// Empty list: times out with redis.Nil, does not wedge the engine.
if _, err := ap.BLPop(ctx, 50*time.Millisecond, "blk:list").Result(); err != redis.Nil {
t.Fatalf("BLPop empty = %v, want redis.Nil", err)
}
// The engine still works afterwards.
if err := ap.Set(ctx, "blk:after", "ok", 0).Err(); err != nil {
t.Fatal(err)
}
})
}
// TestAdaptiveDelayEndToEnd runs real traffic through an autopipeliner
// constructed with AdaptiveDelay (previously only the pure delay math was
// tested; no command had ever executed under this configuration).
func TestAdaptiveDelayEndToEnd(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
if err := c.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
if err := c.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
ap, err := c.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 50,
MaxFlushDelay: 5 * time.Millisecond,
AdaptiveDelay: true,
})
if err != nil {
t.Fatal(err)
}
defer ap.Close()
runWithWatchdog(t, 60*time.Second, func() {
// Windowed bursts exercise the fill-level-dependent delay branches.
const rounds, window = 5, 80
for r := 0; r < rounds; r++ {
cmds := make([]*redis.StatusCmd, 0, window)
for i := 0; i < window; i++ {
cmds = append(cmds, ap.Set(ctx, fmt.Sprintf("ad:%d:%d", r, i), i, 0))
}
for i, cmd := range cmds {
if err := cmd.Err(); err != nil {
t.Fatalf("round %d cmd %d: %v", r, i, err)
}
}
}
for r := 0; r < rounds; r++ {
for i := 0; i < window; i += 17 {
v, err := ap.Get(ctx, fmt.Sprintf("ad:%d:%d", r, i)).Result()
if err != nil || v != fmt.Sprint(i) {
t.Fatalf("round %d get %d: v=%q err=%v", r, i, v, err)
}
}
}
})
}
// TestClusterPipelineUsesDedicatedPool pins the fix for the review finding
// that cluster pipelines acquired connections via getConn (main pool),
// bypassing the dedicated pipeline pool entirely: node clients inherit the
// Pipeline* options, and processPipelineNode now routes through
// withPipelineConn. Covers the manual pipeline and the autopipeline batch
// path (both converge on processPipelineNode).
func TestClusterPipelineUsesDedicatedPool(t *testing.T) {
ctx := context.Background()
c := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: []string{":16600", ":16601", ":16602"},
PipelineReadBufferSize: 64 * 1024,
})
defer c.Close()
skipIfClusterUnhealthy(t, c)
runWithWatchdog(t, 60*time.Second, func() {
// Manual pipeline spanning many slots.
pipe := c.Pipeline()
for i := 0; i < 50; i++ {
pipe.Set(ctx, fmt.Sprintf("cpp:%d", i), i, 0)
}
if _, err := pipe.Exec(ctx); err != nil {
t.Fatalf("pipeline exec: %v", err)
}
// Autopipeline batch (same node dispatch seam underneath).
ap, err := c.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
cmds := make([]*redis.StringCmd, 0, 50)
for i := 0; i < 50; i++ {
cmds = append(cmds, ap.Get(ctx, fmt.Sprintf("cpp:%d", i)))
}
for i, cmd := range cmds {
if v, err := cmd.Result(); err != nil || v != fmt.Sprint(i) {
t.Fatalf("get %d: v=%q err=%v", i, v, err)
}
}
// The dedicated pipeline pool on the nodes must have carried traffic.
var pipelineConns atomic.Uint64 // ForEachMaster runs callbacks concurrently
err = c.ForEachMaster(ctx, func(ctx context.Context, cl *redis.Client) error {
if ps := cl.PoolStats().PipelineStats; ps != nil {
pipelineConns.Add(uint64(ps.TotalConns) + uint64(ps.Hits))
}
return nil
})
if err != nil {
t.Fatal(err)
}
if pipelineConns.Load() == 0 {
t.Fatal("dedicated pipeline pool unused: cluster pipelines still bypass it")
}
})
}
// TestAutoPipelineDoRawOutsidePipeline pins that DoRaw/DoRawWriteTo do NOT
// ride the batching engine (AutoPipeliner embeds cmdable, so without the
// overrides they would): they follow Do's escape-hatch shape on both faces,
// and hooks reading their results cannot deadlock.
func TestAutoPipelineDoRawOutsidePipeline(t *testing.T) {
for _, tc := range []struct {
name string
async bool
}{{"blocking", false}, {"async", true}} {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
if err := c.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
c.AddHook(resultPeekHook{before: false}) // reads results in hooks
var ap *redis.AutoPipeliner
var err error
if tc.async {
ap, err = c.AsyncAutoPipeline()
} else {
ap, err = c.AutoPipeline()
}
if err != nil {
t.Fatal(err)
}
defer ap.Close()
runWithWatchdog(t, 30*time.Second, func() {
raw := ap.DoRaw(ctx, "PING")
if err := raw.Err(); err != nil {
t.Fatalf("DoRaw: %v", err)
}
if v, _ := raw.Result(); !strings.Contains(string(v), "PONG") {
t.Fatalf("DoRaw PING raw = %q, want PONG in RESP", v)
}
var buf bytes.Buffer
wcmd := ap.DoRawWriteTo(ctx, &buf, "PING")
if err := wcmd.Err(); err != nil { // async face: blocks until executed
t.Fatalf("DoRawWriteTo: %v", err)
}
if !strings.Contains(buf.String(), "PONG") {
t.Fatalf("DoRawWriteTo buffer = %q, want PONG in RESP", buf.String())
}
// no-args guard mirrors Do
if err := ap.DoRaw(ctx).Err(); err == nil {
t.Fatal("DoRaw() with no args: want error")
}
})
})
}
}
// TestClusterNodeHookShortCircuit pins that a NODE-level hook returning
// without calling next (circuit-breaker shape, installed via OnNewNode) is
// surfaced on every command instead of the cluster pipeline reporting silent
// success for commands that were never sent.
func TestClusterNodeHookShortCircuit(t *testing.T) {
ctx := context.Background()
errBreaker := errors.New("node breaker open")
c := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: []string{":16600", ":16601", ":16602"},
})
defer c.Close()
// Install BEFORE any command: OnNewNode only fires for newly created node
// clients. The breaker is disarmed during warmup (conn-init handshakes
// are pipelines too) and armed just before the asserted round.
var armed atomic.Bool
c.OnNewNode(func(cl *redis.Client) { cl.AddHook(pipelineBreakerHook{err: errBreaker, armed: &armed}) })
skipIfClusterUnhealthy(t, c)
runWithWatchdog(t, 60*time.Second, func() {
// Warm pooled connections on every node so the asserted round doesn't
// dial fresh ones (whose handshake the armed breaker would kill first).
warm := c.Pipeline()
for i := 0; i < 12; i++ {
warm.Set(ctx, fmt.Sprintf("nsc:%d", i), i, 0)
}
if _, err := warm.Exec(ctx); err != nil {
t.Fatalf("warmup: %v", err)
}
armed.Store(true)
pipe := c.Pipeline()
cmds := make([]*redis.StatusCmd, 0, 12)
for i := 0; i < 12; i++ {
cmds = append(cmds, pipe.Set(ctx, fmt.Sprintf("nsc:%d", i), i, 0))
}
_, execErr := pipe.Exec(ctx)
if execErr == nil {
t.Fatal("cluster pipeline reported success though node hooks short-circuited")
}
for i, cmd := range cmds {
if cmd.Err() == nil {
t.Fatalf("cmd %d: err = nil, want the node hook's error surfaced", i)
}
}
})
}
// postNextErrorHook calls next and then returns its own error regardless —
// ofek's #3867 case: a hook's post-next verdict used to be discarded because
// the batch had already completed at the innermost seam.
type postNextErrorHook struct{ err error }
func (h postNextErrorHook) DialHook(next redis.DialHook) redis.DialHook { return next }
func (h postNextErrorHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
return func(ctx context.Context, cmd redis.Cmder) error {
_ = next(ctx, cmd)
return h.err
}
}
func (h postNextErrorHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return func(ctx context.Context, cmds []redis.Cmder) error {
_ = next(ctx, cmds)
return h.err
}
}
// TestAutoPipelineHookPostNextError pins that an error a hook returns AFTER
// calling next (successful exec underneath) reaches every command on both
// faces, across the batch, solo and Do dispatch paths.
func TestAutoPipelineHookPostNextError(t *testing.T) {
errInjected := errors.New("hook injected after next")
for _, tc := range []struct {
name string
async bool
}{{"blocking", false}, {"async", true}} {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
if err := c.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
c.AddHook(postNextErrorHook{err: errInjected})
var ap *redis.AutoPipeliner
var err error
if tc.async {
ap, err = c.AsyncAutoPipeline()
} else {
ap, err = c.AutoPipeline()
}
if err != nil {
t.Fatal(err)
}
defer ap.Close()
runWithWatchdog(t, 30*time.Second, func() {
// Batch path: concurrent commands flush as one pipeline.
const N = 16
var wg sync.WaitGroup
errs := make([]error, N)
wg.Add(N)
for i := 0; i < N; i++ {
go func(i int) {
defer wg.Done()
errs[i] = ap.Set(ctx, fmt.Sprintf("pn:%d", i), i, 0).Err()
}(i)
}
wg.Wait()
for i, err := range errs {
if !errors.Is(err, errInjected) {
t.Errorf("batch cmd %d: err = %v, want the hook's post-next error", i, err)
}
}
// Solo path (lone command -> Process chain).
if err := ap.Set(ctx, "pn:solo", 1, 0).Err(); !errors.Is(err, errInjected) {
t.Errorf("solo: err = %v, want the hook's post-next error", err)
}
// Do (escape hatch).
if err := ap.Do(ctx, "PING").Err(); !errors.Is(err, errInjected) {
t.Errorf("Do: err = %v, want the hook's post-next error", err)
}
})
})
}
}
// TestAutoPipelineHookPostNextErrorPartialBatch pins the boundary of the
// post-next rule: when the exec recorded ANY per-command outcome-error
// (here a redis.Nil), those results are the source of truth — a hook's
// post-next error must not rewrite them, exactly as a plain pipeline never
// rewrites per-command results (the hook's verdict goes to the Exec caller,
// which does not exist here).
func TestAutoPipelineHookPostNextErrorPartialBatch(t *testing.T) {
errInjected := errors.New("hook injected after next")
ctx := context.Background()
c := redis.NewClient(apTestOptions())
defer c.Close()
if err := c.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
if err := c.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
if err := c.Set(ctx, "pnp:present", "v", 0).Err(); err != nil {
t.Fatal(err)
}
// Wide flush window so all three commands deterministically land in ONE
// pipeline batch (the rule is per-batch: hooks fire per batch).
ap, err := c.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 100,
MaxFlushDelay: 50 * time.Millisecond,
})
if err != nil {
t.Fatal(err)
}
defer ap.Close()
// Initialize the dedicated pipeline-pool connection BEFORE installing the
// error-injecting hook: connection init runs its handshake pipeline through
// the client's hook chain (newConn shares hooksMixin), so a hook that
// unconditionally injects an error would fail the pipeline conn's init and
// this test would measure init poisoning instead of the post-next rule.
// Must be a real multi-command batch — a lone command takes the solo fast
// path (Process on the main pool) and would not init the pipeline conn.
w1 := ap.Set(ctx, "pnp:warm", 1, 0)
w2 := ap.Set(ctx, "pnp:warm2", 1, 0)
if err := w1.Err(); err != nil {
t.Fatal(err)
}
if err := w2.Err(); err != nil {
t.Fatal(err)
}
c.AddHook(postNextErrorHook{err: errInjected})
runWithWatchdog(t, 30*time.Second, func() {
// One batch: a hit, a miss (redis.Nil), and a write.
hit := ap.Get(ctx, "pnp:present")
miss := ap.Get(ctx, "pnp:absent")
set := ap.Set(ctx, "pnp:new", 1, 0)
if err := miss.Err(); err != redis.Nil {
t.Errorf("miss: err = %v, want redis.Nil kept (exec outcome wins)", err)
}
if v, err := hit.Result(); err != nil || v != "v" {
t.Errorf("hit: v=%q err=%v, want success kept", v, err)
}
if err := set.Err(); err != nil {
t.Errorf("set: err = %v, want success kept", err)
}
})
}
// Regression: the async face with NO hooks must survive a multi-command
// batch whose first reply is an error. pipelineProcessCmds re-reads the
// first command's error after the himport re-issue pass; doing that via
// Err() awaited the command's own still-open batch and self-deadlocked the
// dispatcher (the dispGid guard is unarmed without hooks). A plain redis.Nil
// on the first command was enough to trigger it. rawErr() is the fix.
func TestAsyncAutoPipelineErrorFirstReplyNoHooks(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
// Forced single batch: everything lands in one flush, first reply is a
// redis.Nil (missing key). No hooks anywhere — the default configuration.
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 100,
MaxFlushDelay: 50 * time.Millisecond,
})
if err != nil {
t.Fatal(err)
}
runWithWatchdog(t, 30*time.Second, func() {
cmds := make([]*redis.StringCmd, 10)
for i := range cmds {
cmds[i] = ap.Get(ctx, fmt.Sprintf("ap:errfirst:missing:%d", i))
}
for i, cmd := range cmds {
if err := cmd.Err(); err != redis.Nil {
t.Errorf("cmd %d: got %v, want redis.Nil", i, err)
}
}
})
}
// Regression: HIMPORT commands inside an async batch. himportAfterBatch and
// himportRetryFailedSets run on the execution path (standalone and per-node
// cluster goroutines) and read the batch's commands; doing so via Err()
// awaited the still-open batch and self-deadlocked exactly like the case
// above — even when every command succeeded. Works against servers without
// HIMPORT support too: the reads happen regardless of the reply.
func TestAsyncAutoPipelineHImportBatchNoHooks(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 100,
MaxFlushDelay: 50 * time.Millisecond,
})
if err != nil {
t.Fatal(err)
}
runWithWatchdog(t, 30*time.Second, func() {
set := ap.Set(ctx, "ap:himport:plain", "v", 0)
h1 := ap.HImportSet(ctx, "ap:himport:h", "fieldset-a", "f1", "v1")
h2 := ap.HImportSet(ctx, "ap:himport:h", "fieldset-b", "f2", "v2")
disc := ap.HImportDiscard(ctx, "fieldset-a")
// The HIMPORT replies depend on server support — only completion
// matters here (pre-fix this blocked forever). The plain SET in the
// same batch must succeed.
_ = h1.Err()
_ = h2.Err()
_ = disc.Err()
if err := set.Err(); err != nil {
t.Errorf("plain SET in himport batch: %v", err)
}
})
}
// Regression: Publish (and XRead's lag metric) read the command result only
// to record OTel metrics; with no recorder installed that read awaited
// execution, silently turning fire-and-forget publishes on the async face
// into blocking calls. Submission of a window must complete well inside the
// flush delay — i.e. without waiting for any result.
func TestAsyncAutoPipelinePublishDoesNotAwait(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 1000,
MaxFlushDelay: 100 * time.Millisecond,
})
if err != nil {
t.Fatal(err)
}
start := time.Now()
cmds := make([]*redis.IntCmd, 100)
for i := range cmds {
cmds[i] = ap.Publish(ctx, "ap:pub:chan", i)
}
submitted := time.Since(start)
// Generous bound: submission is pure enqueueing (microseconds); awaiting
// even one result would cost the 100ms flush delay.
if submitted > 50*time.Millisecond {
t.Fatalf("submitting 100 publishes took %v — a result read is awaiting execution", submitted)
}
for i, cmd := range cmds {
if err := cmd.Err(); err != nil {
t.Fatalf("publish %d: %v", i, err)
}
}
}
// routeRecordingHook records which dispatch path each command takes: names
// seen by the pipeline hook rode a batch; names seen by the process hook ran
// directly on their own connection.
type routeRecordingHook struct {
mu *sync.Mutex
pipeline map[string]bool
direct map[string]bool
}
func (h routeRecordingHook) DialHook(next redis.DialHook) redis.DialHook { return next }
func (h routeRecordingHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
return func(ctx context.Context, cmd redis.Cmder) error {
h.mu.Lock()
h.direct[cmd.Name()] = true
h.mu.Unlock()
return next(ctx, cmd)
}
}
func (h routeRecordingHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return func(ctx context.Context, cmds []redis.Cmder) error {
h.mu.Lock()
for _, cmd := range cmds {
h.pipeline[cmd.Name()] = true
}
h.mu.Unlock()
return next(ctx, cmds)
}
}
// Regression: SHUTDOWN must never ride a shared pipeline batch — the server
// exits before replying, so every batchmate would fail with EOF and the
// whole batch would be retried against a dead server (review finding by
// cxljs on the pool PR). It runs directly, like a blocking command. The
// client points at a closed port so nothing real is shut down: only the
// routing is asserted, via hooks that record which path each command took.
func TestAutoPipelineShutdownRunsOutsidePipeline(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
deadAddr := ln.Addr().String()
ln.Close() // guaranteed-refused port
hook := routeRecordingHook{mu: &sync.Mutex{}, pipeline: map[string]bool{}, direct: map[string]bool{}}
client := redis.NewClient(&redis.Options{Addr: deadAddr, MaxRetries: -1})
defer client.Close()
client.AddHook(hook)
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 10,
MaxFlushDelay: 10 * time.Millisecond,
})
if err != nil {
t.Fatal(err)
}
set := ap.Set(context.Background(), "k", "v", 0)
down := ap.Shutdown(context.Background())
mon := ap.Monitor(context.Background(), make(chan string, 1))
_ = set.Err() // dial refused — only routing matters
_ = down.Err() // dial refused — only routing matters
_ = mon.Err() // dial refused — only routing matters
hook.mu.Lock()
defer hook.mu.Unlock()
for _, name := range []string{"shutdown", "monitor"} {
if !hook.direct[name] {
t.Fatalf("%s did not take the direct (outside-pipeline) path", name)
}
if hook.pipeline[name] {
t.Fatalf("%s rode a pipeline batch — it would poison its batchmates", name)
}
}
if !hook.pipeline["set"] && !hook.direct["set"] {
t.Fatal("SET was never dispatched at all")
}
}
// armablePostNextPipelineHook calls next and then, when armed, returns its
// own error regardless of the exec outcome — the node-level twin of ofek's
// post-next-verdict case. Disarmed it is transparent, so conn-init
// handshakes (which pipeline through the hook chain) pass during warmup.
type armablePostNextPipelineHook struct {
err error
armed *atomic.Bool
}
func (h armablePostNextPipelineHook) DialHook(next redis.DialHook) redis.DialHook { return next }
func (h armablePostNextPipelineHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
return next
}
func (h armablePostNextPipelineHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return func(ctx context.Context, cmds []redis.Cmder) error {
err := next(ctx, cmds)
if h.armed.Load() {
return h.err
}
return err
}
}
// TestClusterNodeHookPostNextError pins that a NODE-level hook's post-next
// verdict is applied to an all-clean sub-batch (mirroring dispatchCmds):
// the exec succeeds on the wire, the hook returns an error afterwards, and
// every command must carry that error instead of reporting success
// (suppressed Copilot finding on #3867: the verdict was dropped on the
// cluster node path).
func TestClusterNodeHookPostNextError(t *testing.T) {
ctx := context.Background()
errVerdict := errors.New("node post-next verdict")
c := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: []string{":16600", ":16601", ":16602"},
})
defer c.Close()
var armed atomic.Bool
c.OnNewNode(func(cl *redis.Client) {
cl.AddHook(armablePostNextPipelineHook{err: errVerdict, armed: &armed})
})
skipIfClusterUnhealthy(t, c)
runWithWatchdog(t, 60*time.Second, func() {
warm := c.Pipeline()
for i := 0; i < 12; i++ {
warm.Set(ctx, fmt.Sprintf("npn:%d", i), i, 0)
}
if _, err := warm.Exec(ctx); err != nil {
t.Fatalf("warmup: %v", err)
}
armed.Store(true)
defer armed.Store(false)
pipe := c.Pipeline()
cmds := make([]*redis.StatusCmd, 0, 12)
for i := 0; i < 12; i++ {
cmds = append(cmds, pipe.Set(ctx, fmt.Sprintf("npn:%d", i), i, 0))
}
_, _ = pipe.Exec(ctx)
for i, cmd := range cmds {
if !errors.Is(cmd.Err(), errVerdict) {
t.Fatalf("cmd %d: err = %v, want the node hook's post-next verdict", i, cmd.Err())
}
}
})
}
// Regression: ScanIterator on the deferred (async) face. Next() used to read
// the command's page/cursor fields without awaiting execution — the fetch
// only enqueues on this face — so it spun re-issuing SCANs with a stale
// cursor forever, flooding the engine with an unbounded batch and hanging
// the caller. Next() now awaits the fetch before reading.
func TestAsyncAutoPipelineScanIterator(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
if err := client.FlushDB(ctx).Err(); err != nil {
t.Fatal(err)
}
ap, err := client.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
defer ap.Close()
const n = 33
for i := 0; i < n; i++ {
if err := ap.Set(ctx, fmt.Sprintf("apiter:%d", i), i, 0).Err(); err != nil {
t.Fatal(err)
}
}
runWithWatchdog(t, 30*time.Second, func() {
var got int
iter := ap.Scan(ctx, 0, "apiter:*", 5).Iterator()
for iter.Next(ctx) {
got++
}
if err := iter.Err(); err != nil {
t.Fatalf("iterator: %v", err)
}
if got != n {
t.Fatalf("scanned %d keys, want %d", got, n)
}
})
}
// markNonPipelinable installs a resolver marking one command name with a
// fan-out request policy (ReqMultiShard), which cannot ride a pipeline.
func markNonPipelinable(c *redis.ClusterClient, name string) {
c.SetCommandInfoResolver(redis.NewCommandInfoResolver(func(ctx context.Context, cmd redis.Cmder) *routing.CommandPolicy {
if cmd.Name() == name {
return &routing.CommandPolicy{Request: routing.ReqMultiShard}
}
return nil
}))
}
// An explicitly configured custom fan-out policy unsupported by Pipeline is
// rejected before batch dispatch. This validation does not make ordinary
// pipelines atomic; Redis command errors can still accompany successful writes.
func TestClusterPipelineNonPipelinableAllOrNothing(t *testing.T) {
ctx := context.Background()
c := redis.NewClusterClient(&redis.ClusterOptions{Addrs: []string{":16600", ":16601", ":16602"}})
defer c.Close()
if err := c.Ping(ctx).Err(); err != nil {
t.Skipf("no cluster: %v", err)
}
markNonPipelinable(c, "getdel")
_ = c.Del(ctx, "aon:{a}:1", "aon:{a}:2").Err()
pipe := c.Pipeline()
set1 := pipe.Set(ctx, "aon:{a}:1", "v1", 0)
bad := pipe.GetDel(ctx, "aon:{a}:marked")
set2 := pipe.Set(ctx, "aon:{a}:2", "v2", 0)
_, execErr := pipe.Exec(ctx)
if execErr == nil {
t.Fatal("Exec must fail when the batch contains a non-pipelinable command")
}
for i, cmd := range []redis.Cmder{set1, bad, set2} {
if cmd.Err() == nil {
t.Fatalf("cmd %d: err = nil, want the policy error on every command (all-or-nothing)", i)
}
}
// Nothing may have executed.
if n, _ := c.Exists(ctx, "aon:{a}:1", "aon:{a}:2").Result(); n != 0 {
t.Fatalf("writes landed despite the failed batch: %d keys exist", n)
}
}
// Fan-out is rejected on ordered async streams and remains available with an
// explicit Unordered opt-in. Neither outcome affects neighboring batches.
func TestAPClusterFanoutRespectsOrdering(t *testing.T) {
for _, tc := range []struct {
name string
fullDuplex bool
unordered bool
}{
{name: "ordered"},
{name: "ordered-full-duplex", fullDuplex: true},
{name: "unordered", unordered: true},
} {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
c := redis.NewClusterClient(&redis.ClusterOptions{Addrs: []string{":16600", ":16601", ":16602"}})
defer c.Close()
skipIfClusterUnhealthy(t, c)
c.SetCommandInfoResolver(redis.NewCommandInfoResolver(func(_ context.Context, cmd redis.Cmder) *routing.CommandPolicy {
if cmd.Name() == "ping" {
return &routing.CommandPolicy{Request: routing.ReqAllShards, Response: routing.RespAllSucceeded}
}
return nil
}))
var hooksMu sync.Mutex
var hooks []routeRecordingHook
if err := c.ForEachMaster(ctx, func(_ context.Context, node *redis.Client) error {
hook := routeRecordingHook{mu: &sync.Mutex{}, pipeline: map[string]bool{}, direct: map[string]bool{}}
node.AddHook(hook)
hooksMu.Lock()
hooks = append(hooks, hook)
hooksMu.Unlock()
return nil
}); err != nil {
t.Fatal(err)
}
if len(hooks) < 2 {
t.Fatal("fan-out test requires multiple masters")
}
ap, err := c.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 100,
MaxFlushDelay: 20 * time.Millisecond,
FullDuplex: tc.fullDuplex,
Unordered: tc.unordered,
})
if err != nil {
t.Fatal(err)
}
key1 := fmt.Sprintf("apfanout:%s:{a}:1", tc.name)
key2 := fmt.Sprintf("apfanout:%s:{a}:2", tc.name)
defer c.Del(ctx, key1, key2)
runWithWatchdog(t, 30*time.Second, func() {
set1 := ap.Set(ctx, key1, "v1", 0)
ping := ap.Ping(ctx)
set2 := ap.Set(ctx, key2, "v2", 0)
got, err := ping.Result()
if tc.unordered {
if err != nil || got != "PONG" {
t.Fatalf("fan-out PING = %q, %v; want PONG", got, err)
}
} else if err == nil || !strings.Contains(err.Error(), "ordered AsyncAutoPipeline") {
t.Fatalf("fan-out PING = %q, %v; want ordering rejection", got, err)
}
if err := set1.Err(); err != nil {
t.Fatalf("neighboring set1 failed: %v", err)
}
if err := set2.Err(); err != nil {
t.Fatalf("neighboring set2 failed: %v", err)
}
})
for i, hook := range hooks {
hook.mu.Lock()
direct, pipelined := hook.direct["ping"], hook.pipeline["ping"]
hook.mu.Unlock()
if direct != tc.unordered || pipelined {
t.Fatalf("master %d: PING direct=%v pipelined=%v; want direct=%v and no pipelining", i, direct, pipelined, tc.unordered)
}
}
values, err := c.MGet(ctx, key1, key2).Result()
if err != nil || len(values) != 2 || values[0] != "v1" || values[1] != "v2" {
t.Fatalf("neighboring writes = %v, %v; want [v1 v2]", values, err)
}
})
}
}
// Regression: reading a Submit()-ed command's result accessors BEFORE calling
// Wait() must be safe — Submit now marks the command ready like every other
// async entry point, so Err()/Val() self-gate through await(). Previously
// this was a silent data race with the dispatch goroutine (this test trips
// -race on the unfixed code).
func TestAutoPipelineSubmitReadBeforeWait(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 100,
MaxFlushDelay: 10 * time.Millisecond,
})
if err != nil {
t.Fatal(err)
}
runWithWatchdog(t, 30*time.Second, func() {
for i := 0; i < 200; i++ {
cmd := redis.NewStatusCmd(ctx, "set", fmt.Sprintf("subrace:%d", i), i)
_ = ap.Submit(ctx, cmd) // future deliberately ignored
// Misuse-but-must-be-safe: read the command directly, no Wait().
if err := cmd.Err(); err != nil {
t.Fatalf("cmd %d: %v", i, err)
}
}
})
}
// batchSizeRecordingHook records the command count of every dispatch. A
// drain cycle that catches exactly one command takes the engine's SOLO fast
// path (ProcessHook, no pipeline) — timing-dependent on loaded runners — so
// solo dispatches are recorded as size-1 entries to keep the total exact.
type batchSizeRecordingHook struct {
mu *sync.Mutex
sizes *[]int
}
func (h batchSizeRecordingHook) DialHook(next redis.DialHook) redis.DialHook { return next }
func (h batchSizeRecordingHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
return func(ctx context.Context, cmd redis.Cmder) error {
if cmd.Name() == "set" { // only the test's own commands, not handshakes
h.mu.Lock()
*h.sizes = append(*h.sizes, 1)
h.mu.Unlock()
}
return next(ctx, cmd)
}
}
func (h batchSizeRecordingHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return func(ctx context.Context, cmds []redis.Cmder) error {
n := 0
for _, cmd := range cmds {
if cmd.Name() == "set" { // exclude conn-init handshake pipelines
n++
}
}
if n > 0 {
h.mu.Lock()
*h.sizes = append(*h.sizes, n)
h.mu.Unlock()
}
return next(ctx, cmds)
}
}
// TestAutoPipelineMaxBatchBytes pins the byte-volume flush cap: with large
// payloads and a small MaxBatchBytes, a window that would otherwise coalesce
// into one huge burst is split into several bounded batches (300 x 64KiB in
// one batch is ~19MB written down a single connection before any reply is
// read — enough to stall a constrained link past its write deadline).
func TestAutoPipelineMaxBatchBytes(t *testing.T) {
ctx := context.Background()
const payload = 64 * 1024
value := strings.Repeat("x", payload)
run := func(maxBytes int) []int {
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
var mu sync.Mutex
var sizes []int
client.AddHook(batchSizeRecordingHook{mu: &mu, sizes: &sizes})
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 100,
MaxFlushDelay: 30 * time.Millisecond, // would coalesce everything without the byte cap
MaxBatchBytes: maxBytes,
})
if err != nil {
t.Fatal(err)
}
cmds := make([]*redis.StatusCmd, 10)
for i := range cmds {
cmds[i] = ap.Set(ctx, fmt.Sprintf("mbb:%d", i), value, 0)
}
for i, cmd := range cmds {
if err := cmd.Err(); err != nil {
t.Fatalf("cmd %d: %v", i, err)
}
}
mu.Lock()
defer mu.Unlock()
out := make([]int, len(sizes))
copy(out, sizes)
return out
}
runWithWatchdog(t, 60*time.Second, func() {
capped := run(150 * 1024) // ~2 x 64KiB per batch
total, maxBatch := 0, 0
for _, n := range capped {
total += n
if n > maxBatch {
maxBatch = n
}
}
if total != 10 {
t.Fatalf("capped run dispatched %d commands, want 10", total)
}
if len(capped) < 4 {
t.Fatalf("capped run used %d batches (%v), want the byte cap to split into >= 4", len(capped), capped)
}
if maxBatch > 3 {
t.Fatalf("capped run had a %d-command batch (%v) — cap not enforced", maxBatch, capped)
}
// 0 no longer means "uncapped" (it now gets the 128 KiB default
// guardrail — see MaxBatchBytes's GoDoc); opt out explicitly with a
// value comfortably above this run's total payload (10 x 64 KiB).
uncapped := run(10 * 1024 * 1024)
utotal := 0
for _, n := range uncapped {
utotal += n
}
if utotal != 10 {
t.Fatalf("uncapped run dispatched %d commands, want 10", utotal)
}
if len(uncapped) > 4 {
// A slow runner can split a solo straggler off the window; the
// signal is "far fewer dispatches than the capped run", not an
// exact count.
t.Fatalf("uncapped control fragmented into %d batches (%v) — coalescing regressed", len(uncapped), uncapped)
}
})
}
// panicPipelineHook panics inside the pipeline hook chain when armed — a
// stand-in for a hook or command-encoder panic during dispatch.
type panicPipelineHook struct{ armed *atomic.Bool }
func (h panicPipelineHook) DialHook(next redis.DialHook) redis.DialHook { return next }
func (h panicPipelineHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
return next
}
func (h panicPipelineHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return func(ctx context.Context, cmds []redis.Cmder) error {
if h.armed.Load() {
panic("boom: injected dispatch panic")
}
return next(ctx, cmds)
}
}
// TestAutoPipelinePanicInDispatchWakesWaiters pins the engine's panic
// containment: a panic inside the dispatch (hook chain / exec) must not kill
// the process — on a plain client it unwinds into the caller, but the
// engine's dispatch goroutines have no caller. Every waiter must wake with a
// descriptive error, and the engine must stay usable afterwards.
func TestAutoPipelinePanicInDispatchWakesWaiters(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
var armed atomic.Bool
client.AddHook(panicPipelineHook{armed: &armed})
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 100,
MaxFlushDelay: 20 * time.Millisecond,
})
if err != nil {
t.Fatal(err)
}
runWithWatchdog(t, 30*time.Second, func() {
armed.Store(true)
cmds := make([]*redis.StatusCmd, 5)
for i := range cmds {
cmds[i] = ap.Set(ctx, fmt.Sprintf("pan:%d", i), i, 0)
}
for i, cmd := range cmds {
err := cmd.Err() // must wake, not hang, not crash
if err == nil || !strings.Contains(err.Error(), "panic during dispatch") {
t.Fatalf("cmd %d: err = %v, want the recovered-panic error", i, err)
}
}
armed.Store(false)
// Engine survived: the next round executes normally.
if err := ap.Set(ctx, "pan:after", "ok", 0).Err(); err != nil {
t.Fatalf("engine unusable after recovered panic: %v", err)
}
})
}
// TestAPBlockingContextCancelDoesNotCancelExecution is the blocking-face twin
// of TestAPContextCancelDoesNotCancelExecution: cancelling the caller's ctx
// concurrently with a blocking-face call must not cancel the command's
// execution — the batch runs on the engine's context, and the call returns
// the executed result.
func TestAPBlockingContextCancelDoesNotCancelExecution(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
ap, err := client.AutoPipeline()
if err != nil {
t.Fatal(err)
}
runWithWatchdog(t, 30*time.Second, func() {
cctx, cancel := context.WithCancel(ctx)
go func() {
// Cancel while the call is (likely) queued/flushing; the engine
// dispatches on its own context, so this must be irrelevant.
time.Sleep(time.Millisecond)
cancel()
}()
if err := ap.Set(cctx, "apbctx:k", "v", 0).Err(); err != nil {
t.Fatalf("blocking call should return the executed result after ctx cancel, got %v", err)
}
if v, _ := client.Get(ctx, "apbctx:k").Result(); v != "v" {
t.Fatalf("value not persisted: %q", v)
}
client.Del(ctx, "apbctx:k")
})
}
// Regression: a blocking command (BLPOP) submitted on the DEFERRED face must
// not stall the submitter — the call returns immediately and only the result
// accessors block (exactly like Do). Previously the diverted path executed
// inline, so an async ap.BLPop froze the caller for the full block duration.
func TestAsyncAutoPipelineBlockingCommandDoesNotStallSubmit(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
ap, err := client.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
runWithWatchdog(t, 30*time.Second, func() {
start := time.Now()
cmd := ap.BLPop(ctx, 500*time.Millisecond, "apblk:empty")
submitted := time.Since(start)
if submitted > 100*time.Millisecond {
t.Fatalf("async BLPop stalled the submitter for %v — must return immediately", submitted)
}
if err := cmd.Err(); err != redis.Nil { // times out empty -> Nil
t.Fatalf("BLPop result: %v, want redis.Nil", err)
}
if elapsed := time.Since(start); elapsed < 400*time.Millisecond {
t.Fatalf("BLPop completed in %v — did not actually block server-side", elapsed)
}
})
}
// TestWithTimeoutCloneCloseBlocksParentResurrection pins the shared-pool
// closed flag: closing an ordinary WithTimeout clone closes the SHARED pools,
// so the parent's AutoPipeline getters must return ErrClosed instead of
// building fresh flushers against dead pools.
func TestWithTimeoutCloneCloseBlocksParentResurrection(t *testing.T) {
ctx := context.Background()
parent := redis.NewClient(apTestOptions())
if err := parent.Ping(ctx).Err(); err != nil {
parent.Close()
t.Skipf("no redis: %v", err)
}
clone := parent.WithTimeout(5 * time.Second)
if err := clone.Close(); err != nil {
t.Fatalf("clone close: %v", err)
}
if _, err := parent.AutoPipeline(); err != redis.ErrClosed {
t.Fatalf("parent.AutoPipeline after clone Close = %v, want ErrClosed (shared pools are gone)", err)
}
if _, err := parent.AsyncAutoPipeline(); err != redis.ErrClosed {
t.Fatalf("parent.AsyncAutoPipeline after clone Close = %v, want ErrClosed", err)
}
_ = parent.Close()
}
// TestWithTimeoutCloneCloseStopsCachedAutoPipeliner pins the second half of
// the shared-closed contract: an autopipeliner built and CACHED before a
// WithTimeout clone closes the shared pools must refuse new work at submit
// with ErrClosed — not accept enqueues that then fail against dead pools one
// dispatch at a time (codex on #3942).
func TestWithTimeoutCloneCloseStopsCachedAutoPipeliner(t *testing.T) {
ctx := context.Background()
parent := redis.NewClient(apTestOptions())
if err := parent.Ping(ctx).Err(); err != nil {
parent.Close()
t.Skipf("no redis: %v", err)
}
defer parent.Close()
ap, err := parent.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
if err := ap.Set(ctx, "cloneclose:pre", "v", 0).Err(); err != nil {
t.Fatalf("pre-close set: %v", err)
}
clone := parent.WithTimeout(5 * time.Second)
if err := clone.Close(); err != nil {
t.Fatalf("clone close: %v", err)
}
if err := ap.Set(ctx, "cloneclose:post", "v", 0).Err(); err != redis.ErrClosed {
t.Fatalf("cached autopipeliner Set after clone Close = %v, want ErrClosed", err)
}
if err := ap.Do(ctx, "get", "cloneclose:post").Err(); err != redis.ErrClosed {
t.Fatalf("cached autopipeliner Do after clone Close = %v, want ErrClosed", err)
}
// Diverted commands must refuse too, not run against the closed pools.
if err := ap.Do(ctx, "client", "id").Err(); err != redis.ErrClosed {
t.Fatalf("diverted command after clone Close = %v, want ErrClosed", err)
}
}
// TestAPClusterWideCommandsDelegate pins that cluster-wide commands issued on
// the autopipeliner use ClusterClient's fan-out overrides rather than the
// generic single-shard routing (DBSize must sum every master).
func TestAPClusterWideCommandsDelegate(t *testing.T) {
ctx := context.Background()
c := redis.NewClusterClient(&redis.ClusterOptions{Addrs: []string{":16600", ":16601", ":16602"}})
defer c.Close()
skipIfClusterUnhealthy(t, c)
ap, err := c.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
// Keys hashed to different slots so the cluster-wide count differs from
// any single shard's.
for i := 0; i < 30; i++ {
if err := ap.Set(ctx, fmt.Sprintf("apwide:%d", i), i, 0).Err(); err != nil {
t.Fatal(err)
}
}
want, err := c.DBSize(ctx).Result()
if err != nil {
t.Fatal(err)
}
got, err := ap.DBSize(ctx).Result()
if err != nil {
t.Fatal(err)
}
if got != want {
t.Fatalf("ap.DBSize = %d, cluster-wide DBSize = %d — the autopipeliner must delegate to the fan-out override", got, want)
}
sha, err := ap.ScriptLoad(ctx, "return 1").Result()
if err != nil || sha == "" {
t.Fatalf("ap.ScriptLoad: %q, %v", sha, err)
}
exists, err := ap.ScriptExists(ctx, sha).Result()
if err != nil || len(exists) != 1 || !exists[0] {
t.Fatalf("script must exist on EVERY shard after delegated ScriptLoad: %v %v", exists, err)
}
}
// TestAsyncCommandStringAwaits pins String()'s accessor contract on the
// deferred face: formatting an in-flight command from another goroutine
// blocks until execution — reading the value field without that barrier is a
// data race with reply processing (review finding by ofek and cursor on
// #3942) — and then includes the executed result. The dispatch-goroutine
// case (a hook formatting its own batch) is covered by resultPeekHook, which
// also calls String and relies on the dispatch-gid guard in await.
func TestAsyncCommandStringAwaits(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(apTestOptions())
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
gate := make(chan struct{})
var armed atomic.Bool
client.AddHook(gateHook{gate: gate, armed: &armed})
ap, err := client.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
runWithWatchdog(t, 30*time.Second, func() {
armed.Store(true)
cmd := ap.Set(ctx, "strawait", "value-42", 0)
sCh := make(chan string, 1)
go func() { sCh <- cmd.String() }()
select {
case s := <-sCh:
t.Fatalf("String returned %q before the command executed", s)
case <-time.After(100 * time.Millisecond):
}
armed.Store(false)
close(gate)
s := <-sCh
if !strings.Contains(s, "set strawait value-42") || !strings.Contains(s, "OK") {
t.Fatalf("String after execution = %q, want args and the OK result", s)
}
})
}
// gateHook holds every dispatch (solo and pipelined) until gate closes,
// while armed.
type gateHook struct {
gate chan struct{}
armed *atomic.Bool
}
func (h gateHook) DialHook(next redis.DialHook) redis.DialHook { return next }
func (h gateHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
return func(ctx context.Context, cmd redis.Cmder) error {
if h.armed.Load() {
<-h.gate
}
return next(ctx, cmd)
}
}
func (h gateHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return func(ctx context.Context, cmds []redis.Cmder) error {
if h.armed.Load() {
<-h.gate
}
return next(ctx, cmds)
}
}
// ===== from autopipeline_fd_shards_test.go =====
// With N full-duplex engines on one client, every reply must
// still be the reply to its own command. Mis-correlation is the failure mode
// that matters -- N engines means N independent FIFO streams, and a command
// routed to engine A whose reply is read off engine B would corrupt data
// silently rather than error.
//
// Requires a local server (apTestAddr: REDIS_PORT, default 6379).
func TestFDShardsReplyCorrelation(t *testing.T) {
ctx := context.Background()
cl := redis.NewClient(&redis.Options{Addr: apTestAddr()})
defer cl.Close()
if err := cl.Ping(ctx).Err(); err != nil {
t.Skipf("no local redis: %v", err)
}
shards := 8
if v := os.Getenv("FD_SHARDS"); v != "" {
// Overridable so the same test can run at ONE engine, which is how the
// silent half-duplex fallback was first caught: 1 passed, 8 failed.
shards, _ = strconv.Atoi(v)
}
ap, err := cl.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
FullDuplex: true,
NumShards: shards,
MaxConcurrentBatches: 1,
})
if err != nil {
t.Fatalf("construct: %v", err)
}
defer ap.Close()
// Each key's value encodes the key, so a reply landing on the wrong command
// is caught rather than merely suspected.
const keys = 2000
for i := 0; i < keys; i++ {
k := fmt.Sprintf("fdshards:{%d}", i)
if err := ap.Set(ctx, k, fmt.Sprintf("val-%d", i), 0).Err(); err != nil {
t.Fatalf("set %d: %v", i, err)
}
}
var wg sync.WaitGroup
errs := make(chan error, keys)
for w := 0; w < 64; w++ {
wg.Add(1)
go func(w int) {
defer wg.Done()
for i := w; i < keys; i += 64 {
k := fmt.Sprintf("fdshards:{%d}", i)
want := fmt.Sprintf("val-%d", i)
got, err := ap.Get(ctx, k).Result()
if err != nil {
errs <- fmt.Errorf("get %s: %w", k, err)
return
}
if got != want {
errs <- fmt.Errorf("MIS-CORRELATED: %s got %q want %q", k, got, want)
return
}
}
}(w)
}
wg.Wait()
close(errs)
n := 0
for e := range errs {
if n < 5 {
t.Error(e)
}
n++
}
if n > 0 {
t.Fatalf("%d correlation failures across %d keys", n, keys)
}
// Per-caller ordering on the deferred face: write then read the SAME key
// without waiting in between. Key routing must put both on one engine.
for i := 0; i < 500; i++ {
k := fmt.Sprintf("fdshards:order:{%d}", i)
setF := ap.Set(ctx, k, "second", 0)
getF := ap.Get(ctx, k)
if err := setF.Err(); err != nil {
t.Fatalf("ordered set: %v", err)
}
got, err := getF.Result()
if err != nil {
t.Fatalf("ordered get: %v", err)
}
if got != "second" {
t.Fatalf("ORDER VIOLATION at %s: got %q, want the value written just before it", k, got)
}
}
t.Logf("ok: %d keys correlated, 500 same-key write-then-read ordered, %d engines", keys, shards)
}
// The two-sited rule: NumShards>1 under FullDuplex must actually ENGAGE full
// duplex, not fall back to half-duplex shards. Config() reports the EFFECTIVE
// state, so a fallback surfaces as FullDuplex:false.
func TestFDShardsKeepsFullDuplexOn(t *testing.T) {
ctx := context.Background()
cl := redis.NewClient(&redis.Options{Addr: apTestAddr()})
defer cl.Close()
if err := cl.Ping(ctx).Err(); err != nil {
t.Skipf("no local redis: %v", err)
}
ap, err := cl.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
FullDuplex: true,
NumShards: 8,
MaxConcurrentBatches: 1,
})
if err != nil {
t.Fatalf("construct: %v", err)
}
defer ap.Close()
if got := ap.Config(); !got.FullDuplex {
t.Fatal("FullDuplex reported false with NumShards=8: the engine fell back " +
"to half-duplex shards, which are unordered")
}
}
// Full duplex that cannot engage (no pipeline pool) falls back to round-robin
// half-duplex shards, which do not keep submit order. The NumShards>1 ordering
// check must then apply: the FullDuplex exemption follows the effective state,
// not the requested one. Construction only, so no server is needed.
func TestFDShardsFallbackStillNeedsUnordered(t *testing.T) {
cl := redis.NewClient(&redis.Options{
Addr: apTestAddr(),
PipelinePoolSize: -1,
})
defer cl.Close()
ap, err := cl.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
FullDuplex: true,
NumShards: 2,
MaxConcurrentBatches: 1,
})
if err == nil {
ap.Close()
t.Fatal("FullDuplex without a pipeline pool fell back to round-robin shards " +
"without the Unordered opt-in")
}
if !strings.Contains(err.Error(), "Unordered") {
t.Fatalf("error should name the Unordered opt-in, got: %v", err)
}
}
// More engines than the pipeline pool can hold must fail at construction rather
// than leave the surplus spilling to the main pool for the client's lifetime.
func TestFDShardsRejectsMoreEnginesThanPool(t *testing.T) {
ctx := context.Background()
cl := redis.NewClient(&redis.Options{
Addr: apTestAddr(),
PipelinePoolSize: 4,
})
defer cl.Close()
if err := cl.Ping(ctx).Err(); err != nil {
t.Skipf("no local redis: %v", err)
}
_, err := cl.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
FullDuplex: true,
NumShards: 8,
MaxConcurrentBatches: 1,
})
if err == nil {
t.Fatal("expected NumShards=8 with PipelinePoolSize=4 to be rejected")
}
if !strings.Contains(err.Error(), "pipeline pool") {
t.Fatalf("error should name the pipeline pool, got: %v", err)
}
}
// ===== from autopipeline_truth_test.go =====
// Truth battery: adversarial end-to-end validation of the AutoPipeliner's
// core contracts against a live server — per-goroutine ORDER, exact RESPONSE
// demultiplexing, no lost/duplicated executions — under both faces, ordered
// and unordered configs, chunked dispatch, mid-flight cancellation and
// close-drain. Written for the PR #3942 validation pass; every test derives
// an independent ground truth (a model value computed client-side, or a
// differential run against a plain client) rather than trusting the engine's
// own accounting.
func truthAddr() string { return apTestAddr() }
func newTruthClient(t *testing.T) *redis.Client {
t.Helper()
c := redis.NewClient(&redis.Options{Addr: truthAddr(), PoolSize: 64})
if err := c.Ping(context.Background()).Err(); err != nil {
c.Close()
t.Skipf("no redis: %v", err)
}
return c
}
// TestTruthBlockingPerGoroutineOrder pins the blocking face's ordering
// contract with a server-side proof: each goroutine RPUSHes its own strictly
// increasing sequence through a SHARED autopipeliner; the list on the server
// is the ground truth and must come back exactly 0..M-1 for every goroutine.
func TestTruthBlockingPerGoroutineOrder(t *testing.T) {
ctx := context.Background()
client := newTruthClient(t)
defer client.Close()
client.FlushDB(ctx)
ap, err := client.AutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 64, MaxConcurrentBatches: 8, Unordered: true, NumShards: 4,
})
if err != nil {
t.Fatal(err)
}
const G, M = 64, 200
runWithWatchdog(t, 60*time.Second, func() {
var wg sync.WaitGroup
for g := 0; g < G; g++ {
wg.Add(1)
go func(g int) {
defer wg.Done()
key := "order:blocking:" + strconv.Itoa(g)
for i := 0; i < M; i++ {
if err := ap.RPush(ctx, key, i).Err(); err != nil {
t.Errorf("g%d rpush %d: %v", g, i, err)
return
}
}
}(g)
}
wg.Wait()
})
for g := 0; g < G; g++ {
vals, err := client.LRange(ctx, "order:blocking:"+strconv.Itoa(g), 0, -1).Result()
if err != nil {
t.Fatalf("lrange g%d: %v", g, err)
}
if len(vals) != M {
t.Fatalf("g%d: %d elements, want %d (lost or duplicated commands)", g, len(vals), M)
}
for i, v := range vals {
if v != strconv.Itoa(i) {
t.Fatalf("g%d: position %d = %q, want %d (per-goroutine order violated)", g, i, v, i)
}
}
}
}
// TestTruthAsyncOrderedSingleGoroutine pins the deferred face's ordering on
// the ORDERED default config: one goroutine submits an increasing sequence
// without awaiting; the server-side list must be exactly in submit order,
// and a read submitted after a write in the same window must observe it.
func TestTruthAsyncOrderedSingleGoroutine(t *testing.T) {
ctx := context.Background()
client := newTruthClient(t)
defer client.Close()
client.FlushDB(ctx)
ap, err := client.AsyncAutoPipeline() // ordered default
if err != nil {
t.Fatal(err)
}
const N = 3000
runWithWatchdog(t, 60*time.Second, func() {
cmds := make([]*redis.IntCmd, N)
for i := 0; i < N; i++ {
cmds[i] = ap.RPush(ctx, "order:async", i)
}
// read-your-write inside the window: SET then GET, same key, no await
// between them.
set := ap.Set(ctx, "order:ryw", "written", 0)
get := ap.Get(ctx, "order:ryw")
for i, c := range cmds {
if err := c.Err(); err != nil {
t.Fatalf("rpush %d: %v", i, err)
}
}
if err := set.Err(); err != nil {
t.Fatal(err)
}
if v, err := get.Result(); err != nil || v != "written" {
t.Fatalf("read-your-write: v=%q err=%v, want %q", v, err, "written")
}
})
vals, err := client.LRange(ctx, "order:async", 0, -1).Result()
if err != nil {
t.Fatal(err)
}
if len(vals) != N {
t.Fatalf("%d elements, want %d", len(vals), N)
}
for i, v := range vals {
if v != strconv.Itoa(i) {
t.Fatalf("position %d = %q, want %d (submit order violated on ordered face)", i, v, i)
}
}
}
// TestTruthResponseDemux pins reply demultiplexing: every one of G*M GETs
// must return exactly the unique payload its OWN goroutine wrote — any
// cross-wiring of replies between commands or goroutines fails loudly.
// Run on the most adversarial config: unordered, striped, parallel batches.
func TestTruthResponseDemux(t *testing.T) {
ctx := context.Background()
client := newTruthClient(t)
defer client.Close()
client.FlushDB(ctx)
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 128, MaxConcurrentBatches: 32, Unordered: true, NumShards: 8,
})
if err != nil {
t.Fatal(err)
}
const G, M = 100, 300
// Unordered config: a GET may overtake its own SET by design, so the
// phases are separated — all writes awaited, then all reads. The demux
// property under test is untouched: every reply must match its OWN
// command's unique payload.
runWithWatchdog(t, 120*time.Second, func() {
var wg sync.WaitGroup
for g := 0; g < G; g++ {
wg.Add(1)
go func(g int) {
defer wg.Done()
sets := make([]*redis.StatusCmd, M)
for i := 0; i < M; i++ {
key := fmt.Sprintf("demux:%d:%d", g, i)
want := fmt.Sprintf("payload|g=%d|i=%d|%x", g, i, g*1_000_003+i*7919)
sets[i] = ap.Set(ctx, key, want, 0)
}
for i, s := range sets {
if err := s.Err(); err != nil {
t.Errorf("set g%d i%d: %v", g, i, err)
return
}
}
gets := make([]*redis.StringCmd, M)
for i := 0; i < M; i++ {
gets[i] = ap.Get(ctx, fmt.Sprintf("demux:%d:%d", g, i))
}
for i, gc := range gets {
want := fmt.Sprintf("payload|g=%d|i=%d|%x", g, i, g*1_000_003+i*7919)
got, err := gc.Result()
if err != nil {
t.Errorf("get g%d i%d: %v", g, i, err)
return
}
if got != want {
t.Errorf("DEMUX VIOLATION g%d i%d: got %q want %q", g, i, got, want)
return
}
}
}(g)
}
wg.Wait()
})
}
// TestTruthOrderedPairsAcrossGoroutines pins per-goroutine read-your-write
// on the ORDERED default config with many goroutines sharing the pipeliner:
// each goroutine interleaves SET/GET pairs on its own keys without any
// barrier — the ordered face must preserve each goroutine's submit order.
func TestTruthOrderedPairsAcrossGoroutines(t *testing.T) {
ctx := context.Background()
client := newTruthClient(t)
defer client.Close()
client.FlushDB(ctx)
ap, err := client.AsyncAutoPipeline() // ordered default
if err != nil {
t.Fatal(err)
}
const G, M = 64, 150
runWithWatchdog(t, 120*time.Second, func() {
var wg sync.WaitGroup
for g := 0; g < G; g++ {
wg.Add(1)
go func(g int) {
defer wg.Done()
for i := 0; i < M; i++ {
key := fmt.Sprintf("pair:%d:%d", g, i)
want := fmt.Sprintf("v|%d|%d", g, i)
set := ap.Set(ctx, key, want, 0)
get := ap.Get(ctx, key) // no await between: ordered face must sequence
if err := set.Err(); err != nil {
t.Errorf("set g%d i%d: %v", g, i, err)
return
}
got, err := get.Result()
if err != nil || got != want {
t.Errorf("ORDER VIOLATION g%d i%d: got %q err=%v want %q", g, i, got, err, want)
return
}
}
}(g)
}
wg.Wait()
})
}
// TestTruthCountersExact pins no-lost-no-duplicated execution: G goroutines
// INCRBY random deltas on one shared counter; the client-side sum is the
// ground truth and the final server value must equal it exactly. A dropped
// command, a duplicated retry, or a misrouted reply cannot pass.
func TestTruthCountersExact(t *testing.T) {
ctx := context.Background()
client := newTruthClient(t)
defer client.Close()
client.FlushDB(ctx)
for _, cfg := range []struct {
name string
opts *redis.AutoPipelineOptions
}{
{"ordered-default", nil},
{"unordered-parallel", &redis.AutoPipelineOptions{MaxBatchSize: 200, MaxConcurrentBatches: 40, Unordered: true, NumShards: 8}},
{"tiny-batches", &redis.AutoPipelineOptions{MaxBatchSize: 2, MaxConcurrentBatches: 4, Unordered: true}},
} {
t.Run(cfg.name, func(t *testing.T) {
var ap *redis.AutoPipeliner
var err error
if cfg.opts == nil {
ap, err = client.AsyncAutoPipeline()
} else {
ap, err = client.AsyncAutoPipelineWithOptions(cfg.opts)
}
if err != nil {
t.Fatal(err)
}
key := "counter:" + cfg.name
client.Del(ctx, key)
const G, M = 50, 200
var wantTotal atomic.Int64
runWithWatchdog(t, 90*time.Second, func() {
var wg sync.WaitGroup
for g := 0; g < G; g++ {
wg.Add(1)
go func(g int) {
defer wg.Done()
rng := rand.New(rand.NewSource(int64(g)*7919 + 1))
futs := make([]*redis.IntCmd, M)
deltas := make([]int64, M)
for i := 0; i < M; i++ {
deltas[i] = rng.Int63n(1000) - 500
futs[i] = ap.IncrBy(ctx, key, deltas[i])
}
for i, f := range futs {
if err := f.Err(); err != nil {
t.Errorf("incrby: %v", err)
return
}
wantTotal.Add(deltas[i])
}
}(g)
}
wg.Wait()
})
got, err := client.Get(ctx, key).Int64()
if err != nil {
t.Fatal(err)
}
if got != wantTotal.Load() {
t.Fatalf("counter drift: server=%d client-sum=%d (lost or duplicated executions)", got, wantTotal.Load())
}
})
}
}
// TestTruthDifferentialVsPlainClient runs an identical randomized mixed
// workload through a plain client (the model) and the autopipeliner (the
// subject) on disjoint keyspaces, then diffs the full keyspace dumps. Any
// divergence in values, hashes, or list contents is an engine correctness
// bug by construction.
func TestTruthDifferentialVsPlainClient(t *testing.T) {
ctx := context.Background()
client := newTruthClient(t)
defer client.Close()
client.FlushDB(ctx)
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 100, MaxConcurrentBatches: 16, Unordered: true, NumShards: 4,
})
if err != nil {
t.Fatal(err)
}
type op struct {
kind int
key, field string
val int64
}
const G, M = 32, 150
plans := make([][]op, G)
for g := 0; g < G; g++ {
rng := rand.New(rand.NewSource(int64(g) + 42))
plans[g] = make([]op, M)
for i := range plans[g] {
plans[g][i] = op{
kind: rng.Intn(4),
key: strconv.Itoa(rng.Intn(20)), // few keys -> heavy contention
field: "f" + strconv.Itoa(rng.Intn(5)),
val: rng.Int63n(100),
}
}
}
run := func(prefix string, exec redis.Cmdable) {
var wg sync.WaitGroup
for g := 0; g < G; g++ {
wg.Add(1)
go func(g int) {
defer wg.Done()
for _, o := range plans[g] {
key := prefix + o.key
var err error
switch o.kind {
case 0:
err = exec.IncrBy(ctx, key+":c", o.val).Err()
case 1:
err = exec.HIncrBy(ctx, key+":h", o.field, o.val).Err()
case 2:
err = exec.SAdd(ctx, key+":s", o.val).Err()
case 3:
err = exec.RPush(ctx, key+":l:"+strconv.Itoa(g), o.val).Err()
}
if err != nil {
t.Errorf("%s op: %v", prefix, err)
return
}
}
}(g)
}
wg.Wait()
}
runWithWatchdog(t, 120*time.Second, func() {
run("model:", client) // plain client = the model
run("subject:", ap) // autopipeliner = the subject
})
dump := func(prefix string) map[string]string {
out := map[string]string{}
keys, err := client.Keys(ctx, prefix+"*").Result()
if err != nil {
t.Fatal(err)
}
for _, k := range keys {
short := strings.TrimPrefix(k, prefix)
switch typ := client.Type(ctx, k).Val(); typ {
case "string":
out[short] = client.Get(ctx, k).Val()
case "hash":
out[short] = fmt.Sprint(client.HGetAll(ctx, k).Val())
case "set":
vals, _ := client.SMembers(ctx, k).Result()
m := map[string]bool{}
for _, v := range vals {
m[v] = true
}
out[short] = fmt.Sprint(len(m)) // set cardinality (order-free)
case "list":
out[short] = strings.Join(client.LRange(ctx, k, 0, -1).Val(), ",")
default:
out[short] = "type:" + typ
}
}
return out
}
model, subject := dump("model:"), dump("subject:")
if len(model) != len(subject) {
t.Fatalf("keyspace size differs: model=%d subject=%d", len(model), len(subject))
}
for k, mv := range model {
if sv, ok := subject[k]; !ok || sv != mv {
t.Errorf("DIVERGENCE key %q: model=%q subject=%q", k, mv, sv)
}
}
}
// TestTruthChunkedDispatchIntegrity drives the MaxBatchBytes chunked path
// with values large enough to split every drain into several chunks and
// verifies exact payload echo — chunk boundaries must not tear, reorder, or
// cross-wire replies.
func TestTruthChunkedDispatchIntegrity(t *testing.T) {
ctx := context.Background()
client := newTruthClient(t)
defer client.Close()
client.FlushDB(ctx)
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 500, MaxConcurrentBatches: 8, Unordered: true,
MaxBatchBytes: 64 * 1024, // ~4 x 16KiB values per chunk
})
if err != nil {
t.Fatal(err)
}
const G, M = 16, 40
base := strings.Repeat("x", 16*1024)
runWithWatchdog(t, 120*time.Second, func() {
var wg sync.WaitGroup
for g := 0; g < G; g++ {
wg.Add(1)
go func(g int) {
defer wg.Done()
sets := make([]*redis.StatusCmd, M)
for i := 0; i < M; i++ {
key := fmt.Sprintf("chunk:%d:%d", g, i)
want := fmt.Sprintf("%s|tail=%d:%d", base, g, i)
sets[i] = ap.Set(ctx, key, want, 0)
}
for i, s := range sets {
if err := s.Err(); err != nil {
t.Errorf("set g%d i%d: %v", g, i, err)
return
}
}
for i := 0; i < M; i++ {
key := fmt.Sprintf("chunk:%d:%d", g, i)
want := fmt.Sprintf("%s|tail=%d:%d", base, g, i)
got, err := ap.Get(ctx, key).Result()
if err != nil {
t.Errorf("get %s: %v", key, err)
return
}
if got != want {
t.Errorf("CHUNK TEAR %s: got %d bytes tail %q, want %d bytes", key, len(got), got[len(got)-24:], len(want))
return
}
}
}(g)
}
wg.Wait()
})
}
// TestTruthCancelNeverLies pins cancellation semantics: a context canceled
// mid-flight may fail the WAIT, but any command that reports success must
// have really executed, and any that reports an error must not have half a
// result. Verified by re-reading the keyspace with a fresh context.
func TestTruthCancelNeverLies(t *testing.T) {
ctx := context.Background()
client := newTruthClient(t)
defer client.Close()
client.FlushDB(ctx)
ap, err := client.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
const N = 2000
runWithWatchdog(t, 60*time.Second, func() {
cctx, cancel := context.WithCancel(ctx)
futs := make([]*redis.StatusCmd, N)
for i := 0; i < N; i++ {
futs[i] = ap.Set(cctx, "cancel:"+strconv.Itoa(i), "v"+strconv.Itoa(i), 0)
if i == N/2 {
cancel()
}
}
okCount, errCount := 0, 0
for i, f := range futs {
if err := f.Err(); err == nil {
okCount++
// claimed success must be true on the server
v, gerr := client.Get(ctx, "cancel:"+strconv.Itoa(i)).Result()
if gerr != nil || v != "v"+strconv.Itoa(i) {
t.Fatalf("cmd %d reported OK but server has %q/%v", i, v, gerr)
}
} else {
errCount++
}
}
t.Logf("cancel split: %d ok, %d error (both outcomes legal; lies are not)", okCount, errCount)
cancel()
})
}
// TestTruthCloseDrains pins Close's drain contract under load: every future
// submitted BEFORE Close returns a definite outcome (value or error, no
// hang), and successes are really on the server.
func TestTruthCloseDrains(t *testing.T) {
ctx := context.Background()
client := newTruthClient(t)
defer client.Close()
client.FlushDB(ctx)
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 64, MaxConcurrentBatches: 8, Unordered: true,
})
if err != nil {
t.Fatal(err)
}
const N = 5000
runWithWatchdog(t, 60*time.Second, func() {
futs := make([]*redis.StatusCmd, N)
for i := 0; i < N; i++ {
futs[i] = ap.Set(ctx, "drain:"+strconv.Itoa(i), "v", 0)
}
if err := ap.Close(); err != nil {
t.Fatalf("close: %v", err)
}
okCount := 0
for i, f := range futs {
if err := f.Err(); err == nil {
okCount++
} else if err != redis.ErrClosed {
t.Fatalf("future %d: unexpected error %v", i, err)
}
}
// Every claimed success must exist server-side.
var present int64
for i := 0; i < N; i++ {
if client.Exists(ctx, "drain:"+strconv.Itoa(i)).Val() == 1 {
present++
}
}
if int(present) < okCount {
t.Fatalf("%d futures claimed OK but only %d keys present", okCount, present)
}
t.Logf("close drain: %d/%d confirmed executed, rest cleanly rejected", okCount, N)
})
}
// TestTruthClusterOrderAndDemux repeats the order and demux proofs against a
// live cluster through the sharded autopipeliner: per-goroutine order on
// hashtagged keys (one slot per goroutine, so order is meaningful) and exact
// payload echo across slot-spread keys.
func TestTruthClusterOrderAndDemux(t *testing.T) {
ctx := context.Background()
cc := redis.NewClusterClient(&redis.ClusterOptions{Addrs: []string{":16600", ":16601", ":16602"}})
defer cc.Close()
skipIfClusterUnhealthy(t, cc)
for _, n := range []string{":16600", ":16601", ":16602"} {
nc := redis.NewClient(&redis.Options{Addr: n})
nc.FlushDB(ctx)
nc.Close()
}
ap, err := cc.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
const G, M = 32, 100
runWithWatchdog(t, 120*time.Second, func() {
var wg sync.WaitGroup
for g := 0; g < G; g++ {
wg.Add(1)
go func(g int) {
defer wg.Done()
// order proof: hashtag pins the goroutine's list to one slot
key := fmt.Sprintf("{ord%d}:list", g)
futs := make([]*redis.IntCmd, M)
for i := 0; i < M; i++ {
futs[i] = ap.RPush(ctx, key, i)
}
for i, f := range futs {
if err := f.Err(); err != nil {
t.Errorf("g%d rpush %d: %v", g, i, err)
return
}
}
// demux proof: slot-spread unique echoes, phase-separated
sets := make([]*redis.StatusCmd, M)
for i := 0; i < M; i++ {
sets[i] = ap.Set(ctx, fmt.Sprintf("cdemux:%d:%d", g, i), fmt.Sprintf("p|%d|%d", g, i), 0)
}
for i, s := range sets {
if err := s.Err(); err != nil {
t.Errorf("g%d set %d: %v", g, i, err)
return
}
}
for i := 0; i < M; i++ {
want := fmt.Sprintf("p|%d|%d", g, i)
got, err := ap.Get(ctx, fmt.Sprintf("cdemux:%d:%d", g, i)).Result()
if err != nil || got != want {
t.Errorf("CLUSTER DEMUX g%d i%d: got %q err=%v want %q", g, i, got, err, want)
return
}
}
}(g)
}
wg.Wait()
})
for g := 0; g < G; g++ {
vals, err := cc.LRange(ctx, fmt.Sprintf("{ord%d}:list", g), 0, -1).Result()
if err != nil {
t.Fatal(err)
}
if len(vals) != M {
t.Fatalf("cluster g%d: %d elements want %d", g, len(vals), M)
}
for i, v := range vals {
if v != strconv.Itoa(i) {
t.Fatalf("CLUSTER ORDER g%d: pos %d = %q want %d", g, i, v, i)
}
}
}
}
// TestTruthRESP2Demux repeats the demultiplexing proof on RESP2, where
// replies carry no push-frame framing and a desynchronized stream shows up
// as cross-wired values rather than a protocol error.
func TestTruthRESP2Demux(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(&redis.Options{Addr: truthAddr(), Protocol: 2, PoolSize: 32})
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
client.FlushDB(ctx)
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 100, MaxConcurrentBatches: 16, Unordered: true, NumShards: 4,
})
if err != nil {
t.Fatal(err)
}
const G, M = 40, 150
runWithWatchdog(t, 90*time.Second, func() {
var wg sync.WaitGroup
for g := 0; g < G; g++ {
wg.Add(1)
go func(g int) {
defer wg.Done()
sets := make([]*redis.StatusCmd, M)
for i := 0; i < M; i++ {
sets[i] = ap.Set(ctx, fmt.Sprintf("r2:%d:%d", g, i), fmt.Sprintf("v|%d|%d", g, i), 0)
}
for _, s := range sets {
if err := s.Err(); err != nil {
t.Errorf("resp2 set: %v", err)
return
}
}
// mixed reply SHAPES in one batch: string, integer, array,
// hash, nil — a demux slip shows up as a type/shape mismatch.
for i := 0; i < M; i++ {
key := fmt.Sprintf("r2:%d:%d", g, i)
sv := ap.Get(ctx, key)
iv := ap.StrLen(ctx, key)
av := ap.LRange(ctx, "r2:missing:list", 0, -1)
nv := ap.Get(ctx, "r2:missing:key")
want := fmt.Sprintf("v|%d|%d", g, i)
if got, err := sv.Result(); err != nil || got != want {
t.Errorf("RESP2 string demux g%d i%d: %q %v", g, i, got, err)
return
}
if got, err := iv.Result(); err != nil || got != int64(len(want)) {
t.Errorf("RESP2 int demux g%d i%d: %d %v want %d", g, i, got, err, len(want))
return
}
if got, err := av.Result(); err != nil || len(got) != 0 {
t.Errorf("RESP2 array demux g%d i%d: %v %v", g, i, got, err)
return
}
if _, err := nv.Result(); err != redis.Nil {
t.Errorf("RESP2 nil demux g%d i%d: err=%v want redis.Nil", g, i, err)
return
}
}
}(g)
}
wg.Wait()
})
}
// TestTruthPipelinePoolInterleaving runs the autopipeliner against a client
// configured with the DEDICATED pipeline pool while a plain client hammers
// the same server through the regular pool: batched traffic and single
// commands must not contaminate each other's connections.
func TestTruthPipelinePoolInterleaving(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(&redis.Options{
Addr: truthAddr(), PoolSize: 32,
PipelineReadBufferSize: 128 << 10, PipelineWriteBufferSize: 128 << 10,
PipelinePoolSize: 4,
})
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
client.FlushDB(ctx)
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 128, MaxConcurrentBatches: 16, Unordered: true,
})
if err != nil {
t.Fatal(err)
}
const G, M = 24, 200
runWithWatchdog(t, 120*time.Second, func() {
var wg sync.WaitGroup
// plain-client noise on the regular pool
stop := make(chan struct{})
for n := 0; n < 8; n++ {
wg.Add(1)
go func(n int) {
defer wg.Done()
for i := 0; ; i++ {
select {
case <-stop:
return
default:
}
key := fmt.Sprintf("noise:%d", n)
want := fmt.Sprintf("n|%d|%d", n, i)
if err := client.Set(ctx, key, want, 0).Err(); err != nil {
t.Errorf("noise set: %v", err)
return
}
if got, err := client.Get(ctx, key).Result(); err != nil || got != want {
t.Errorf("NOISE CONTAMINATION: got %q err=%v want %q", got, err, want)
return
}
}
}(n)
}
// autopipelined traffic on the pipeline pool
var apWg sync.WaitGroup
for g := 0; g < G; g++ {
apWg.Add(1)
go func(g int) {
defer apWg.Done()
sets := make([]*redis.StatusCmd, M)
for i := 0; i < M; i++ {
sets[i] = ap.Set(ctx, fmt.Sprintf("pp:%d:%d", g, i), fmt.Sprintf("p|%d|%d", g, i), 0)
}
for _, s := range sets {
if err := s.Err(); err != nil {
t.Errorf("pp set: %v", err)
return
}
}
for i := 0; i < M; i++ {
want := fmt.Sprintf("p|%d|%d", g, i)
got, err := ap.Get(ctx, fmt.Sprintf("pp:%d:%d", g, i)).Result()
if err != nil || got != want {
t.Errorf("PIPELINE POOL DEMUX g%d i%d: got %q err=%v want %q", g, i, got, err, want)
return
}
}
}(g)
}
apWg.Wait()
close(stop)
wg.Wait()
})
}
// TestTruthRetriesDoNotDuplicate pins at-most-once execution across the
// engine's retry machinery: non-idempotent INCRs run through an
// autopipeliner whose client has retries ENABLED while the server is
// pounded concurrently; the final counter must equal the exact number of
// successful futures — a retried batch that re-executed a committed command
// would overshoot.
func TestTruthRetriesDoNotDuplicate(t *testing.T) {
ctx := context.Background()
client := redis.NewClient(&redis.Options{Addr: truthAddr(), PoolSize: 32, MaxRetries: 3})
defer client.Close()
if err := client.Ping(ctx).Err(); err != nil {
t.Skipf("no redis: %v", err)
}
client.FlushDB(ctx)
ap, err := client.AsyncAutoPipelineWithOptions(&redis.AutoPipelineOptions{
MaxBatchSize: 64, MaxConcurrentBatches: 16, Unordered: true, NumShards: 4,
})
if err != nil {
t.Fatal(err)
}
const G, M = 40, 250
var okCount atomic.Int64
runWithWatchdog(t, 120*time.Second, func() {
var wg sync.WaitGroup
for g := 0; g < G; g++ {
wg.Add(1)
go func(g int) {
defer wg.Done()
futs := make([]*redis.IntCmd, M)
for i := 0; i < M; i++ {
futs[i] = ap.Incr(ctx, "retry:counter")
}
for _, f := range futs {
if err := f.Err(); err == nil {
okCount.Add(1)
}
}
}(g)
}
wg.Wait()
})
got, err := client.Get(ctx, "retry:counter").Int64()
if err != nil {
t.Fatal(err)
}
if got != okCount.Load() {
t.Fatalf("at-most-once violated: counter=%d successful futures=%d", got, okCount.Load())
}
t.Logf("at-most-once: %d increments == %d successful futures", got, okCount.Load())
}
// ===== from osscluster_autopipeline_fd_moved_test.go =====
// clusterFDAddrs is the local 6-node cluster (3 masters + 3 replicas) the FD
// MOVED tests target; SwapNodes needs the replicas to swap a slot's master.
var clusterFDAddrs = []string{":16600", ":16601", ":16602", ":16603", ":16604", ":16605"}
// TestAPClusterFDMovedRedirectFollowed is the discriminating proof that native
// cluster full-duplex FOLLOWS a MOVED redirect. SwapNodes forces the client to
// believe a replica is the master for a slot; a command routed there is answered
// with MOVED. The FD child's reader diverts the redirect to the redirect-aware
// ClusterClient path, which re-routes to the real master and reloads topology, so
// the value is still read back correctly. Before the redirect fix this returned
// the MOVED error and the test failed.
func TestAPClusterFDMovedRedirectFollowed(t *testing.T) {
ctx := context.Background()
c := redis.NewClusterClient(&redis.ClusterOptions{Addrs: clusterFDAddrs})
defer c.Close()
skipIfClusterUnhealthy(t, c)
ap, err := c.AutoPipelineWithOptions(&redis.AutoPipelineOptions{FullDuplex: true})
if err != nil {
t.Fatal(err)
}
defer ap.Close()
if !ap.Config().FullDuplex {
t.Fatalf("Config().FullDuplex = false, want true (cluster FD not active)")
}
const key, val = "fdmoved:key", "fdmoved:val"
if err := ap.Set(ctx, key, val, 0).Err(); err != nil {
t.Fatalf("initial set: %v", err)
}
// Point the slot at the wrong node so the next access is answered with MOVED
// and must be followed.
if err := c.SwapNodes(ctx, key); err != nil {
t.Fatalf("SwapNodes: %v", err)
}
got, err := ap.Get(ctx, key).Result()
if err != nil {
t.Fatalf("get after MOVED: %v (FD redirect not followed)", err)
}
if got != val {
t.Fatalf("get after MOVED = %q, want %q (FD redirect not followed correctly)", got, val)
}
}
// TestAPClusterFDMovedSurfacedWithoutRedirects anchors the positive test: with
// MaxRedirects disabled (-1 -> 0, one attempt), the FD divert still routes through
// ClusterClient.process, which now honors the zero-redirect budget and surfaces
// the forced MOVED as an error. This proves SwapNodes really induced a redirect on
// the FD path (so the positive test exercised the redirect logic, not a trivial
// pass) and that the FD reprocess goes through the cluster budget.
func TestAPClusterFDMovedSurfacedWithoutRedirects(t *testing.T) {
ctx := context.Background()
c := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: clusterFDAddrs,
MaxRedirects: -1, // normalized to 0: a single attempt, no follow-through
})
defer c.Close()
skipIfClusterUnhealthy(t, c)
ap, err := c.AutoPipelineWithOptions(&redis.AutoPipelineOptions{FullDuplex: true})
if err != nil {
t.Fatal(err)
}
defer ap.Close()
if !ap.Config().FullDuplex {
t.Fatalf("Config().FullDuplex = false, want true (cluster FD not active)")
}
const key = "fdmoved:noredir:key"
if err := ap.Set(ctx, key, "v", 0).Err(); err != nil {
t.Fatalf("initial set: %v", err)
}
// SwapNodes is racy against the periodic LazyReload, so re-swap and retry until
// we observe the MOVED (or give up and skip — the swap never stuck).
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
if err := c.SwapNodes(ctx, key); err != nil {
t.Fatalf("SwapNodes: %v", err)
}
err := ap.Get(ctx, key).Err()
if err != nil && strings.Contains(err.Error(), "MOVED") {
return // proven: the redirect was needed and, with no redirects, surfaced
}
time.Sleep(100 * time.Millisecond)
}
t.Skip("could not observe a MOVED (swap kept being reverted by topology reload)")
}
// ===== from osscluster_autopipeline_nodehook_test.go =====
type nodePeekHook struct{ armed *atomic.Bool }
func (h nodePeekHook) DialHook(next redis.DialHook) redis.DialHook { return next }
func (h nodePeekHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { return next }
func (h nodePeekHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return func(ctx context.Context, cmds []redis.Cmder) error {
if h.armed.Load() {
for _, c := range cmds {
_ = c.Err() // redisotel via rediscmd.CmdsString does exactly this pre-next
_ = c.String()
}
}
err := next(ctx, cmds)
if h.armed.Load() {
for _, c := range cmds {
_ = c.Err()
_ = c.String()
}
}
return err
}
}
// TestClusterNodeHookResultPeekNoDeadlock pins the batch-executor guard for
// cluster per-node goroutines: redisotel installs tracing hooks on NODE
// clients via OnNewNode, and rediscmd.CmdsString reads every command's Err()
// BEFORE next() — on the deferred face those reads used to block on a batch
// that completes only after the node call returns, wedging the dispatch
// permanently (found while assessing the String review round on #3942).
func TestClusterNodeHookResultPeekNoDeadlock(t *testing.T) {
ctx := context.Background()
probe := redis.NewClusterClient(&redis.ClusterOptions{Addrs: []string{":16600", ":16601", ":16602"}})
if err := probe.Ping(ctx).Err(); err != nil {
probe.Close()
t.Skipf("no cluster: %v", err)
}
probe.Close()
cc := redis.NewClusterClient(&redis.ClusterOptions{Addrs: []string{":16600", ":16601", ":16602"}})
defer cc.Close()
var armed atomic.Bool
var installs atomic.Int32
cc.OnNewNode(func(nc *redis.Client) {
installs.Add(1)
nc.AddHook(nodePeekHook{armed: &armed})
})
cc.Ping(ctx)
t.Logf("node hooks installed: %d", installs.Load())
ap, err := cc.AsyncAutoPipeline()
if err != nil {
t.Fatal(err)
}
done := make(chan error, 1)
go func() {
armed.Store(true)
defer armed.Store(false)
cmds := make([]*redis.StatusCmd, 50)
for i := range cmds {
cmds[i] = ap.Set(ctx, "nodehook:"+string(rune('a'+i%26))+":"+string(rune('0'+i%10)), "v", 0)
}
var firstErr error
for _, c := range cmds {
if err := c.Err(); err != nil && firstErr == nil {
firstErr = err
}
}
done <- firstErr
}()
select {
case err := <-done:
t.Logf("no hang, err=%v", err)
case <-time.After(10 * time.Second):
t.Fatal("node-level hook result peek wedged the async cluster dispatch (batch executor registration regressed)")
}
}