1
0
mirror of https://github.com/redis/go-redis.git synced 2025-07-29 17:41:15 +03:00
This commit is contained in:
ofekshenawa
2024-06-20 02:30:37 +03:00
commit 0b95fd7fa5
188 changed files with 45644 additions and 0 deletions

View File

@ -0,0 +1,11 @@
# Delete keys without a ttl
This example demonstrates how to use `SCAN` and pipelines to efficiently delete keys without a TTL.
To run this example:
```shell
go run .
```
See [documentation](https://redis.uptrace.dev/guide/get-all-keys.html) for more details.

View File

@ -0,0 +1,17 @@
module github.com/redis/go-redis/example/del-keys-without-ttl
go 1.18
replace github.com/redis/go-redis/v9 => ../..
require (
github.com/redis/go-redis/v9 v9.5.3
go.uber.org/zap v1.24.0
)
require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
)

View File

@ -0,0 +1,19 @@
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View File

@ -0,0 +1,155 @@
package main
import (
"context"
"fmt"
"sync"
"time"
"go.uber.org/zap"
"github.com/redis/go-redis/v9"
)
func main() {
ctx := context.Background()
rdb := redis.NewClient(&redis.Options{
Addr: ":6379",
})
_ = rdb.Set(ctx, "key_with_ttl", "bar", time.Minute).Err()
_ = rdb.Set(ctx, "key_without_ttl_1", "", 0).Err()
_ = rdb.Set(ctx, "key_without_ttl_2", "", 0).Err()
checker := NewKeyChecker(rdb, 100)
start := time.Now()
checker.Start(ctx)
iter := rdb.Scan(ctx, 0, "", 0).Iterator()
for iter.Next(ctx) {
checker.Add(iter.Val())
}
if err := iter.Err(); err != nil {
panic(err)
}
deleted := checker.Stop()
fmt.Println("deleted", deleted, "keys", "in", time.Since(start))
}
type KeyChecker struct {
rdb *redis.Client
batchSize int
ch chan string
delCh chan string
wg sync.WaitGroup
deleted int
logger *zap.Logger
}
func NewKeyChecker(rdb *redis.Client, batchSize int) *KeyChecker {
return &KeyChecker{
rdb: rdb,
batchSize: batchSize,
ch: make(chan string, batchSize),
delCh: make(chan string, batchSize),
logger: zap.L(),
}
}
func (c *KeyChecker) Add(key string) {
c.ch <- key
}
func (c *KeyChecker) Start(ctx context.Context) {
c.wg.Add(1)
go func() {
defer c.wg.Done()
if err := c.del(ctx); err != nil {
panic(err)
}
}()
c.wg.Add(1)
go func() {
defer c.wg.Done()
defer close(c.delCh)
keys := make([]string, 0, c.batchSize)
for key := range c.ch {
keys = append(keys, key)
if len(keys) < cap(keys) {
continue
}
if err := c.checkKeys(ctx, keys); err != nil {
c.logger.Error("checkKeys failed", zap.Error(err))
}
keys = keys[:0]
}
if len(keys) > 0 {
if err := c.checkKeys(ctx, keys); err != nil {
c.logger.Error("checkKeys failed", zap.Error(err))
}
keys = nil
}
}()
}
func (c *KeyChecker) Stop() int {
close(c.ch)
c.wg.Wait()
return c.deleted
}
func (c *KeyChecker) checkKeys(ctx context.Context, keys []string) error {
cmds, err := c.rdb.Pipelined(ctx, func(pipe redis.Pipeliner) error {
for _, key := range keys {
pipe.TTL(ctx, key)
}
return nil
})
if err != nil {
return err
}
for i, cmd := range cmds {
d, err := cmd.(*redis.DurationCmd).Result()
if err != nil {
return err
}
if d == -1 {
c.delCh <- keys[i]
}
}
return nil
}
func (c *KeyChecker) del(ctx context.Context) error {
pipe := c.rdb.Pipeline()
for key := range c.delCh {
fmt.Printf("deleting %s...\n", key)
pipe.Del(ctx, key)
c.deleted++
if pipe.Len() < c.batchSize {
continue
}
if _, err := pipe.Exec(ctx); err != nil {
return err
}
}
if _, err := pipe.Exec(ctx); err != nil {
return err
}
return nil
}