package/delayqueue
1
0
Fork 0
mirror of https://github.com/HDT3213/delayqueue.git synced 2025-09-27 03:26:05 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

support redis cluster

Browse Source
This commit is contained in:
finley
2023-08-20 17:34:43 +08:00
parent a8447bd9ff
commit 023e1604b0
7 changed files with 341 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
README.md
View File

@@ -16,7 +16,8 @@ Core Advantages:
- Auto retry failed messages - Auto retry failed messages
- Works out of the box, Config Nothing and Deploy Nothing, A Redis is all you need. - Works out of the box, Config Nothing and Deploy Nothing, A Redis is all you need.
- Natively adapted to the distributed environment, messages processed concurrently on multiple machines - Natively adapted to the distributed environment, messages processed concurrently on multiple machines
. workers can be added, removed or migrated at any time . Workers can be added, removed or migrated at any time
- Support Redis Cluster for high availability
## Install ## Install
@@ -26,8 +27,6 @@ DelayQueue requires a Go version with modules support. Run following command lin
go get github.com/hdt3213/delayqueue go get github.com/hdt3213/delayqueue
``` ```
> if you are using github.com/go-redis/redis/v8 please use `go get github.com/hdt3213/delayqueue@v8`
## Get Started ## Get Started
```go ```go
@@ -69,6 +68,9 @@ func main() {
} }
``` ```
> if you are using github.com/go-redis/redis/v8 please use `go get github.com/hdt3213/delayqueue@v8`
> If you are using redis client other than go-redis, you could wrap your redis client into [RedisCli](https://pkg.go.dev/github.com/hdt3213/delayqueue#RedisCli) interface
## Options ## Options
```go ```go
@@ -125,7 +127,38 @@ WithDefaultRetryCount customizes the max number of retry, it effects of messages
use WithRetryCount during DelayQueue.SendScheduleMsg or DelayQueue.SendDelayMsg to specific retry count of particular message use WithRetryCount during DelayQueue.SendScheduleMsg or DelayQueue.SendDelayMsg to specific retry count of particular message
# More Details ## Cluster
If you are using Redis Cluster, please use `NewQueueOnCluster`
```go
redisCli := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: []string{
"127.0.0.1:7000",
"127.0.0.1:7001",
"127.0.0.1:7002",
},
})
callback := func(s string) bool {
return true
}
queue := NewQueueOnCluster("test", redisCli, callback)
```
If you are using transparent clusters, such as codis, twemproxy, or the redis of cluster architecture on aliyun, tencentcloud,
just use `NewQueue` and enable hash tag
```go
redisCli := redis.NewClient(&redis.Options{
Addr: "127.0.0.1:6379",
})
callback := func(s string) bool {
return true
}
queue := delayqueue.NewQueue("example", redisCli, callback, UseHashTagKey())
```
## More Details
Here is the complete flowchart: Here is the complete flowchart:

34
README_CN.md
View File

@@ -12,6 +12,7 @@ DelayQueue 的主要优势：
- 自动重试处理失败的消息 - 自动重试处理失败的消息
- 开箱即用, 无需部署或安装中间件, 只需要一个 Redis 即可工作 - 开箱即用, 无需部署或安装中间件, 只需要一个 Redis 即可工作
- 原生适配分布式环境, 可在多台机器上并发的处理消息. 可以随时增加、减少或迁移 Worker - 原生适配分布式环境, 可在多台机器上并发的处理消息. 可以随时增加、减少或迁移 Worker
- 支持各类 Redis 集群
# 安装 # 安装
@@ -64,6 +65,9 @@ func main() {
} }
``` ```
> 如果您仍在使用 redis/v8 请使用 v8 分支: `go get github.com/hdt3213/delayqueue@v8`
> 如果您在使用其他的 redis 客户端, 可以将其包装到 [RedisCli](https://pkg.go.dev/github.com/hdt3213/delayqueue#RedisCli) 接口中
# 选项 # 选项
```go ```go
@@ -117,6 +121,36 @@ WithDefaultRetryCount(count uint)
在调用 DelayQueue.SendScheduleMsg or DelayQueue.SendDelayMsg 发送消息时，可以调用 WithRetryCount 为这条消息单独指定重试次数。 在调用 DelayQueue.SendScheduleMsg or DelayQueue.SendDelayMsg 发送消息时，可以调用 WithRetryCount 为这条消息单独指定重试次数。
# 集群
如果需要在 Redis Cluster 上工作, 请使用 `NewQueueOnCluster`:
```go
redisCli := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: []string{
"127.0.0.1:7000",
"127.0.0.1:7001",
"127.0.0.1:7002",
},
})
callback := func(s string) bool {
return true
}
queue := NewQueueOnCluster("test", redisCli, callback)
```
如果是阿里云,腾讯云的 Redis 集群版或 codis, twemproxy 这类透明式的集群, 使用 `NewQueue` 并启用 UseHashTagKey() 即可:
```go
redisCli := redis.NewClient(&redis.Options{
Addr: "127.0.0.1:6379",
})
callback := func(s string) bool {
return true
}
queue := delayqueue.NewQueue("example", redisCli, callback, UseHashTagKey())
```
# 更多细节 # 更多细节
完整流程如图所示： 完整流程如图所示：

97
delayqueue.go
View File

@@ -1,11 +1,11 @@
package delayqueue package delayqueue
import ( import (
"context" "errors"
"fmt" "fmt"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/redis/go-redis/v9"
"log" "log"
"strconv"
"sync" "sync"
"time" "time"
) )
@@ -14,7 +14,7 @@ import (
type DelayQueue struct { type DelayQueue struct {
// name for this Queue. Make sure the name is unique in redis database // name for this Queue. Make sure the name is unique in redis database
name string name string
redisCli *redis.Client redisCli RedisCli
cb func(string) bool cb func(string) bool
pendingKey string // sorted set: message id -> delivery time pendingKey string // sorted set: message id -> delivery time
readyKey string // list readyKey string // list
@@ -22,6 +22,7 @@ type DelayQueue struct {
retryKey string // list retryKey string // list
retryCountKey string // hash: message id -> remain retry count retryCountKey string // hash: message id -> remain retry count
garbageKey string // set: message id garbageKey string // set: message id
useHashTag bool
ticker *time.Ticker ticker *time.Ticker
logger *log.Logger logger *log.Logger
close chan struct{} close chan struct{}
@@ -31,8 +32,24 @@ type DelayQueue struct {
defaultRetryCount uint defaultRetryCount uint
fetchInterval time.Duration fetchInterval time.Duration
fetchLimit uint fetchLimit uint
concurrent uint
}
concurrent uint // NilErr represents redis nil
var NilErr = errors.New("nil")
// RedisCli is abstraction for redis client, required commands only not all commands
type RedisCli interface {
Eval(script string, keys []string, args []interface{}) (interface{}, error) // args should be string, integer or float
Set(key string, value string, expiration time.Duration) error
Get(key string) (string, error)
Del(keys []string) error
HSet(key string, field string, value string) error
HDel(key string, fields []string) error
SMembers(key string) ([]string, error)
SRem(key string, members []string) error
ZAdd(key string, values map[string]float64) error
ZRem(key string, fields []string) error
} }
type hashTagKeyOpt int type hashTagKeyOpt int
@@ -45,9 +62,9 @@ func UseHashTagKey() interface{} {
return hashTagKeyOpt(1) return hashTagKeyOpt(1)
} }
// NewQueue creates a new queue, use DelayQueue.StartConsume to consume or DelayQueue.SendScheduleMsg to publish message // NewQueue0 creates a new queue, use DelayQueue.StartConsume to consume or DelayQueue.SendScheduleMsg to publish message
// callback returns true to confirm successful consumption. If callback returns false or not return within maxConsumeDuration, DelayQueue will re-deliver this message // callback returns true to confirm successful consumption. If callback returns false or not return within maxConsumeDuration, DelayQueue will re-deliver this message
func NewQueue(name string, cli *redis.Client, callback func(string) bool, opts ...interface{}) *DelayQueue { func NewQueue0(name string, cli RedisCli, callback func(string) bool, opts ...interface{}) *DelayQueue {
if name == "" { if name == "" {
panic("name is required") panic("name is required")
} }
@@ -80,6 +97,7 @@ func NewQueue(name string, cli *redis.Client, callback func(string) bool, opts .
retryKey: keyPrefix + ":retry", retryKey: keyPrefix + ":retry",
retryCountKey: keyPrefix + ":retry:cnt", retryCountKey: keyPrefix + ":retry:cnt",
garbageKey: keyPrefix + ":garbage", garbageKey: keyPrefix + ":garbage",
useHashTag: useHashTag,
close: make(chan struct{}, 1), close: make(chan struct{}, 1),
maxConsumeDuration: 5 * time.Second, maxConsumeDuration: 5 * time.Second,
msgTTL: time.Hour, msgTTL: time.Hour,
@@ -132,6 +150,9 @@ func (q *DelayQueue) WithDefaultRetryCount(count uint) *DelayQueue {
} }
func (q *DelayQueue) genMsgKey(idStr string) string { func (q *DelayQueue) genMsgKey(idStr string) string {
if q.useHashTag {
return "{dp:" + q.name + "}" + ":msg:" + idStr
}
return "dp:" + q.name + ":msg:" + idStr return "dp:" + q.name + ":msg:" + idStr
} }
@@ -165,21 +186,20 @@ func (q *DelayQueue) SendScheduleMsg(payload string, t time.Time, opts ...interf
} }
// generate id // generate id
idStr := uuid.Must(uuid.NewRandom()).String() idStr := uuid.Must(uuid.NewRandom()).String()
ctx := context.Background()
now := time.Now() now := time.Now()
// store msg // store msg
msgTTL := t.Sub(now) + q.msgTTL // delivery + q.msgTTL msgTTL := t.Sub(now) + q.msgTTL // delivery + q.msgTTL
err := q.redisCli.Set(ctx, q.genMsgKey(idStr), payload, msgTTL).Err() err := q.redisCli.Set(q.genMsgKey(idStr), payload, msgTTL)
if err != nil { if err != nil {
return fmt.Errorf("store msg failed: %v", err) return fmt.Errorf("store msg failed: %v", err)
} }
// store retry count // store retry count
err = q.redisCli.HSet(ctx, q.retryCountKey, idStr, retryCount).Err() err = q.redisCli.HSet(q.retryCountKey, idStr, strconv.Itoa(int(retryCount)))
if err != nil { if err != nil {
return fmt.Errorf("store retry count failed: %v", err) return fmt.Errorf("store retry count failed: %v", err)
} }
// put to pending // put to pending
err = q.redisCli.ZAdd(ctx, q.pendingKey, redis.Z{Score: float64(t.Unix()), Member: idStr}).Err() err = q.redisCli.ZAdd(q.pendingKey, map[string]float64{idStr: float64(t.Unix())})
if err != nil { if err != nil {
return fmt.Errorf("push to pending failed: %v", err) return fmt.Errorf("push to pending failed: %v", err)
} }
@@ -214,10 +234,9 @@ redis.call('ZRemRangeByScore', KEYS[1], '0', ARGV[1]) -- remove msgs from pendi
func (q *DelayQueue) pending2Ready() error { func (q *DelayQueue) pending2Ready() error {
now := time.Now().Unix() now := time.Now().Unix()
ctx := context.Background()
keys := []string{q.pendingKey, q.readyKey} keys := []string{q.pendingKey, q.readyKey}
err := q.redisCli.Eval(ctx, pending2ReadyScript, keys, now).Err() _, err := q.redisCli.Eval(pending2ReadyScript, keys, []interface{}{now})
if err != nil && err != redis.Nil { if err != nil && err != NilErr {
return fmt.Errorf("pending2ReadyScript failed: %v", err) return fmt.Errorf("pending2ReadyScript failed: %v", err)
} }
return nil return nil
@@ -235,10 +254,9 @@ return msg
func (q *DelayQueue) ready2Unack() (string, error) { func (q *DelayQueue) ready2Unack() (string, error) {
retryTime := time.Now().Add(q.maxConsumeDuration).Unix() retryTime := time.Now().Add(q.maxConsumeDuration).Unix()
ctx := context.Background()
keys := []string{q.readyKey, q.unAckKey} keys := []string{q.readyKey, q.unAckKey}
ret, err := q.redisCli.Eval(ctx, ready2UnackScript, keys, retryTime).Result() ret, err := q.redisCli.Eval(ready2UnackScript, keys, []interface{}{retryTime})
if err == redis.Nil { if err == NilErr {
return "", err return "", err
} }
if err != nil { if err != nil {
@@ -253,11 +271,10 @@ func (q *DelayQueue) ready2Unack() (string, error) {
func (q *DelayQueue) retry2Unack() (string, error) { func (q *DelayQueue) retry2Unack() (string, error) {
retryTime := time.Now().Add(q.maxConsumeDuration).Unix() retryTime := time.Now().Add(q.maxConsumeDuration).Unix()
ctx := context.Background()
keys := []string{q.retryKey, q.unAckKey} keys := []string{q.retryKey, q.unAckKey}
ret, err := q.redisCli.Eval(ctx, ready2UnackScript, keys, retryTime, q.retryKey, q.unAckKey).Result() ret, err := q.redisCli.Eval(ready2UnackScript, keys, []interface{}{retryTime, q.retryKey, q.unAckKey})
if err == redis.Nil { if err == NilErr {
return "", redis.Nil return "", NilErr
} }
if err != nil { if err != nil {
return "", fmt.Errorf("ready2UnackScript failed: %v", err) return "", fmt.Errorf("ready2UnackScript failed: %v", err)
@@ -270,9 +287,8 @@ func (q *DelayQueue) retry2Unack() (string, error) {
} }
func (q *DelayQueue) callback(idStr string) error { func (q *DelayQueue) callback(idStr string) error {
ctx := context.Background() payload, err := q.redisCli.Get(q.genMsgKey(idStr))
payload, err := q.redisCli.Get(ctx, q.genMsgKey(idStr)).Result() if err == NilErr {
if err == redis.Nil {
return nil return nil
} }
if err != nil { if err != nil {
@@ -326,24 +342,21 @@ func (q *DelayQueue) batchCallback(ids []string) {
} }
func (q *DelayQueue) ack(idStr string) error { func (q *DelayQueue) ack(idStr string) error {
ctx := context.Background() err := q.redisCli.ZRem(q.unAckKey, []string{idStr})
err := q.redisCli.ZRem(ctx, q.unAckKey, idStr).Err()
if err != nil { if err != nil {
return fmt.Errorf("remove from unack failed: %v", err) return fmt.Errorf("remove from unack failed: %v", err)
} }
// msg key has ttl, ignore result of delete // msg key has ttl, ignore result of delete
_ = q.redisCli.Del(ctx, q.genMsgKey(idStr)).Err() _ = q.redisCli.Del([]string{q.genMsgKey(idStr)})
q.redisCli.HDel(ctx, q.retryCountKey, idStr) _ = q.redisCli.HDel(q.retryCountKey, []string{idStr})
return nil return nil
} }
func (q *DelayQueue) nack(idStr string) error { func (q *DelayQueue) nack(idStr string) error {
ctx := context.Background()
// update retry time as now, unack2Retry will move it to retry immediately // update retry time as now, unack2Retry will move it to retry immediately
err := q.redisCli.ZAdd(ctx, q.unAckKey, redis.Z{ err := q.redisCli.ZAdd(q.unAckKey, map[string]float64{
Member: idStr, idStr: float64(time.Now().Unix()),
Score: float64(time.Now().Unix()), })
}).Err()
if err != nil { if err != nil {
return fmt.Errorf("negative ack failed: %v", err) return fmt.Errorf("negative ack failed: %v", err)
} }
@@ -392,19 +405,17 @@ redis.call('ZRemRangeByScore', KEYS[1], '0', ARGV[1]) -- remove msgs from unack
` `
func (q *DelayQueue) unack2Retry() error { func (q *DelayQueue) unack2Retry() error {
ctx := context.Background()
keys := []string{q.unAckKey, q.retryCountKey, q.retryKey, q.garbageKey} keys := []string{q.unAckKey, q.retryCountKey, q.retryKey, q.garbageKey}
now := time.Now() now := time.Now()
err := q.redisCli.Eval(ctx, unack2RetryScript, keys, now.Unix()).Err() _, err := q.redisCli.Eval(unack2RetryScript, keys, []interface{}{now.Unix()})
if err != nil && err != redis.Nil { if err != nil && err != NilErr {
return fmt.Errorf("unack to retry script failed: %v", err) return fmt.Errorf("unack to retry script failed: %v", err)
} }
return nil return nil
} }
func (q *DelayQueue) garbageCollect() error { func (q *DelayQueue) garbageCollect() error {
ctx := context.Background() msgIds, err := q.redisCli.SMembers(q.garbageKey)
msgIds, err := q.redisCli.SMembers(ctx, q.garbageKey).Result()
if err != nil { if err != nil {
return fmt.Errorf("smembers failed: %v", err) return fmt.Errorf("smembers failed: %v", err)
} }
@@ -416,12 +427,12 @@ func (q *DelayQueue) garbageCollect() error {
for _, idStr := range msgIds { for _, idStr := range msgIds {
msgKeys = append(msgKeys, q.genMsgKey(idStr)) msgKeys = append(msgKeys, q.genMsgKey(idStr))
} }
err = q.redisCli.Del(ctx, msgKeys...).Err() err = q.redisCli.Del(msgKeys)
if err != nil && err != redis.Nil { if err != nil && err != NilErr {
return fmt.Errorf("del msgs failed: %v", err) return fmt.Errorf("del msgs failed: %v", err)
} }
err = q.redisCli.SRem(ctx, q.garbageKey, msgIds).Err() err = q.redisCli.SRem(q.garbageKey, msgIds)
if err != nil && err != redis.Nil { if err != nil && err != NilErr {
return fmt.Errorf("remove from garbage key failed: %v", err) return fmt.Errorf("remove from garbage key failed: %v", err)
} }
return nil return nil
@@ -437,7 +448,7 @@ func (q *DelayQueue) consume() error {
ids := make([]string, 0, q.fetchLimit) ids := make([]string, 0, q.fetchLimit)
for { for {
idStr, err := q.ready2Unack() idStr, err := q.ready2Unack()
if err == redis.Nil { // consumed all if err == NilErr { // consumed all
break break
} }
if err != nil { if err != nil {
@@ -464,7 +475,7 @@ func (q *DelayQueue) consume() error {
ids = make([]string, 0, q.fetchLimit) ids = make([]string, 0, q.fetchLimit)
for { for {
idStr, err := q.retry2Unack() idStr, err := q.retry2Unack()
if err == redis.Nil { // consumed all if err == NilErr { // consumed all
break break
} }
if err != nil { if err != nil {

40
delayqueue_test.go
View File

@@ -57,6 +57,46 @@ func TestDelayQueue_consume(t *testing.T) {
} }
} }
func TestDelayQueueOnCluster(t *testing.T) {
redisCli := redis.NewClusterClient(&redis.ClusterOptions{
Addrs: []string{
"127.0.0.1:7000",
"127.0.0.1:7001",
"127.0.0.1:7002",
},
})
redisCli.FlushDB(context.Background())
size := 1000
succeed := 0
cb := func(s string) bool {
succeed++
return true
}
queue := NewQueueOnCluster("test", redisCli, cb).
WithFetchInterval(time.Millisecond * 50).
WithMaxConsumeDuration(0).
WithLogger(log.New(os.Stderr, "[DelayQueue]", log.LstdFlags)).
WithFetchLimit(2).
WithConcurrent(1)
for i := 0; i < size; i++ {
err := queue.SendDelayMsg(strconv.Itoa(i), 0)
if err != nil {
t.Error(err)
}
}
for i := 0; i < 10*size; i++ {
err := queue.consume()
if err != nil {
t.Errorf("consume error: %v", err)
return
}
}
if succeed != size {
t.Error("msg not consumed")
}
}
func TestDelayQueue_ConcurrentConsume(t *testing.T) { func TestDelayQueue_ConcurrentConsume(t *testing.T) {
redisCli := redis.NewClient(&redis.Options{ redisCli := redis.NewClient(&redis.Options{
Addr: "127.0.0.1:6379", Addr: "127.0.0.1:6379",

2
go.mod
View File

@@ -4,5 +4,5 @@ go 1.16
require ( require (
github.com/google/uuid v1.3.0 github.com/google/uuid v1.3.0
github.com/redis/go-redis/v9 v9.0.5 // indirect github.com/redis/go-redis/v9 v9.0.5
) )

97
go.sum
View File

@@ -1,105 +1,12 @@
github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w= github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w=
github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y=
github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg=
github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c=
github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o=
github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

173
wrapper.go Normal file
View File

@@ -0,0 +1,173 @@
package delayqueue
import (
"context"
"github.com/redis/go-redis/v9"
"time"
)
func NewQueue(name string, cli *redis.Client, callback func(string) bool, opts ...interface{}) *DelayQueue {
rc := &redisV9Wrapper{
inner: cli,
}
return NewQueue0(name, rc, callback, opts...)
}
func wrapErr(err error) error {
if err == redis.Nil {
return NilErr
}
return err
}
type redisV9Wrapper struct {
inner *redis.Client
}
func (r *redisV9Wrapper) Eval(script string, keys []string, args []interface{}) (interface{}, error) {
ctx := context.Background()
ret, err := r.inner.Eval(ctx, script, keys, args...).Result()
return ret, wrapErr(err)
}
func (r *redisV9Wrapper) Set(key string, value string, expiration time.Duration) error {
ctx := context.Background()
return wrapErr(r.inner.Set(ctx, key, value, expiration).Err())
}
func (r *redisV9Wrapper) Get(key string) (string, error) {
ctx := context.Background()
ret, err := r.inner.Get(ctx, key).Result()
return ret, wrapErr(err)
}
func (r *redisV9Wrapper) Del(keys []string) error {
ctx := context.Background()
return wrapErr(r.inner.Del(ctx, keys...).Err())
}
func (r *redisV9Wrapper) HSet(key string, field string, value string) error {
ctx := context.Background()
return wrapErr(r.inner.HSet(ctx, key, field, value).Err())
}
func (r *redisV9Wrapper) HDel(key string, fields []string) error {
ctx := context.Background()
return wrapErr(r.inner.HDel(ctx, key, fields...).Err())
}
func (r *redisV9Wrapper) SMembers(key string) ([]string, error) {
ctx := context.Background()
ret, err := r.inner.SMembers(ctx, key).Result()
return ret, wrapErr(err)
}
func (r *redisV9Wrapper) SRem(key string, members []string) error {
ctx := context.Background()
members2 := make([]interface{}, len(members))
for i, v := range members {
members2[i] = v
}
return wrapErr(r.inner.SRem(ctx, key, members2...).Err())
}
func (r *redisV9Wrapper) ZAdd(key string, values map[string]float64) error {
ctx := context.Background()
var zs []redis.Z
for member, score := range values {
zs = append(zs, redis.Z{
Score: score,
Member: member,
})
}
return wrapErr(r.inner.ZAdd(ctx, key, zs...).Err())
}
func (r *redisV9Wrapper) ZRem(key string, members []string) error {
ctx := context.Background()
members2 := make([]interface{}, len(members))
for i, v := range members {
members2[i] = v
}
return wrapErr(r.inner.ZRem(ctx, key, members2...).Err())
}
type redisClusterWrapper struct {
inner *redis.ClusterClient
}
func (r *redisClusterWrapper) Eval(script string, keys []string, args []interface{}) (interface{}, error) {
ctx := context.Background()
ret, err := r.inner.Eval(ctx, script, keys, args...).Result()
return ret, wrapErr(err)
}
func (r *redisClusterWrapper) Set(key string, value string, expiration time.Duration) error {
ctx := context.Background()
return wrapErr(r.inner.Set(ctx, key, value, expiration).Err())
}
func (r *redisClusterWrapper) Get(key string) (string, error) {
ctx := context.Background()
ret, err := r.inner.Get(ctx, key).Result()
return ret, wrapErr(err)
}
func (r *redisClusterWrapper) Del(keys []string) error {
ctx := context.Background()
return wrapErr(r.inner.Del(ctx, keys...).Err())
}
func (r *redisClusterWrapper) HSet(key string, field string, value string) error {
ctx := context.Background()
return wrapErr(r.inner.HSet(ctx, key, field, value).Err())
}
func (r *redisClusterWrapper) HDel(key string, fields []string) error {
ctx := context.Background()
return wrapErr(r.inner.HDel(ctx, key, fields...).Err())
}
func (r *redisClusterWrapper) SMembers(key string) ([]string, error) {
ctx := context.Background()
ret, err := r.inner.SMembers(ctx, key).Result()
return ret, wrapErr(err)
}
func (r *redisClusterWrapper) SRem(key string, members []string) error {
ctx := context.Background()
members2 := make([]interface{}, len(members))
for i, v := range members {
members2[i] = v
}
return wrapErr(r.inner.SRem(ctx, key, members2...).Err())
}
func (r *redisClusterWrapper) ZAdd(key string, values map[string]float64) error {
ctx := context.Background()
var zs []redis.Z
for member, score := range values {
zs = append(zs, redis.Z{
Score: score,
Member: member,
})
}
return wrapErr(r.inner.ZAdd(ctx, key, zs...).Err())
}
func (r *redisClusterWrapper) ZRem(key string, members []string) error {
ctx := context.Background()
members2 := make([]interface{}, len(members))
for i, v := range members {
members2[i] = v
}
return wrapErr(r.inner.ZRem(ctx, key, members2...).Err())
}
func NewQueueOnCluster(name string, cli *redis.ClusterClient, callback func(string) bool, opts ...interface{}) *DelayQueue {
rc := &redisClusterWrapper{
inner: cli,
}
opts = append(opts, UseHashTagKey())
return NewQueue0(name, rc, callback, opts...)
}