zishuo/golib
1
0
Fork 0
mirror of https://github.com/nabbar/golib.git synced 2025-09-26 20:01:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Package Cluster / nutsdb:

Browse Source 
- removing package cluster & nutsdb (+ test) in waiting stabilisation of
  dependencies
- adding olld source code in tarball file

Dependencies golang/x/exp:
- removing experiment dependency when using slices to root/slices
  package

Bump dependencies (some dependences still use exp who's need go1.22)
This commit is contained in:
nabbar
2024-10-06 18:28:59 +02:00
parent 3c6318b33a
commit 3235cc0909
39 changed files with 56 additions and 7498 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
cluster.old.tar.gz Normal file
View File

Binary file not shown.

85
cluster/async.go
View File

@@ -1,85 +0,0 @@
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || riscv64 || s390x || sparc64 || wasm
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le riscv64 s390x sparc64 wasm
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package cluster
import (
dgbclt "github.com/lni/dragonboat/v3"
dgbcli "github.com/lni/dragonboat/v3/client"
liberr "github.com/nabbar/golib/errors"
)
func (c *cRaft) AsyncPropose(session *dgbcli.Session, cmd []byte) (*dgbclt.RequestState, liberr.Error) {
r, e := c.nodeHost.Propose(session, cmd, c.timeoutCmdASync)
if e != nil {
return r, ErrorCommandASync.Error(c.getErrorCommand("Propose"), e)
}
return r, nil
}
func (c *cRaft) AsyncProposeSession(session *dgbcli.Session) (*dgbclt.RequestState, liberr.Error) {
r, e := c.nodeHost.ProposeSession(session, c.timeoutCmdASync)
if e != nil {
return r, ErrorCommandASync.Error(c.getErrorCommand("ProposeSession"), e)
}
return r, nil
}
func (c *cRaft) AsyncReadIndex() (*dgbclt.RequestState, liberr.Error) {
r, e := c.nodeHost.ReadIndex(c.config.ClusterID, c.timeoutCmdASync)
if e != nil {
return r, ErrorCommandASync.Error(c.getErrorCluster(), c.getErrorCommand("ReadIndex"), e)
}
return r, nil
}
func (c *cRaft) AsyncRequestCompaction(nodeID uint64) (*dgbclt.SysOpState, liberr.Error) {
var er error
if nodeID == 0 {
nodeID = c.config.NodeID
er = c.getErrorNode()
} else {
er = c.getErrorNodeTarget(nodeID)
}
r, e := c.nodeHost.RequestCompaction(c.config.ClusterID, nodeID)
if e != nil {
return r, ErrorCommandASync.Error(c.getErrorCluster(), er, c.getErrorCommand("RequestCompaction"), e)
}
return r, nil
}

298
cluster/cluster.go
View File

@@ -1,298 +0,0 @@
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || riscv64 || s390x || sparc64 || wasm
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le riscv64 s390x sparc64 wasm
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package cluster
import (
"fmt"
"io"
"time"
dgbclt "github.com/lni/dragonboat/v3"
dgbcli "github.com/lni/dragonboat/v3/client"
dgbcfg "github.com/lni/dragonboat/v3/config"
dgbstm "github.com/lni/dragonboat/v3/statemachine"
liberr "github.com/nabbar/golib/errors"
)
type cRaft struct {
memberInit map[uint64]dgbclt.Target
fctCreate interface{}
config dgbcfg.Config
nodeHost *dgbclt.NodeHost
timeoutCmdSync time.Duration
timeoutCmdASync time.Duration
}
func (c *cRaft) getErrorCluster() error {
//nolint #goerr113
return fmt.Errorf("cluster: %v", c.config.ClusterID)
}
func (c *cRaft) getErrorNode() error {
//nolint #goerr113
return fmt.Errorf("node: %v", c.config.NodeID)
}
func (c *cRaft) getErrorNodeTarget(target uint64) error {
//nolint #goerr113
return fmt.Errorf("target node: %v", target)
}
func (c *cRaft) getErrorCommand(cmd string) error {
//nolint #goerr113
return fmt.Errorf("command: %v", cmd)
}
func (c *cRaft) GetConfig() dgbcfg.Config {
return c.config
}
func (c *cRaft) SetConfig(cfg dgbcfg.Config) {
c.config = cfg
}
func (c *cRaft) GetFctCreate() dgbstm.CreateStateMachineFunc {
if f, ok := c.fctCreate.(dgbstm.CreateStateMachineFunc); ok {
return f
}
return nil
}
func (c *cRaft) GetFctCreateConcurrent() dgbstm.CreateConcurrentStateMachineFunc {
if f, ok := c.fctCreate.(dgbstm.CreateConcurrentStateMachineFunc); ok {
return f
}
return nil
}
func (c *cRaft) GetFctCreateOnDisk() dgbstm.CreateOnDiskStateMachineFunc {
if f, ok := c.fctCreate.(dgbstm.CreateOnDiskStateMachineFunc); ok {
return f
}
return nil
}
func (c *cRaft) SetFctCreate(fctCreate interface{}) {
c.fctCreate = fctCreate
}
func (c *cRaft) SetFctCreateSTM(fctCreate dgbstm.CreateStateMachineFunc) {
c.fctCreate = fctCreate
}
func (c *cRaft) SetFctCreateSTMConcurrent(fctCreate dgbstm.CreateConcurrentStateMachineFunc) {
c.fctCreate = fctCreate
}
func (c *cRaft) SetFctCreateSTMOnDisk(fctCreate dgbstm.CreateOnDiskStateMachineFunc) {
c.fctCreate = fctCreate
}
func (c *cRaft) GetMemberInit() map[uint64]dgbclt.Target {
return c.memberInit
}
func (c *cRaft) SetMemberInit(memberList map[uint64]dgbclt.Target) {
c.memberInit = memberList
}
func (c *cRaft) SetTimeoutCommandSync(timeout time.Duration) {
c.timeoutCmdSync = timeout
}
func (c *cRaft) SetTimeoutCommandASync(timeout time.Duration) {
c.timeoutCmdASync = timeout
}
func (c *cRaft) GetNodeHostConfig() dgbcfg.NodeHostConfig {
return c.nodeHost.NodeHostConfig()
}
func (c *cRaft) RaftAddress() string {
return c.nodeHost.RaftAddress()
}
func (c *cRaft) ID() string {
return c.nodeHost.ID()
}
func (c *cRaft) ClusterStart(join bool) liberr.Error {
err := ErrorNodeHostStart.Error(nil)
if join {
err = ErrorNodeHostJoin.Error(nil)
}
if f, ok := c.fctCreate.(dgbstm.CreateStateMachineFunc); ok {
err.Add(c.nodeHost.StartCluster(c.memberInit, join, f, c.config))
} else if f, ok := c.fctCreate.(dgbstm.CreateConcurrentStateMachineFunc); ok {
err.Add(c.nodeHost.StartConcurrentCluster(c.memberInit, join, f, c.config))
} else if f, ok := c.fctCreate.(dgbstm.CreateOnDiskStateMachineFunc); ok {
err.Add(c.nodeHost.StartOnDiskCluster(c.memberInit, join, f, c.config))
} else {
//nolint #goerr113
return ErrorParamsMismatching.Error(fmt.Errorf("create function is not one of type of CreateStateMachineFunc, CreateConcurrentStateMachineFunc, CreateOnDiskStateMachineFunc"))
}
if err.HasParent() {
return err
}
return nil
}
func (c *cRaft) ClusterStop(force bool) liberr.Error {
e := c.nodeHost.StopCluster(c.config.ClusterID)
if e != nil && !force {
return ErrorNodeHostStop.Error(c.getErrorCluster(), e)
}
c.nodeHost.Stop()
return nil
}
func (c *cRaft) ClusterRestart(force bool) liberr.Error {
if err := c.ClusterStop(force); err != nil {
return ErrorNodeHostRestart.Error(err)
}
return c.ClusterStart(false)
}
func (c *cRaft) NodeStop(target uint64) liberr.Error {
var en error
if target == 0 {
target = c.config.NodeID
en = c.getErrorNode()
} else {
en = c.getErrorNodeTarget(target)
}
e := c.nodeHost.StopNode(c.config.ClusterID, target)
if e != nil {
return ErrorNodeHostStop.Error(c.getErrorCluster(), en, e)
}
return nil
}
func (c *cRaft) NodeRestart(force bool) liberr.Error {
var join = false
if l, ok, err := c.GetLeaderID(); err == nil && ok && l == c.config.NodeID {
join = true
var sErr = ErrorNodeHostRestart.Error(c.getErrorCluster(), c.getErrorNode())
for id, nd := range c.memberInit {
if id == c.config.NodeID {
continue
}
if nd == c.RaftAddress() {
continue
}
if err = c.RequestLeaderTransfer(id); err == nil {
sErr.Add(err)
break
}
}
if l, ok, err = c.GetLeaderID(); err == nil && ok && l == c.config.NodeID && !force {
return sErr
}
} else if err == nil && ok {
join = true
}
if err := c.NodeStop(0); err != nil && !force {
return ErrorNodeHostRestart.Error(err)
} else if err != nil && force {
join = false
_ = c.ClusterStop(true)
}
return c.ClusterStart(join)
}
func (c *cRaft) GetLeaderID() (leader uint64, valid bool, err liberr.Error) {
var e error
leader, valid, e = c.nodeHost.GetLeaderID(c.config.ClusterID)
if e != nil {
err = ErrorLeader.Error(c.getErrorCluster(), e)
}
return
}
func (c *cRaft) GetNoOPSession() *dgbcli.Session {
return c.nodeHost.GetNoOPSession(c.config.ClusterID)
}
func (c *cRaft) GetNodeUser() (dgbclt.INodeUser, liberr.Error) {
r, e := c.nodeHost.GetNodeUser(c.config.ClusterID)
if e != nil {
return nil, ErrorNodeUser.Error(c.getErrorCluster())
}
return r, nil
}
func (c *cRaft) HasNodeInfo(nodeId uint64) bool {
if nodeId == 0 {
nodeId = c.config.NodeID
}
return c.nodeHost.HasNodeInfo(c.config.ClusterID, nodeId)
}
func (c *cRaft) GetNodeHostInfo(opt dgbclt.NodeHostInfoOption) *dgbclt.NodeHostInfo {
return c.nodeHost.GetNodeHostInfo(opt)
}
func (c *cRaft) StaleReadDangerous(query interface{}) (interface{}, error) {
return c.nodeHost.StaleRead(c.config.ClusterID, query)
}
func (c *cRaft) RequestLeaderTransfer(targetNodeID uint64) liberr.Error {
e := c.nodeHost.RequestLeaderTransfer(c.config.ClusterID, targetNodeID)
if e != nil {
return ErrorLeaderTransfer.Error(c.getErrorCluster(), c.getErrorNodeTarget(targetNodeID), e)
}
return nil
}
func (c *cRaft) HandlerMetrics(w io.Writer) {
dgbclt.WriteHealthMetrics(w)
}

85
cluster/config.go
View File

@@ -1,85 +0,0 @@
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || riscv64 || s390x || sparc64 || wasm
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le riscv64 s390x sparc64 wasm
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package cluster
import (
"fmt"
libval "github.com/go-playground/validator/v10"
dgbclt "github.com/lni/dragonboat/v3"
dgbcfg "github.com/lni/dragonboat/v3/config"
liberr "github.com/nabbar/golib/errors"
)
type Config struct {
Node ConfigNode `mapstructure:"node" json:"node" yaml:"node" toml:"node" validate:""`
Cluster ConfigCluster `mapstructure:"cluster" json:"cluster" yaml:"cluster" toml:"cluster" validate:""`
InitMember map[uint64]string `mapstructure:"init_member" json:"init_member" yaml:"init_member" toml:"init_member"`
}
func (c Config) GetDGBConfigCluster() dgbcfg.Config {
return c.Cluster.GetDGBConfigCluster()
}
func (c Config) GetDGBConfigNode() dgbcfg.NodeHostConfig {
return c.Node.GetDGBConfigNodeHost()
}
func (c Config) GetInitMember() map[uint64]dgbclt.Target {
var m = make(map[uint64]dgbclt.Target)
for k, v := range c.InitMember {
m[k] = v
}
return m
}
func (c Config) Validate() liberr.Error {
err := ErrorValidateConfig.Error(nil)
if er := libval.New().Struct(c); er != nil {
if e, ok := er.(*libval.InvalidValidationError); ok {
err.Add(e)
}
for _, e := range er.(libval.ValidationErrors) {
//nolint goerr113
err.Add(fmt.Errorf("config field '%s' is not validated by constraint '%s'", e.Namespace(), e.ActualTag()))
}
}
if err.HasParent() {
return err
}
return nil
}

265
cluster/configCluster.go
View File

@@ -1,265 +0,0 @@
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || riscv64 || s390x || sparc64 || wasm
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le riscv64 s390x sparc64 wasm
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package cluster
import (
"fmt"
libval "github.com/go-playground/validator/v10"
dgbcfg "github.com/lni/dragonboat/v3/config"
liberr "github.com/nabbar/golib/errors"
)
// nolint #maligned
type ConfigCluster struct {
// NodeID is a non-zero value used to identify a node within a Raft cluster.
NodeID uint64 `mapstructure:"node_id" json:"node_id" yaml:"node_id" toml:"node_id"`
// ClusterID is the unique value used to identify a Raft cluster.
ClusterID uint64 `mapstructure:"cluster_id" json:"cluster_id" yaml:"cluster_id" toml:"cluster_id"`
// CheckQuorum specifies whether the leader node should periodically check
// non-leader node status and step down to become a follower node when it no
// longer has the quorum.
CheckQuorum bool `mapstructure:"check_quorum" json:"check_quorum" yaml:"check_quorum" toml:"check_quorum"`
// ElectionRTT is the minimum number of message RTT between elections. Message
// RTT is defined by NodeHostConfig.RTTMillisecond. The Raft paper suggests it
// to be a magnitude greater than HeartbeatRTT, which is the interval between
// two heartbeats. In Raft, the actual interval between elections is
// randomized to be between ElectionRTT and 2 * ElectionRTT.
//
// As an example, assuming NodeHostConfig.RTTMillisecond is 100 millisecond,
// to set the election interval to be 1 second, then ElectionRTT should be set
// to 10.
//
// When CheckQuorum is enabled, ElectionRTT also defines the interval for
// checking leader quorum.
ElectionRTT uint64 `mapstructure:"election_rtt" json:"election_rtt" yaml:"election_rtt" toml:"election_rtt"`
// HeartbeatRTT is the number of message RTT between heartbeats. Message
// RTT is defined by NodeHostConfig.RTTMillisecond. The Raft paper suggest the
// heartbeat interval to be close to the average RTT between nodes.
//
// As an example, assuming NodeHostConfig.RTTMillisecond is 100 millisecond,
// to set the heartbeat interval to be every 200 milliseconds, then
// HeartbeatRTT should be set to 2.
HeartbeatRTT uint64 `mapstructure:"heartbeat_rtt" json:"heartbeat_rtt" yaml:"heartbeat_rtt" toml:"heartbeat_rtt"`
// SnapshotEntries defines how often the state machine should be snapshotted
// automcatically. It is defined in terms of the number of applied Raft log
// entries. SnapshotEntries can be set to 0 to disable such automatic
// snapshotting.
//
// When SnapshotEntries is set to N, it means a snapshot is created for
// roughly every N applied Raft log entries (proposals). This also implies
// that sending N log entries to a follower is more expensive than sending a
// snapshot.
//
// Once a snapshot is generated, Raft log entries covered by the new snapshot
// can be compacted. This involves two steps, redundant log entries are first
// marked as deleted, then they are physically removed from the underlying
// storage when a LogDB compaction is issued at a later stage. See the godoc
// on CompactionOverhead for details on what log entries are actually removed
// and compacted after generating a snapshot.
//
// Once automatic snapshotting is disabled by setting the SnapshotEntries
// field to 0, users can still use NodeHost's RequestSnapshot or
// SyncRequestSnapshot methods to manually request snapshots.
SnapshotEntries uint64 `mapstructure:"snapshot_entries" json:"snapshot_entries" yaml:"snapshot_entries" toml:"snapshot_entries"`
// CompactionOverhead defines the number of most recent entries to keep after
// each Raft log compaction. Raft log compaction is performance automatically
// every time when a snapshot is created.
//
// For example, when a snapshot is created at let's say index 10,000, then all
// Raft log entries with index <= 10,000 can be removed from that node as they
// have already been covered by the created snapshot image. This frees up the
// maximum storage space but comes at the cost that the full snapshot will
// have to be sent to the follower if the follower requires any Raft log entry
// at index <= 10,000. When CompactionOverhead is set to say 500, Dragonboat
// then compacts the Raft log up to index 9,500 and keeps Raft log entries
// between index (9,500, 1,0000]. As a result, the node can still replicate
// Raft log entries between index (9,500, 1,0000] to other peers and only fall
// back to stream the full snapshot if any Raft log entry with index <= 9,500
// is required to be replicated.
CompactionOverhead uint64 `mapstructure:"compaction_overhead" json:"compaction_overhead" yaml:"compaction_overhead" toml:"compaction_overhead"`
// OrderedConfigChange determines whether Raft membership change is enforced
// with ordered config change ID.
OrderedConfigChange bool `mapstructure:"ordered_config_change" json:"ordered_config_change" yaml:"ordered_config_change" toml:"ordered_config_change"`
// MaxInMemLogSize is the target size in bytes allowed for storing in memory
// Raft logs on each Raft node. In memory Raft logs are the ones that have
// not been applied yet.
// MaxInMemLogSize is a target value implemented to prevent unbounded memory
// growth, it is not for precisely limiting the exact memory usage.
// When MaxInMemLogSize is 0, the target is set to math.MaxUint64. When
// MaxInMemLogSize is set and the target is reached, error will be returned
// when clients try to make new proposals.
// MaxInMemLogSize is recommended to be significantly larger than the biggest
// proposal you are going to use.
MaxInMemLogSize uint64 `mapstructure:"max_in_mem_log_size" json:"max_in_mem_log_size" yaml:"max_in_mem_log_size" toml:"max_in_mem_log_size"`
// SnapshotCompressionType is the compression type to use for compressing
// generated snapshot data. No compression is used by default.
SnapshotCompressionType dgbcfg.CompressionType `mapstructure:"snapshot_compression" json:"snapshot_compression" yaml:"snapshot_compression" toml:"snapshot_compression"`
// EntryCompressionType is the compression type to use for compressing the
// payload of user proposals. When Snappy is used, the maximum proposal
// payload allowed is roughly limited to 3.42GBytes. No compression is used
// by default.
EntryCompressionType dgbcfg.CompressionType `mapstructure:"entry_compression" json:"entry_compression" yaml:"entry_compression" toml:"entry_compression"`
// DisableAutoCompactions disables auto compaction used for reclaiming Raft
// log entry storage spaces. By default, compaction request is issued every
// time when a snapshot is created, this helps to reclaim disk spaces as
// soon as possible at the cost of immediate higher IO overhead. Users can
// disable such auto compactions and use NodeHost.RequestCompaction to
// manually request such compactions when necessary.
DisableAutoCompactions bool `mapstructure:"disable_auto_compactions" json:"disable_auto_compactions" yaml:"disable_auto_compactions" toml:"disable_auto_compactions"`
// IsObserver indicates whether this is an observer Raft node without voting
// power. Described as non-voting members in the section 4.2.1 of Diego
// Ongaro's thesis, observer nodes are usually used to allow a new node to
// join the cluster and catch up with other existing ndoes without impacting
// the availability. Extra observer nodes can also be introduced to serve
// read-only requests without affecting system write throughput.
//
// Observer support is currently experimental.
IsObserver bool `mapstructure:"is_observer" json:"is_observer" yaml:"is_observer" toml:"is_observer"`
// IsWitness indicates whether this is a witness Raft node without actual log
// replication and do not have state machine. It is mentioned in the section
// 11.7.2 of Diego Ongaro's thesis.
//
// Witness support is currently experimental.
IsWitness bool `mapstructure:"is_witness" json:"is_witness" yaml:"is_witness" toml:"is_witness"`
// Quiesce specifies whether to let the Raft cluster enter quiesce mode when
// there is no cluster activity. Clusters in quiesce mode do not exchange
// heartbeat messages to minimize bandwidth consumption.
//
// Quiesce support is currently experimental.
Quiesce bool `mapstructure:"quiesce" json:"quiesce" yaml:"quiesce" toml:"quiesce"`
}
func (c ConfigCluster) GetDGBConfigCluster() dgbcfg.Config {
d := dgbcfg.Config{
NodeID: c.NodeID,
ClusterID: c.ClusterID,
SnapshotCompressionType: 0,
EntryCompressionType: 0,
}
if c.CheckQuorum {
d.CheckQuorum = true
}
if c.ElectionRTT > 0 {
d.ElectionRTT = c.ElectionRTT
}
if c.HeartbeatRTT > 0 {
d.HeartbeatRTT = c.HeartbeatRTT
}
if c.SnapshotEntries > 0 {
d.SnapshotEntries = c.SnapshotEntries
}
if c.CompactionOverhead > 0 {
d.CompactionOverhead = c.CompactionOverhead
}
if c.OrderedConfigChange {
d.OrderedConfigChange = true
}
if c.MaxInMemLogSize > 0 {
d.MaxInMemLogSize = c.MaxInMemLogSize
}
if c.DisableAutoCompactions {
d.DisableAutoCompactions = true
}
if c.IsObserver {
d.IsObserver = true
}
if c.IsWitness {
d.IsWitness = true
}
if c.Quiesce {
d.Quiesce = true
}
//nolint #exhaustive
switch c.SnapshotCompressionType {
case dgbcfg.Snappy:
d.SnapshotCompressionType = dgbcfg.Snappy
default:
d.SnapshotCompressionType = dgbcfg.NoCompression
}
//nolint #exhaustive
switch c.EntryCompressionType {
case dgbcfg.Snappy:
d.EntryCompressionType = dgbcfg.Snappy
default:
d.EntryCompressionType = dgbcfg.NoCompression
}
return d
}
func (c ConfigCluster) Validate() liberr.Error {
err := ErrorValidateConfig.Error(nil)
if er := libval.New().Struct(c); er != nil {
if e, ok := er.(*libval.InvalidValidationError); ok {
err.Add(e)
}
for _, e := range er.(libval.ValidationErrors) {
//nolint goerr113
err.Add(fmt.Errorf("config field '%s' is not validated by constraint '%s'", e.Namespace(), e.ActualTag()))
}
}
if err.HasParent() {
return err
}
return nil
}

109
cluster/configEngine.go
View File

@@ -1,109 +0,0 @@
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || riscv64 || s390x || sparc64 || wasm
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le riscv64 s390x sparc64 wasm
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package cluster
import (
"fmt"
libval "github.com/go-playground/validator/v10"
dgbcfg "github.com/lni/dragonboat/v3/config"
liberr "github.com/nabbar/golib/errors"
)
type ConfigEngine struct {
// ExecShards is the number of execution shards in the first stage of the
// execution engine. Default value is 16. Once deployed, this value can not
// be changed later.
ExecShards uint64 `mapstructure:"exec_shards" json:"exec_shards" yaml:"exec_shards" toml:"exec_shards"`
// CommitShards is the number of commit shards in the second stage of the
// execution engine. Default value is 16.
CommitShards uint64 `mapstructure:"commit_shards" json:"commit_shards" yaml:"commit_shards" toml:"commit_shards"`
// ApplyShards is the number of apply shards in the third stage of the
// execution engine. Default value is 16.
ApplyShards uint64 `mapstructure:"apply_shards" json:"apply_shards" yaml:"apply_shards" toml:"apply_shards"`
// SnapshotShards is the number of snapshot shards in the forth stage of the
// execution engine. Default value is 48.
SnapshotShards uint64 `mapstructure:"snapshot_shards" json:"snapshot_shards" yaml:"snapshot_shards" toml:"snapshot_shards"`
// CloseShards is the number of close shards used for closing stopped
// state machines. Default value is 32.
CloseShards uint64 `mapstructure:"close_shards" json:"close_shards" yaml:"close_shards" toml:"close_shards"`
}
func (c ConfigEngine) GetDGBConfigEngine() dgbcfg.EngineConfig {
d := dgbcfg.EngineConfig{}
if c.ExecShards > 0 {
d.ExecShards = c.ExecShards
}
if c.CommitShards > 0 {
d.CommitShards = c.CommitShards
}
if c.ApplyShards > 0 {
d.ApplyShards = c.ApplyShards
}
if c.SnapshotShards > 0 {
d.SnapshotShards = c.SnapshotShards
}
if c.CloseShards > 0 {
d.CloseShards = c.CloseShards
}
return d
}
func (c ConfigEngine) Validate() liberr.Error {
err := ErrorValidateConfig.Error(nil)
if er := libval.New().Struct(c); er != nil {
if e, ok := er.(*libval.InvalidValidationError); ok {
err.Add(e)
}
for _, e := range er.(libval.ValidationErrors) {
//nolint goerr113
err.Add(fmt.Errorf("config field '%s' is not validated by constraint '%s'", e.Namespace(), e.ActualTag()))
}
}
if err.HasParent() {
return err
}
return nil
}

90
cluster/configExpert.go
View File

@@ -1,90 +0,0 @@
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || riscv64 || s390x || sparc64 || wasm
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le riscv64 s390x sparc64 wasm
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package cluster
import (
"fmt"
"time"
libval "github.com/go-playground/validator/v10"
dgbcfg "github.com/lni/dragonboat/v3/config"
liberr "github.com/nabbar/golib/errors"
)
type ConfigExpert struct {
// Engine is the cponfiguration for the execution engine.
Engine ConfigEngine `mapstructure:"engine" json:"engine" yaml:"engine" toml:"engine"`
// TestNodeHostID is the NodeHostID value to be used by the NodeHost instance.
// This field is expected to be used in tests only.
TestNodeHostID uint64 `mapstructure:"test_node_host_id" json:"test_node_host_id" yaml:"test_node_host_id" toml:"test_node_host_id"`
// TestGossipProbeInterval define the probe interval used by the gossip
// service in tests.
TestGossipProbeInterval time.Duration `mapstructure:"test_gossip_probe_interval" json:"test_gossip_probe_interval" yaml:"test_gossip_probe_interval" toml:"test_gossip_probe_interval"`
}
func (c ConfigExpert) GetDGBConfigExpert() dgbcfg.ExpertConfig {
d := dgbcfg.ExpertConfig{
Engine: c.Engine.GetDGBConfigEngine(),
}
if c.TestNodeHostID > 0 {
d.TestNodeHostID = c.TestNodeHostID
}
if c.TestGossipProbeInterval > 0 {
d.TestGossipProbeInterval = c.TestGossipProbeInterval
}
return d
}
func (c ConfigExpert) Validate() liberr.Error {
err := ErrorValidateConfig.Error(nil)
if er := libval.New().Struct(c); er != nil {
if e, ok := er.(*libval.InvalidValidationError); ok {
err.Add(e)
}
for _, e := range er.(libval.ValidationErrors) {
//nolint goerr113
err.Add(fmt.Errorf("config field '%s' is not validated by constraint '%s'", e.Namespace(), e.ActualTag()))
}
}
if err.HasParent() {
return err
}
return nil
}

114
cluster/configGossip.go
View File

@@ -1,114 +0,0 @@
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || riscv64 || s390x || sparc64 || wasm
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le riscv64 s390x sparc64 wasm
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package cluster
import (
"fmt"
libval "github.com/go-playground/validator/v10"
dgbcfg "github.com/lni/dragonboat/v3/config"
liberr "github.com/nabbar/golib/errors"
)
type ConfigGossip struct {
// BindAddress is the address for the gossip service to bind to and listen on.
// Both UDP and TCP ports are used by the gossip service. The local gossip
// service should be able to receive gossip service related messages by
// binding to and listening on this address. BindAddress is usually in the
// format of IP:Port, Hostname:Port or DNS Name:Port.
BindAddress string `mapstructure:"bind_address" json:"bind_address" yaml:"bind_address" toml:"bind_address" validate:"omitempty,hostname_port"`
// AdvertiseAddress is the address to advertise to other NodeHost instances
// used for NAT traversal. Gossip services running on remote NodeHost
// instances will use AdvertiseAddress to exchange gossip service related
// messages. AdvertiseAddress is in the format of IP:Port.
AdvertiseAddress string `mapstructure:"advertise_address" json:"advertise_address" yaml:"advertise_address" toml:"advertise_address" validate:"omitempty,printascii"`
// Seed is a list of AdvertiseAddress of remote NodeHost instances. Local
// NodeHost instance will try to contact all of them to bootstrap the gossip
// service. At least one reachable NodeHost instance is required to
// successfully bootstrap the gossip service. Each seed address is in the
// format of IP:Port, Hostname:Port or DNS Name:Port.
//
// It is ok to include seed addresses that are temporarily unreachable, e.g.
// when launching the first NodeHost instance in your deployment, you can
// include AdvertiseAddresses from other NodeHost instances that you plan to
// launch shortly afterwards.
Seed []string `mapstructure:"seed" json:"seed" yaml:"seed" toml:"seed"`
}
func (c ConfigGossip) GetDGBConfigGossip() dgbcfg.GossipConfig {
d := dgbcfg.GossipConfig{}
if c.BindAddress != "" {
d.BindAddress = c.BindAddress
}
if c.AdvertiseAddress != "" {
d.AdvertiseAddress = c.AdvertiseAddress
}
if len(c.Seed) > 0 {
d.Seed = make([]string, 0)
for _, v := range c.Seed {
if v == "" {
continue
}
d.Seed = append(d.Seed, v)
}
}
return d
}
func (c ConfigGossip) Validate() liberr.Error {
err := ErrorValidateConfig.Error(nil)
if er := libval.New().Struct(c); er != nil {
if e, ok := er.(*libval.InvalidValidationError); ok {
err.Add(e)
}
for _, e := range er.(libval.ValidationErrors) {
//nolint goerr113
err.Add(fmt.Errorf("config field '%s' is not validated by constraint '%s'", e.Namespace(), e.ActualTag()))
}
}
if err.HasParent() {
return err
}
return nil
}

339
cluster/configNode.go
View File

@@ -1,339 +0,0 @@
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || riscv64 || s390x || sparc64 || wasm
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le riscv64 s390x sparc64 wasm
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package cluster
import (
"fmt"
libval "github.com/go-playground/validator/v10"
dgbcfg "github.com/lni/dragonboat/v3/config"
liberr "github.com/nabbar/golib/errors"
)
// nolint #maligned
type ConfigNode struct {
// DeploymentID is used to determine whether two NodeHost instances belong to
// the same deployment and thus allowed to communicate with each other. This
// helps to prvent accidentially misconfigured NodeHost instances to cause
// data corruption errors by sending out of context messages to unrelated
// Raft nodes.
// For a particular dragonboat based application, you can set DeploymentID
// to the same uint64 value on all production NodeHost instances, then use
// different DeploymentID values on your staging and dev environment. It is
// also recommended to use different DeploymentID values for different
// dragonboat based applications.
// When not set, the default value 0 will be used as the deployment ID and
// thus allowing all NodeHost instances with deployment ID 0 to communicate
// with each other.
DeploymentID uint64 `mapstructure:"deployment_id" json:"deployment_id" yaml:"deployment_id" toml:"deployment_id"`
// WALDir is the directory used for storing the WAL of Raft entries. It is
// recommended to use low latency storage such as NVME SSD with power loss
// protection to store such WAL data. Leave WALDir to have zero value will
// have everything stored in NodeHostDir.
WALDir string `mapstructure:"wal_dir" json:"wal_dir" yaml:"wal_dir" toml:"wal_dir" validate:"omitempty,printascii"`
// NodeHostDir is where everything else is stored.
NodeHostDir string `mapstructure:"node_host_dir" json:"node_host_dir" yaml:"node_host_dir" toml:"node_host_dir" validate:"omitempty,printascii"`
//nolint #godox
// RTTMillisecond defines the average Rround Trip Time (RTT) in milliseconds
// between two NodeHost instances. Such a RTT interval is internally used as
// a logical clock tick, Raft heartbeat and election intervals are both
// defined in term of how many such logical clock ticks (RTT intervals).
// Note that RTTMillisecond is the combined delays between two NodeHost
// instances including all delays caused by network transmission, delays
// caused by NodeHost queuing and processing. As an example, when fully
// loaded, the average Rround Trip Time between two of our NodeHost instances
// used for benchmarking purposes is up to 500 microseconds when the ping time
// between them is 100 microseconds. Set RTTMillisecond to 1 when it is less
// than 1 million in your environment.
RTTMillisecond uint64 `mapstructure:"rtt_millisecond" json:"rtt_millisecond" yaml:"rtt_millisecond" toml:"rtt_millisecond"`
// RaftAddress is a DNS name:port or IP:port address used by the transport
// module for exchanging Raft messages, snapshots and metadata between
// NodeHost instances. It should be set to the public address that can be
// accessed from remote NodeHost instances.
//
// When the NodeHostConfig.ListenAddress field is empty, NodeHost listens on
// RaftAddress for incoming Raft messages. When hostname or domain name is
// used, it will be resolved to IPv4 addresses first and Dragonboat listens
// to all resolved IPv4 addresses.
//
// By default, the RaftAddress value is not allowed to change between NodeHost
// restarts. AddressByNodeHostID should be set to true when the RaftAddress
// value might change after restart.
RaftAddress string `mapstructure:"raft_address" json:"raft_address" yaml:"raft_address" toml:"raft_address" validate:"omitempty,printascii"`
//nolint #godox
// AddressByNodeHostID indicates that NodeHost instances should be addressed
// by their NodeHostID values. This feature is usually used when only dynamic
// addresses are available. When enabled, NodeHostID values should be used
// as the target parameter when calling NodeHost's StartCluster,
// RequestAddNode, RequestAddObserver and RequestAddWitness methods.
//
// Enabling AddressByNodeHostID also enables the internal gossip service,
// NodeHostConfig.Gossip must be configured to control the behaviors of the
// gossip service.
//
// Note that once enabled, the AddressByNodeHostID setting can not be later
// disabled after restarts.
//
// Please see the godocs of the NodeHostConfig.Gossip field for a detailed
// example on how AddressByNodeHostID and gossip works.
AddressByNodeHostID bool `mapstructure:"address_by_node_host_id" json:"address_by_node_host_id" yaml:"address_by_node_host_id" toml:"address_by_node_host_id"`
// ListenAddress is an optional field in the hostname:port or IP:port address
// form used by the transport module to listen on for Raft message and
// snapshots. When the ListenAddress field is not set, The transport module
// listens on RaftAddress. If 0.0.0.0 is specified as the IP of the
// ListenAddress, Dragonboat listens to the specified port on all network
// interfaces. When hostname or domain name is used, it will be resolved to
// IPv4 addresses first and Dragonboat listens to all resolved IPv4 addresses.
ListenAddress string `mapstructure:"listen_address" json:"listen_address" yaml:"listen_address" toml:"listen_address" validate:"omitempty,hostname_port"`
// MutualTLS defines whether to use mutual TLS for authenticating servers
// and clients. Insecure communication is used when MutualTLS is set to
// False.
// See https://github.com/lni/dragonboat/wiki/TLS-in-Dragonboat for more
// details on how to use Mutual TLS.
MutualTLS bool `mapstructure:"mutual_tls" json:"mutual_tls" yaml:"tls" toml:"tls"`
// CAFile is the path of the CA certificate file. This field is ignored when
// MutualTLS is false.
CAFile string `mapstructure:"ca_file" json:"ca_file" yaml:"ca_file" toml:"ca_file"`
// CertFile is the path of the node certificate file. This field is ignored
// when MutualTLS is false.
CertFile string `mapstructure:"cert_file" json:"cert_file" yaml:"cert_file" toml:"cert_file"`
// KeyFile is the path of the node key file. This field is ignored when
// MutualTLS is false.
KeyFile string `mapstructure:"key_file" json:"key_file" yaml:"key_file" toml:"key_file"`
// EnableMetrics determines whether health metrics in Prometheus format should
// be enabled.
EnableMetrics bool `mapstructure:"enable_metrics" json:"enable_metrics" yaml:"enable_metrics" toml:"enable_metrics"`
// MaxSendQueueSize is the maximum size in bytes of each send queue.
// Once the maximum size is reached, further replication messages will be
// dropped to restrict memory usage. When set to 0, it means the send queue
// size is unlimited.
MaxSendQueueSize uint64 `mapstructure:"max_send_queue_size" json:"max_send_queue_size" yaml:"max_send_queue_size" toml:"max_send_queue_size"`
// MaxReceiveQueueSize is the maximum size in bytes of each receive queue.
// Once the maximum size is reached, further replication messages will be
// dropped to restrict memory usage. When set to 0, it means the queue size
// is unlimited.
MaxReceiveQueueSize uint64 `mapstructure:"max_receive_queue_size" json:"max_receive_queue_size" yaml:"max_receive_queue_size" toml:"max_receive_queue_size"`
// MaxSnapshotSendBytesPerSecond defines how much snapshot data can be sent
// every second for all Raft clusters managed by the NodeHost instance.
// The default value 0 means there is no limit set for snapshot streaming.
MaxSnapshotSendBytesPerSecond uint64 `mapstructure:"max_snapshot_send_bytes_per_second" json:"max_snapshot_send_bytes_per_second" yaml:"max_snapshot_send_bytes_per_second" toml:"max_snapshot_send_bytes_per_second"`
// MaxSnapshotRecvBytesPerSecond defines how much snapshot data can be
// received each second for all Raft clusters managed by the NodeHost instance.
// The default value 0 means there is no limit for receiving snapshot data.
MaxSnapshotRecvBytesPerSecond uint64 `mapstructure:"max_snapshot_recv_bytes_per_second" json:"max_snapshot_recv_bytes_per_second" yaml:"max_snapshot_recv_bytes_per_second" toml:"max_snapshot_recv_bytes_per_second"`
// NotifyCommit specifies whether clients should be notified when their
// regular proposals and config change requests are committed. By default,
// commits are not notified, clients are only notified when their proposals
// are both committed and applied.
NotifyCommit bool `mapstructure:"notify_commit" json:"notify_commit" yaml:"notify_commit" toml:"notify_commit"`
// Gossip contains configurations for the gossip service. When the
// AddressByNodeHostID field is set to true, each NodeHost instance will use
// an internal gossip service to exchange knowledges of known NodeHost
// instances including their RaftAddress and NodeHostID values. This Gossip
// field contains configurations that controls how the gossip service works.
//
// As an detailed example on how to use the gossip service in the situation
// where all available machines have dynamically assigned IPs on reboot -
//
// Consider that there are three NodeHost instances on three machines, each
// of them has a dynamically assigned IP address which will change on reboot.
// NodeHostConfig.RaftAddress should be set to the current address that can be
// reached by remote NodeHost instance. In this example, we will assume they
// are
//
// 10.0.0.100:24000
// 10.0.0.200:24000
// 10.0.0.300:24000
//
// To use these machines, first enable the NodeHostConfig.AddressByNodeHostID
// field and start the NodeHost instances. The NodeHostID value of each
// NodeHost instance can be obtained by calling NodeHost.ID(). Let's say they
// are
//
// "nhid-xxxxx",
// "nhid-yyyyy",
// "nhid-zzzzz".
//
// All these NodeHostID are fixed, they will never change after reboots.
//
// When starting Raft nodes or requesting new nodes to be added, use the above
// mentioned NodeHostID values as the target parameters (which are of the
// Target type). Let's say we want to start a Raft Node as a part of a three
// replicas Raft cluster, the initialMembers parameter of the StartCluster
// method can be set to
//
// initialMembers := map[uint64]Target {
// 1: "nhid-xxxxx",
// 2: "nhid-yyyyy",
// 3: "nhid-zzzzz",
// }
//
// This indicates that node 1 of the cluster will be running on the NodeHost
// instance identified by the NodeHostID value "nhid-xxxxx", node 2 of the
// same cluster will be running on the NodeHost instance identified by the
// NodeHostID value of "nhid-yyyyy" and so on.
//
// The internal gossip service exchanges NodeHost details, including their
// NodeHostID and RaftAddress values, with all other known NodeHost instances.
// Thanks to the nature of gossip, it will eventually allow each NodeHost
// instance to be aware of the current details of all NodeHost instances.
// As a result, let's say when Raft node 1 wants to send a Raft message to
// node 2, it first figures out that node 2 is running on the NodeHost
// identified by the NodeHostID value "nhid-yyyyy", RaftAddress information
// from the gossip service further shows that "nhid-yyyyy" maps to a machine
// currently reachable at 10.0.0.200:24000. Raft messages can thus be
// delivered.
//
// The Gossip field here is used to configure how the gossip service works.
// In this example, let's say we choose to use the following configurations
// for those three NodeHost instaces.
//
// GossipConfig {
// BindAddress: "10.0.0.100:24001",
// Seed: []string{10.0.0.200:24001},
// }
//
// GossipConfig {
// BindAddress: "10.0.0.200:24001",
// Seed: []string{10.0.0.300:24001},
// }
//
// GossipConfig {
// BindAddress: "10.0.0.300:24001",
// Seed: []string{10.0.0.100:24001},
// }
//
// For those three machines, the gossip component listens on
// "10.0.0.100:24001", "10.0.0.200:24001" and "10.0.0.300:24001" respectively
// for incoming gossip messages. The Seed field is a list of known gossip end
// points the local gossip service will try to talk to. The Seed field doesn't
// need to include all gossip end points, a few well connected nodes in the
// gossip network is enough.
Gossip ConfigGossip `mapstructure:"gossip" json:"gossip" yaml:"gossip" toml:"gossip" validate:"omitempty"`
// Expert contains options for expert users who are familiar with the internals
// of Dragonboat. Users are recommended not to use this field unless
// absoloutely necessary. It is important to note that any change to this field
// may cause an existing instance unable to restart, it may also cause negative
// performance impacts.
Expert ConfigExpert `mapstructure:"expert" json:"expert" yaml:"expert" toml:"expert" validate:"omitempty"`
}
func (c ConfigNode) GetDGBConfigNodeHost() dgbcfg.NodeHostConfig {
d := dgbcfg.NodeHostConfig{
DeploymentID: c.DeploymentID,
WALDir: c.WALDir,
NodeHostDir: c.NodeHostDir,
RaftAddress: c.RaftAddress,
ListenAddress: c.ListenAddress,
Gossip: c.Gossip.GetDGBConfigGossip(),
Expert: c.Expert.GetDGBConfigExpert(),
}
if c.RTTMillisecond > 0 {
d.RTTMillisecond = c.RTTMillisecond
}
if c.AddressByNodeHostID {
d.AddressByNodeHostID = true
}
if c.MutualTLS && c.CAFile != "" && c.CertFile != "" && c.KeyFile != "" {
d.MutualTLS = true
d.CAFile = c.CAFile
d.CertFile = c.CertFile
d.KeyFile = c.KeyFile
}
if c.EnableMetrics {
d.EnableMetrics = true
}
if c.MaxSendQueueSize > 0 {
d.MaxSendQueueSize = c.MaxSendQueueSize
}
if c.MaxReceiveQueueSize > 0 {
d.MaxReceiveQueueSize = c.MaxReceiveQueueSize
}
if c.MaxSnapshotSendBytesPerSecond > 0 {
d.MaxSnapshotSendBytesPerSecond = c.MaxSnapshotSendBytesPerSecond
}
if c.MaxSnapshotRecvBytesPerSecond > 0 {
d.MaxSnapshotRecvBytesPerSecond = c.MaxSnapshotRecvBytesPerSecond
}
if c.NotifyCommit {
d.NotifyCommit = true
}
return d
}
func (c ConfigNode) Validate() liberr.Error {
err := ErrorValidateConfig.Error(nil)
if er := libval.New().Struct(c); er != nil {
if e, ok := er.(*libval.InvalidValidationError); ok {
err.Add(e)
}
for _, e := range er.(libval.ValidationErrors) {
//nolint goerr113
err.Add(fmt.Errorf("config field '%s' is not validated by constraint '%s'", e.Namespace(), e.ActualTag()))
}
}
if err.HasParent() {
return err
}
return nil
}

114
cluster/errors.go
View File

@@ -1,114 +0,0 @@
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || riscv64 || s390x || sparc64 || wasm
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le riscv64 s390x sparc64 wasm
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package cluster
import liberr "github.com/nabbar/golib/errors"
const (
ErrorParamsEmpty liberr.CodeError = iota + liberr.MinPkgCluster
ErrorParamsMissing
ErrorParamsMismatching
ErrorLeader
ErrorLeaderTransfer
ErrorNodeUser
ErrorNodeHostNew
ErrorNodeHostStart
ErrorNodeHostJoin
ErrorNodeHostStop
ErrorNodeHostRestart
ErrorCommandSync
ErrorCommandASync
ErrorCommandLocal
ErrorValidateConfig
ErrorValidateCluster
ErrorValidateNode
ErrorValidateGossip
ErrorValidateExpert
ErrorValidateEngine
)
var isCodeError = false
func IsCodeError() bool {
return isCodeError
}
func init() {
isCodeError = liberr.ExistInMapMessage(ErrorParamsEmpty)
liberr.RegisterIdFctMessage(ErrorParamsEmpty, getMessage)
}
func getMessage(code liberr.CodeError) (message string) {
switch code {
case ErrorParamsEmpty:
return "at least one given parameter is empty"
case ErrorParamsMissing:
return "at least one given parameter is missing"
case ErrorParamsMismatching:
return "at least one given parameter does not match the awaiting type"
case ErrorLeader:
return "unable to retrieve cluster leader"
case ErrorLeaderTransfer:
return "unable to transfer cluster leader"
case ErrorNodeUser:
return "unable to retrieve cluster node user"
case ErrorNodeHostNew:
return "unable to init new cluster NodeHost"
case ErrorNodeHostStart:
return "unable to start cluster"
case ErrorNodeHostJoin:
return "unable to join cluster"
case ErrorNodeHostStop:
return "unable to stop cluster or node"
case ErrorNodeHostRestart:
return "unable to restart cluster node properly"
case ErrorCommandSync:
return "unable to call cluster synchronous command"
case ErrorCommandASync:
return "unable to call cluster asynchronous command"
case ErrorCommandLocal:
return "unable to call cluster local command"
case ErrorValidateConfig:
return "cluster main config seems to be invalid"
case ErrorValidateCluster:
return "cluster config seems to be invalid"
case ErrorValidateNode:
return "cluster node config seems to be invalid"
case ErrorValidateGossip:
return "cluster gossip config seems to be invalid"
case ErrorValidateExpert:
return "cluster expert config seems to be invalid"
case ErrorValidateEngine:
return "cluster engine config seems to be invalid"
}
return liberr.NullMessage
}

127
cluster/interface.go
View File

@@ -1,127 +0,0 @@
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || riscv64 || s390x || sparc64 || wasm
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le riscv64 s390x sparc64 wasm
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package cluster
import (
"context"
"io"
"time"
dgbclt "github.com/lni/dragonboat/v3"
dgbcli "github.com/lni/dragonboat/v3/client"
dgbcfg "github.com/lni/dragonboat/v3/config"
dgbstm "github.com/lni/dragonboat/v3/statemachine"
liberr "github.com/nabbar/golib/errors"
)
const (
_DefaultTimeoutCommandSync = 10 * time.Second
_DefaultTimeoutCommandAsync = 100 * time.Second
)
type Cluster interface {
GetConfig() dgbcfg.Config
SetConfig(cfg dgbcfg.Config)
GetNodeHostConfig() dgbcfg.NodeHostConfig
GetFctCreate() dgbstm.CreateStateMachineFunc
GetFctCreateConcurrent() dgbstm.CreateConcurrentStateMachineFunc
GetFctCreateOnDisk() dgbstm.CreateOnDiskStateMachineFunc
SetFctCreate(fctCreate interface{})
SetFctCreateSTM(fctCreate dgbstm.CreateStateMachineFunc)
SetFctCreateSTMConcurrent(fctCreate dgbstm.CreateConcurrentStateMachineFunc)
SetFctCreateSTMOnDisk(fctCreate dgbstm.CreateOnDiskStateMachineFunc)
GetMemberInit() map[uint64]dgbclt.Target
SetMemberInit(memberList map[uint64]dgbclt.Target)
SetTimeoutCommandSync(timeout time.Duration)
SetTimeoutCommandASync(timeout time.Duration)
HasNodeInfo(nodeId uint64) bool
RaftAddress() string
ID() string
ClusterStart(join bool) liberr.Error
ClusterStop(force bool) liberr.Error
ClusterRestart(force bool) liberr.Error
NodeStop(target uint64) liberr.Error
NodeRestart(force bool) liberr.Error
GetLeaderID() (leader uint64, valid bool, err liberr.Error)
GetNoOPSession() *dgbcli.Session
GetNodeHostInfo(opt dgbclt.NodeHostInfoOption) *dgbclt.NodeHostInfo
RequestLeaderTransfer(targetNodeID uint64) liberr.Error
HandlerMetrics(w io.Writer)
StaleReadDangerous(query interface{}) (interface{}, error)
SyncPropose(parent context.Context, session *dgbcli.Session, cmd []byte) (dgbstm.Result, liberr.Error)
SyncRead(parent context.Context, query interface{}) (interface{}, liberr.Error)
SyncGetClusterMembership(parent context.Context) (*dgbclt.Membership, liberr.Error)
SyncGetSession(parent context.Context) (*dgbcli.Session, liberr.Error)
SyncCloseSession(parent context.Context, cs *dgbcli.Session) liberr.Error
SyncRequestSnapshot(parent context.Context, opt dgbclt.SnapshotOption) (uint64, liberr.Error)
SyncRequestDeleteNode(parent context.Context, nodeID uint64, configChangeIndex uint64) liberr.Error
SyncRequestAddNode(parent context.Context, nodeID uint64, target string, configChangeIndex uint64) liberr.Error
SyncRequestAddObserver(parent context.Context, nodeID uint64, target string, configChangeIndex uint64) liberr.Error
SyncRequestAddWitness(parent context.Context, nodeID uint64, target string, configChangeIndex uint64) liberr.Error
SyncRemoveData(parent context.Context, nodeID uint64) liberr.Error
AsyncPropose(session *dgbcli.Session, cmd []byte) (*dgbclt.RequestState, liberr.Error)
AsyncProposeSession(session *dgbcli.Session) (*dgbclt.RequestState, liberr.Error)
AsyncReadIndex() (*dgbclt.RequestState, liberr.Error)
AsyncRequestCompaction(nodeID uint64) (*dgbclt.SysOpState, liberr.Error)
LocalReadNode(rs *dgbclt.RequestState, query interface{}) (interface{}, liberr.Error)
LocalNAReadNode(rs *dgbclt.RequestState, query []byte) ([]byte, liberr.Error)
}
func NewCluster(cfg Config, fctCreate interface{}) (Cluster, liberr.Error) {
c := &cRaft{
memberInit: cfg.GetInitMember(),
fctCreate: fctCreate,
config: cfg.GetDGBConfigCluster(),
nodeHost: nil,
timeoutCmdSync: _DefaultTimeoutCommandSync,
timeoutCmdASync: _DefaultTimeoutCommandAsync,
}
if n, e := dgbclt.NewNodeHost(cfg.GetDGBConfigNode()); e != nil {
return nil, ErrorNodeHostNew.Error(e)
} else {
c.nodeHost = n
}
return c, nil
}

56
cluster/local.go
View File

@@ -1,56 +0,0 @@
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || riscv64 || s390x || sparc64 || wasm
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le riscv64 s390x sparc64 wasm
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package cluster
import (
dgbclt "github.com/lni/dragonboat/v3"
liberr "github.com/nabbar/golib/errors"
)
func (c *cRaft) LocalReadNode(rs *dgbclt.RequestState, query interface{}) (interface{}, liberr.Error) {
i, e := c.nodeHost.ReadLocalNode(rs, query)
if e != nil {
return i, ErrorCommandLocal.Error(c.getErrorCommand("ReadNode"), e)
}
return i, nil
}
func (c *cRaft) LocalNAReadNode(rs *dgbclt.RequestState, query []byte) ([]byte, liberr.Error) {
r, e := c.nodeHost.NAReadLocalNode(rs, query)
if e != nil {
return r, ErrorCommandLocal.Error(c.getErrorCommand("ReadNode"), e)
}
return r, nil
}

110
cluster/logger.go
View File

@@ -1,110 +0,0 @@
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || riscv64 || s390x || sparc64 || wasm
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le riscv64 s390x sparc64 wasm
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package cluster
import (
"context"
dgblog "github.com/lni/dragonboat/v3/logger"
liblog "github.com/nabbar/golib/logger"
loglvl "github.com/nabbar/golib/logger/level"
)
const LogLib = "DragonBoat"
func SetLoggerFactory(log liblog.FuncLog) {
if log == nil {
log = func() liblog.Logger {
return liblog.New(context.Background)
}
}
dgblog.SetLoggerFactory(func(pkgName string) dgblog.ILogger {
return &logDragonBoat{
pkg: pkgName,
log: log,
}
})
}
type logDragonBoat struct {
pkg string
log liblog.FuncLog
}
func (l *logDragonBoat) SetLevel(level dgblog.LogLevel) {
if l.log == nil {
return
}
switch level {
case dgblog.CRITICAL:
l.log().SetLevel(loglvl.FatalLevel)
case dgblog.ERROR:
l.log().SetLevel(loglvl.ErrorLevel)
case dgblog.WARNING:
l.log().SetLevel(loglvl.WarnLevel)
case dgblog.INFO:
l.log().SetLevel(loglvl.InfoLevel)
case dgblog.DEBUG:
l.log().SetLevel(loglvl.DebugLevel)
}
}
func (l *logDragonBoat) logMsg(lvl loglvl.Level, message string, args ...interface{}) {
if l.log == nil {
l.log = func() liblog.Logger {
return liblog.New(context.Background)
}
}
l.log().Entry(lvl, message, args...).FieldAdd("lib", LogLib).FieldAdd("pkg", l.pkg).Log()
}
func (l *logDragonBoat) Debugf(format string, args ...interface{}) {
l.logMsg(loglvl.DebugLevel, format, args...)
}
func (l *logDragonBoat) Infof(format string, args ...interface{}) {
l.logMsg(loglvl.InfoLevel, format, args...)
}
func (l *logDragonBoat) Warningf(format string, args ...interface{}) {
l.logMsg(loglvl.WarnLevel, format, args...)
}
func (l *logDragonBoat) Errorf(format string, args ...interface{}) {
l.logMsg(loglvl.ErrorLevel, format, args...)
}
func (l *logDragonBoat) Panicf(format string, args ...interface{}) {
l.logMsg(loglvl.FatalLevel, format, args...)
}

248
cluster/sync.go
View File

@@ -1,248 +0,0 @@
//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || riscv64 || s390x || sparc64 || wasm
// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le riscv64 s390x sparc64 wasm
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package cluster
import (
"context"
"time"
dgbclt "github.com/lni/dragonboat/v3"
dgbcli "github.com/lni/dragonboat/v3/client"
dgbstm "github.com/lni/dragonboat/v3/statemachine"
liberr "github.com/nabbar/golib/errors"
)
func (c *cRaft) syncCtxTimeout(parent context.Context) (context.Context, context.CancelFunc) {
var (
ctx context.Context
cnl context.CancelFunc
)
if parent != nil {
ctx, cnl = context.WithDeadline(parent, time.Now().Add(c.timeoutCmdSync))
} else {
ctx, cnl = context.WithDeadline(context.Background(), time.Now().Add(c.timeoutCmdSync))
}
return ctx, cnl
}
func (c *cRaft) syncCtxCancel(cancel context.CancelFunc) {
if cancel != nil {
cancel()
}
}
func (c *cRaft) SyncPropose(parent context.Context, session *dgbcli.Session, cmd []byte) (dgbstm.Result, liberr.Error) {
ctx, cnl := c.syncCtxTimeout(parent)
defer c.syncCtxCancel(cnl)
r, e := c.nodeHost.SyncPropose(ctx, session, cmd)
if e != nil {
return r, ErrorCommandSync.Error(c.getErrorCommand("Propose"), e)
}
return r, nil
}
func (c *cRaft) SyncRead(parent context.Context, query interface{}) (interface{}, liberr.Error) {
ctx, cnl := c.syncCtxTimeout(parent)
defer c.syncCtxCancel(cnl)
r, e := c.nodeHost.SyncRead(ctx, c.config.ClusterID, query)
if e != nil {
return r, ErrorCommandSync.Error(c.getErrorCluster(), c.getErrorCommand("Read"), e)
}
return r, nil
}
func (c *cRaft) SyncGetClusterMembership(parent context.Context) (*dgbclt.Membership, liberr.Error) {
ctx, cnl := c.syncCtxTimeout(parent)
defer c.syncCtxCancel(cnl)
r, e := c.nodeHost.SyncGetClusterMembership(ctx, c.config.ClusterID)
if e != nil {
return r, ErrorCommandSync.Error(c.getErrorCluster(), c.getErrorCommand("GetClusterMembership"), e)
}
return r, nil
}
func (c *cRaft) SyncGetSession(parent context.Context) (*dgbcli.Session, liberr.Error) {
ctx, cnl := c.syncCtxTimeout(parent)
defer c.syncCtxCancel(cnl)
r, e := c.nodeHost.SyncGetSession(ctx, c.config.ClusterID)
if e != nil {
return r, ErrorCommandSync.Error(c.getErrorCluster(), c.getErrorCommand("GetSession"), e)
}
return r, nil
}
func (c *cRaft) SyncCloseSession(parent context.Context, cs *dgbcli.Session) liberr.Error {
ctx, cnl := c.syncCtxTimeout(parent)
defer c.syncCtxCancel(cnl)
e := c.nodeHost.SyncCloseSession(ctx, cs)
if e != nil {
return ErrorCommandSync.Error(c.getErrorCommand("CloseSession"), e)
}
return nil
}
func (c *cRaft) SyncRequestSnapshot(parent context.Context, opt dgbclt.SnapshotOption) (uint64, liberr.Error) {
ctx, cnl := c.syncCtxTimeout(parent)
defer c.syncCtxCancel(cnl)
r, e := c.nodeHost.SyncRequestSnapshot(ctx, c.config.ClusterID, opt)
if e != nil {
return r, ErrorCommandSync.Error(c.getErrorCluster(), c.getErrorCommand("RequestSnapshot"), e)
}
return r, nil
}
func (c *cRaft) SyncRequestDeleteNode(parent context.Context, nodeID uint64, configChangeIndex uint64) liberr.Error {
ctx, cnl := c.syncCtxTimeout(parent)
defer c.syncCtxCancel(cnl)
var en error
if nodeID == 0 {
nodeID = c.config.NodeID
en = c.getErrorNode()
} else {
en = c.getErrorNodeTarget(nodeID)
}
e := c.nodeHost.SyncRequestDeleteNode(ctx, c.config.ClusterID, nodeID, configChangeIndex)
if e != nil {
return ErrorCommandSync.Error(c.getErrorCluster(), en, c.getErrorCommand("RequestDeleteNode"), e)
}
return nil
}
// nolint #dupl
func (c *cRaft) SyncRequestAddNode(parent context.Context, nodeID uint64, target string, configChangeIndex uint64) liberr.Error {
ctx, cnl := c.syncCtxTimeout(parent)
defer c.syncCtxCancel(cnl)
var en error
if nodeID == 0 {
nodeID = c.config.NodeID
en = c.getErrorNode()
} else {
en = c.getErrorNodeTarget(nodeID)
}
e := c.nodeHost.SyncRequestAddNode(ctx, c.config.ClusterID, nodeID, target, configChangeIndex)
if e != nil {
return ErrorCommandSync.Error(c.getErrorCluster(), en, c.getErrorCommand("RequestAddNode"), e)
}
return nil
}
// nolint #dupl
func (c *cRaft) SyncRequestAddObserver(parent context.Context, nodeID uint64, target string, configChangeIndex uint64) liberr.Error {
ctx, cnl := c.syncCtxTimeout(parent)
defer c.syncCtxCancel(cnl)
var en error
if nodeID == 0 {
nodeID = c.config.NodeID
en = c.getErrorNode()
} else {
en = c.getErrorNodeTarget(nodeID)
}
e := c.nodeHost.SyncRequestAddObserver(ctx, c.config.ClusterID, nodeID, target, configChangeIndex)
if e != nil {
return ErrorCommandSync.Error(c.getErrorCluster(), en, c.getErrorCommand("RequestAddObserver"), e)
}
return nil
}
// nolint #dupl
func (c *cRaft) SyncRequestAddWitness(parent context.Context, nodeID uint64, target string, configChangeIndex uint64) liberr.Error {
ctx, cnl := c.syncCtxTimeout(parent)
defer c.syncCtxCancel(cnl)
var en error
if nodeID == 0 {
nodeID = c.config.NodeID
en = c.getErrorNode()
} else {
en = c.getErrorNodeTarget(nodeID)
}
e := c.nodeHost.SyncRequestAddWitness(ctx, c.config.ClusterID, nodeID, target, configChangeIndex)
if e != nil {
return ErrorCommandSync.Error(c.getErrorCluster(), en, c.getErrorCommand("RequestAddWitness"), e)
}
return nil
}
func (c *cRaft) SyncRemoveData(parent context.Context, nodeID uint64) liberr.Error {
ctx, cnl := c.syncCtxTimeout(parent)
defer c.syncCtxCancel(cnl)
var en error
if nodeID == 0 {
nodeID = c.config.NodeID
en = c.getErrorNode()
} else {
en = c.getErrorNodeTarget(nodeID)
}
e := c.nodeHost.SyncRemoveData(ctx, c.config.ClusterID, nodeID)
if e != nil {
return ErrorCommandSync.Error(c.getErrorCluster(), en, c.getErrorCommand("RemoveData"), e)
}
return nil
}

2
config/components.go
View File

@@ -31,12 +31,12 @@ import (
"encoding/json"
"fmt"
"io"
"slices"
cfgcst "github.com/nabbar/golib/config/const"
cfgtps "github.com/nabbar/golib/config/types"
loglvl "github.com/nabbar/golib/logger/level"
spfcbr "github.com/spf13/cobra"
"golang.org/x/exp/slices"
)
func (c *configModel) ComponentHas(key string) bool {

3
context/config.go
View File

@@ -26,10 +26,9 @@ package context
import (
"context"
"slices"
"sync"
"time"
"golang.org/x/exp/slices"
)
type FuncContext func() context.Context

126
go.mod
View File

@@ -1,15 +1,17 @@
module github.com/nabbar/golib
go 1.22.6
go 1.23
toolchain go1.23.2
require (
github.com/aws/aws-sdk-go v1.55.5
github.com/aws/aws-sdk-go-v2 v1.31.0
github.com/aws/aws-sdk-go-v2/config v1.27.39
github.com/aws/aws-sdk-go-v2/credentials v1.17.37
github.com/aws/aws-sdk-go-v2/service/iam v1.36.3
github.com/aws/aws-sdk-go-v2/service/s3 v1.63.3
github.com/aws/smithy-go v1.21.0
github.com/aws/aws-sdk-go-v2 v1.32.0
github.com/aws/aws-sdk-go-v2/config v1.27.41
github.com/aws/aws-sdk-go-v2/credentials v1.17.39
github.com/aws/aws-sdk-go-v2/service/iam v1.37.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0
github.com/aws/smithy-go v1.22.0
github.com/bits-and-blooms/bitset v1.14.3
github.com/c-bata/go-prompt v0.2.6
github.com/dsnet/compress v0.0.1
@@ -25,7 +27,6 @@ require (
github.com/hashicorp/go-uuid v1.0.3
github.com/hashicorp/go-version v1.7.0
github.com/jlaffaye/ftp v0.2.0
github.com/lni/dragonboat/v3 v3.3.8
github.com/matcornic/hermes/v2 v2.1.0
github.com/mattn/go-colorable v0.1.13
github.com/mitchellh/go-homedir v1.1.0
@@ -33,7 +34,6 @@ require (
github.com/nats-io/jwt/v2 v2.7.0
github.com/nats-io/nats-server/v2 v2.10.21
github.com/nats-io/nats.go v1.37.0
github.com/nutsdb/nutsdb v1.0.4
github.com/onsi/ginkgo/v2 v2.20.2
github.com/onsi/gomega v1.34.2
github.com/pelletier/go-toml v1.9.5
@@ -47,15 +47,13 @@ require (
github.com/ugorji/go/codec v1.2.12
github.com/ulikunitz/xz v0.5.12
github.com/vbauerster/mpb/v8 v8.8.3
github.com/xanzy/go-gitlab v0.109.0
github.com/xanzy/go-gitlab v0.110.0
github.com/xhit/go-simple-mail v2.2.2+incompatible
github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
golang.org/x/net v0.29.0
golang.org/x/net v0.30.0
golang.org/x/oauth2 v0.23.0
golang.org/x/sync v0.8.0
golang.org/x/sys v0.25.0
golang.org/x/term v0.24.0
golang.org/x/sys v0.26.0
golang.org/x/term v0.25.0
gopkg.in/yaml.v3 v3.0.1
gorm.io/driver/clickhouse v0.6.1
gorm.io/driver/mysql v1.5.7
@@ -70,103 +68,80 @@ require (
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/ClickHouse/ch-go v0.62.0 // indirect
github.com/ClickHouse/clickhouse-go/v2 v2.29.0 // indirect
github.com/Masterminds/semver v1.4.2 // indirect
github.com/Masterminds/sprig v2.16.0+incompatible // indirect
github.com/PuerkitoBio/goquery v1.5.0 // indirect
github.com/VictoriaMetrics/metrics v1.6.2 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
github.com/Masterminds/sprig v2.22.0+incompatible // indirect
github.com/PuerkitoBio/goquery v1.10.0 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/andybalholm/cascadia v1.0.0 // indirect
github.com/antlabs/stl v0.0.2 // indirect
github.com/antlabs/timer v0.1.4 // indirect
github.com/aokoli/goutils v1.0.1 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.23.3 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.31.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.0 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.32.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bwmarrin/snowflake v0.3.0 // indirect
github.com/bytedance/sonic v1.12.3 // indirect
github.com/bytedance/sonic/loader v0.2.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cloudwego/base64x v0.1.4 // indirect
github.com/cloudwego/iasm v0.2.0 // indirect
github.com/cockroachdb/errors v1.7.5 // indirect
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
github.com/cockroachdb/pebble v0.0.0-20210331181633-27fc006b8bfb // indirect
github.com/cockroachdb/redact v1.0.6 // indirect
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
github.com/gabriel-vasile/mimetype v1.4.5 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.7.1 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/gofrs/flock v0.12.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/css v1.0.0 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v0.5.3 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-sockaddr v1.0.0 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/memberlist v0.2.2 // indirect
github.com/huandu/xstrings v1.2.0 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/huandu/xstrings v1.5.0 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.7.1 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 // indirect
github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/juju/ratelimit v1.0.2-0.20191002062651-f60b32039441 // indirect
github.com/klauspost/compress v1.17.10 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lni/goutils v1.3.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mattn/go-sqlite3 v1.14.23 // indirect
github.com/mattn/go-sqlite3 v1.14.24 // indirect
github.com/mattn/go-tty v0.0.7 // indirect
github.com/microsoft/go-mssqldb v1.7.2 // indirect
github.com/miekg/dns v1.1.43 // indirect
github.com/minio/highwayhash v1.0.3 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@@ -178,14 +153,12 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/term v1.2.0-beta.2 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/common v0.60.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sagikazarmark/locafero v0.6.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
@@ -194,23 +167,20 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tidwall/btree v1.7.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/valyala/fastrand v1.0.0 // indirect
github.com/valyala/histogram v1.0.1 // indirect
github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 // indirect
github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe // indirect
github.com/vanng822/css v1.0.1 // indirect
github.com/vanng822/go-premailer v1.21.0 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xujiajun/mmap-go v1.0.1 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/otel v1.30.0 // indirect
go.opentelemetry.io/otel/trace v1.30.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/arch v0.10.0 // indirect
golang.org/x/crypto v0.27.0 // indirect
golang.org/x/text v0.18.0 // indirect
golang.org/x/time v0.6.0 // indirect
golang.org/x/tools v0.25.0 // indirect
golang.org/x/arch v0.11.0 // indirect
golang.org/x/crypto v0.28.0 // indirect
golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 // indirect
golang.org/x/text v0.19.0 // indirect
golang.org/x/time v0.7.0 // indirect
golang.org/x/tools v0.26.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
)

2
mail/email.go
View File

@@ -25,7 +25,7 @@
package mail
import "golang.org/x/exp/slices"
import "slices"
const (
headerFrom = "From"

2
monitor/metrics.go
View File

@@ -28,11 +28,11 @@ package monitor
import (
"context"
"slices"
"time"
monsts "github.com/nabbar/golib/monitor/status"
libprm "github.com/nabbar/golib/prometheus"
"golang.org/x/exp/slices"
)
func (o *mon) RegisterMetricsName(names ...string) {

225
nutsdb/Command.go
View File

@@ -1,225 +0,0 @@
//go:build !386 && !arm && !mips && !mipsle
// +build !386,!arm,!mips,!mipsle
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package nutsdb
import (
"github.com/nutsdb/nutsdb"
)
type Commands interface {
CommandTransaction
CommandBPTree
CommandSet
CommandList
CommandZSet
}
type CommandTransaction interface {
// Put sets the value for a key in the bucket.
Put(bucket string, key, value []byte, ttl uint32) error
// PutWithTimestamp sets the value for a key in the bucket but allow capabilities to custom the timestamp for ttl.
PutWithTimestamp(bucket string, key, value []byte, ttl uint32, timestamp uint64) error
}
type CommandBPTree interface {
// Get retrieves the value for a key in the bucket.
// The returned value is only valid for the life of the transaction.
Get(bucket string, key []byte) (e *nutsdb.Entry, err error)
//GetAll returns all keys and values of the bucket stored at given bucket.
GetAll(bucket string) (entries nutsdb.Entries, err error)
// RangeScan query a range at given bucket, start and end slice.
RangeScan(bucket string, start, end []byte) (es nutsdb.Entries, err error)
// PrefixScan iterates over a key prefix at given bucket, prefix and limitNum.
// LimitNum will limit the number of entries return.
PrefixScan(bucket string, prefix []byte, offsetNum int, limitNum int) (es nutsdb.Entries, off int, err error)
// PrefixSearchScan iterates over a key prefix at given bucket, prefix, match regular expression and limitNum.
// LimitNum will limit the number of entries return.
PrefixSearchScan(bucket string, prefix []byte, reg string, offsetNum int, limitNum int) (es nutsdb.Entries, off int, err error)
// Delete removes a key from the bucket at given bucket and key.
Delete(bucket string, key []byte) error
}
type CommandSet interface {
// SAdd adds the specified members to the set stored int the bucket at given bucket,key and items.
SAdd(bucket string, key []byte, items ...[]byte) error
// SRem removes the specified members from the set stored int the bucket at given bucket,key and items.
SRem(bucket string, key []byte, items ...[]byte) error
// SAreMembers returns if the specified members are the member of the set int the bucket at given bucket,key and items.
SAreMembers(bucket string, key []byte, items ...[]byte) (bool, error)
// SIsMember returns if member is a member of the set stored int the bucket at given bucket,key and item.
SIsMember(bucket string, key, item []byte) (bool, error)
// SMembers returns all the members of the set value stored int the bucket at given bucket and key.
SMembers(bucket string, key []byte) (list [][]byte, err error)
// SHasKey returns if the set in the bucket at given bucket and key.
SHasKey(bucket string, key []byte) (bool, error)
// SPop removes and returns one or more random elements from the set value store in the bucket at given bucket and key.
SPop(bucket string, key []byte) ([]byte, error)
// SCard returns the set cardinality (number of elements) of the set stored in the bucket at given bucket and key.
SCard(bucket string, key []byte) (int, error)
// SDiffByOneBucket returns the members of the set resulting from the difference
// between the first set and all the successive sets in one bucket.
SDiffByOneBucket(bucket string, key1, key2 []byte) (list [][]byte, err error)
// SDiffByTwoBuckets returns the members of the set resulting from the difference
// between the first set and all the successive sets in two buckets.
SDiffByTwoBuckets(bucket1 string, key1 []byte, bucket2 string, key2 []byte) (list [][]byte, err error)
// SMoveByOneBucket moves member from the set at source to the set at destination in one bucket.
SMoveByOneBucket(bucket string, key1, key2, item []byte) (bool, error)
// SMoveByTwoBuckets moves member from the set at source to the set at destination in two buckets.
SMoveByTwoBuckets(bucket1 string, key1 []byte, bucket2 string, key2, item []byte) (bool, error)
// SUnionByOneBucket the members of the set resulting from the union of all the given sets in one bucket.
SUnionByOneBucket(bucket string, key1, key2 []byte) (list [][]byte, err error)
// SUnionByTwoBuckets the members of the set resulting from the union of all the given sets in two buckets.
SUnionByTwoBuckets(bucket1 string, key1 []byte, bucket2 string, key2 []byte) (list [][]byte, err error)
}
type CommandList interface {
// RPop removes and returns the last element of the list stored in the bucket at given bucket and key.
RPop(bucket string, key []byte) (item []byte, err error)
// RPeek returns the last element of the list stored in the bucket at given bucket and key.
RPeek(bucket string, key []byte) (item []byte, err error)
// RPush inserts the values at the tail of the list stored in the bucket at given bucket,key and values.
RPush(bucket string, key []byte, values ...[]byte) error
// LPush inserts the values at the head of the list stored in the bucket at given bucket,key and values.
LPush(bucket string, key []byte, values ...[]byte) error
// LPop removes and returns the first element of the list stored in the bucket at given bucket and key.
LPop(bucket string, key []byte) (item []byte, err error)
// LPeek returns the first element of the list stored in the bucket at given bucket and key.
LPeek(bucket string, key []byte) (item []byte, err error)
// LSize returns the size of key in the bucket in the bucket at given bucket and key.
LSize(bucket string, key []byte) (int, error)
// LRange returns the specified elements of the list stored in the bucket at given bucket,key, start and end.
// The offsets start and stop are zero-based indexes 0 being the first element of the list (the head of the list),
// 1 being the next element and so on.
// Start and end can also be negative numbers indicating offsets from the end of the list,
// where -1 is the last element of the list, -2 the penultimate element and so on.
LRange(bucket string, key []byte, start, end int) (list [][]byte, err error)
// LRem removes the first count occurrences of elements equal to value from the list stored in the bucket at given bucket,key,count.
// The count argument influences the operation in the following ways:
// count > 0: Remove elements equal to value moving from head to tail.
// count < 0: Remove elements equal to value moving from tail to head.
// count = 0: Remove all elements equal to value.
LRem(bucket string, key []byte, count int, value []byte) (removedNum int, err error)
// LSet sets the list element at index to value.
LSet(bucket string, key []byte, index int, value []byte) error
// LTrim trims an existing list so that it will contain only the specified range of elements specified.
// the offsets start and stop are zero-based indexes 0 being the first element of the list (the head of the list),
// 1 being the next element and so on.
// start and end can also be negative numbers indicating offsets from the end of the list,
// where -1 is the last element of the list, -2 the penultimate element and so on.
LTrim(bucket string, key []byte, start, end int) error
}
type CommandZSet interface {
// ZAdd adds the specified member key with the specified score and specified val to the sorted set stored at bucket.
ZAdd(bucket string, key []byte, score float64, val []byte) error
// ZMembers returns all the members of the set value stored at bucket.
ZMembers(bucket string) (map[string]*nutsdb.SortedSetMember, error)
// ZCard returns the sorted set cardinality (number of elements) of the sorted set stored at bucket.
ZCard(bucket string) (int, error)
// ZCount returns the number of elements in the sorted set at bucket with a score between min and max and opts.
// opts includes the following parameters:
// Limit int // limit the max nodes to return
// ExcludeStart bool // exclude start value, so it search in interval (start, end] or (start, end)
// ExcludeEnd bool // exclude end value, so it search in interval [start, end) or (start, end)
ZCount(bucket string, start, end float64, opts *nutsdb.GetByScoreRangeOptions) (int, error)
// ZPopMax removes and returns the member with the highest score in the sorted set stored at bucket.
ZPopMax(bucket string) (*nutsdb.SortedSetMember, error)
// ZPopMin removes and returns the member with the lowest score in the sorted set stored at bucket.
ZPopMin(bucket string) (*nutsdb.SortedSetMember, error)
// ZPeekMax returns the member with the highest score in the sorted set stored at bucket.
ZPeekMax(bucket string) (*nutsdb.SortedSetMember, error)
// ZPeekMin returns the member with the lowest score in the sorted set stored at bucket.
ZPeekMin(bucket string) (*nutsdb.SortedSetMember, error)
// ZRangeByScore returns all the elements in the sorted set at bucket with a score between min and max.
ZRangeByScore(bucket string, start, end float64, opts *nutsdb.GetByScoreRangeOptions) ([]*nutsdb.SortedSetMember, error)
// ZRangeByRank returns all the elements in the sorted set in one bucket and key
// with a rank between start and end (including elements with rank equal to start or end).
ZRangeByRank(bucket string, start, end int) ([]*nutsdb.SortedSetMember, error)
// ZRem removes the specified members from the sorted set stored in one bucket at given bucket and key.
ZRem(bucket, key string) error
// ZRemRangeByRank removes all elements in the sorted set stored in one bucket at given bucket with rank between start and end.
// the rank is 1-based integer. Rank 1 means the first node; Rank -1 means the last node.
ZRemRangeByRank(bucket string, start, end int) error
// ZRank returns the rank of member in the sorted set stored in the bucket at given bucket and key,
// with the scores ordered from low to high.
ZRank(bucket string, key []byte) (int, error)
// ZRevRank returns the rank of member in the sorted set stored in the bucket at given bucket and key,
// with the scores ordered from high to low.
ZRevRank(bucket string, key []byte) (int, error)
// ZScore returns the score of member in the sorted set in the bucket at given bucket and key.
ZScore(bucket string, key []byte) (float64, error)
// ZGetByKey returns node in the bucket at given bucket and key.
ZGetByKey(bucket string, key []byte) (*nutsdb.SortedSetMember, error)
}

1595
nutsdb/client.go
View File

File diff suppressed because it is too large Load Diff

777
nutsdb/cmdCode.go
View File

@@ -1,777 +0,0 @@
//go:build !386 && !arm && !mips && !mipsle
// +build !386,!arm,!mips,!mipsle
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package nutsdb
type CmdCode uint32
const (
// CmdUnknown is no Command.
CmdUnknown CmdCode = iota
// Command for transaction.
CmdPut
CmdPutWithTimestamp
// Command for BPTree.
CmdGet
CmdGetAll
CmdRangeScan
CmdPrefixScan
CmdPrefixSearchScan
CmdDelete
CmdFindTxIDOnDisk
CmdFindOnDisk
CmdFindLeafOnDisk
// Command for Set.
CmdSAdd
CmdSRem
CmdSAreMembers
CmdSIsMember
CmdSMembers
CmdSHasKey
CmdSPop
CmdSCard
CmdSDiffByOneBucket
CmdSDiffByTwoBuckets
CmdSMoveByOneBucket
CmdSMoveByTwoBuckets
CmdSUnionByOneBucket
CmdSUnionByTwoBuckets
// Command for List.
CmdRPop
CmdRPeek
CmdRPush
CmdLPush
CmdLPop
CmdLPeek
CmdLSize
CmdLRange
CmdLRem
CmdLSet
CmdLTrim
// Command for ZSet.
CmdZAdd
CmdZMembers
CmdZCard
CmdZCount
CmdZPopMax
CmdZPopMin
CmdZPeekMax
CmdZPeekMin
CmdZRangeByScore
CmdZRangeByRank
CmdZRem
CmdZRemRangeByRank
CmdZRank
CmdZRevRank
CmdZScore
CmdZGetByKey
)
// nolint #funlen
func CmdCodeFromName(name string) CmdCode {
switch name {
case CmdPut.Name():
return CmdPut
case CmdPutWithTimestamp.Name():
return CmdPutWithTimestamp
case CmdGet.Name():
return CmdGet
case CmdGetAll.Name():
return CmdGetAll
case CmdRangeScan.Name():
return CmdRangeScan
case CmdPrefixScan.Name():
return CmdPrefixScan
case CmdPrefixSearchScan.Name():
return CmdPrefixSearchScan
case CmdDelete.Name():
return CmdDelete
case CmdFindTxIDOnDisk.Name():
return CmdFindTxIDOnDisk
case CmdFindOnDisk.Name():
return CmdFindOnDisk
case CmdFindLeafOnDisk.Name():
return CmdFindLeafOnDisk
case CmdSAdd.Name():
return CmdSAdd
case CmdSRem.Name():
return CmdSRem
case CmdSAreMembers.Name():
return CmdSAreMembers
case CmdSIsMember.Name():
return CmdSIsMember
case CmdSMembers.Name():
return CmdSMembers
case CmdSHasKey.Name():
return CmdSHasKey
case CmdSPop.Name():
return CmdSPop
case CmdSCard.Name():
return CmdSCard
case CmdSDiffByOneBucket.Name():
return CmdSDiffByOneBucket
case CmdSDiffByTwoBuckets.Name():
return CmdSDiffByTwoBuckets
case CmdSMoveByOneBucket.Name():
return CmdSMoveByOneBucket
case CmdSMoveByTwoBuckets.Name():
return CmdSMoveByTwoBuckets
case CmdSUnionByOneBucket.Name():
return CmdSUnionByOneBucket
case CmdSUnionByTwoBuckets.Name():
return CmdSUnionByTwoBuckets
case CmdRPop.Name():
return CmdRPop
case CmdRPeek.Name():
return CmdRPeek
case CmdRPush.Name():
return CmdRPush
case CmdLPush.Name():
return CmdLPush
case CmdLPop.Name():
return CmdLPop
case CmdLPeek.Name():
return CmdLPeek
case CmdLSize.Name():
return CmdLSize
case CmdLRange.Name():
return CmdLRange
case CmdLRem.Name():
return CmdLRem
case CmdLSet.Name():
return CmdLSet
case CmdLTrim.Name():
return CmdLTrim
case CmdZAdd.Name():
return CmdZAdd
case CmdZMembers.Name():
return CmdZMembers
case CmdZCard.Name():
return CmdZCard
case CmdZCount.Name():
return CmdZCount
case CmdZPopMax.Name():
return CmdZPopMax
case CmdZPopMin.Name():
return CmdZPopMin
case CmdZPeekMax.Name():
return CmdZPeekMax
case CmdZPeekMin.Name():
return CmdZPeekMin
case CmdZRangeByScore.Name():
return CmdZRangeByScore
case CmdZRangeByRank.Name():
return CmdZRangeByRank
case CmdZRem.Name():
return CmdZRem
case CmdZRemRangeByRank.Name():
return CmdZRemRangeByRank
case CmdZRank.Name():
return CmdZRank
case CmdZRevRank.Name():
return CmdZRevRank
case CmdZScore.Name():
return CmdZScore
case CmdZGetByKey.Name():
return CmdZGetByKey
default:
return CmdUnknown
}
}
// nolint #funlen
func (c CmdCode) Name() string {
switch c {
case CmdPut:
return "Put"
case CmdPutWithTimestamp:
return "PutWithTimestamp"
case CmdGet:
return "Get"
case CmdGetAll:
return "GetAll"
case CmdRangeScan:
return "RangeScan"
case CmdPrefixScan:
return "PrefixScan"
case CmdPrefixSearchScan:
return "PrefixSearchScan"
case CmdDelete:
return "Delete"
case CmdFindTxIDOnDisk:
return "FindTxIDOnDisk"
case CmdFindOnDisk:
return "FindOnDisk"
case CmdFindLeafOnDisk:
return "FindLeafOnDisk"
case CmdSAdd:
return "SAdd"
case CmdSRem:
return "SRem"
case CmdSAreMembers:
return "SAreMembers"
case CmdSIsMember:
return "SIsMember"
case CmdSMembers:
return "SMembers"
case CmdSHasKey:
return "SHasKey"
case CmdSPop:
return "SPop"
case CmdSCard:
return "SCard"
case CmdSDiffByOneBucket:
return "SDiffByOneBucket"
case CmdSDiffByTwoBuckets:
return "SDiffByTwoBuckets"
case CmdSMoveByOneBucket:
return "SMoveByOneBucket"
case CmdSMoveByTwoBuckets:
return "SMoveByTwoBuckets"
case CmdSUnionByOneBucket:
return "SUnionByOneBucket"
case CmdSUnionByTwoBuckets:
return "SUnionByTwoBuckets"
case CmdRPop:
return "RPop"
case CmdRPeek:
return "RPeek"
case CmdRPush:
return "RPush"
case CmdLPush:
return "LPush"
case CmdLPop:
return "LPop"
case CmdLPeek:
return "LPeek"
case CmdLSize:
return "LSize"
case CmdLRange:
return "LRange"
case CmdLRem:
return "LRem"
case CmdLSet:
return "LSet"
case CmdLTrim:
return "LTrim"
case CmdZAdd:
return "ZAdd"
case CmdZMembers:
return "ZMembers"
case CmdZCard:
return "ZCard"
case CmdZCount:
return "ZCount"
case CmdZPopMax:
return "ZPopMax"
case CmdZPopMin:
return "ZPopMin"
case CmdZPeekMax:
return "ZPeekMax"
case CmdZPeekMin:
return "ZPeekMin"
case CmdZRangeByScore:
return "ZRangeByScore"
case CmdZRangeByRank:
return "ZRangeByRank"
case CmdZRem:
return "ZRem"
case CmdZRemRangeByRank:
return "ZRemRangeByRank"
case CmdZRank:
return "ZRank"
case CmdZRevRank:
return "ZRevRank"
case CmdZScore:
return "ZScore"
case CmdZGetByKey:
return "ZGetByKey"
default:
return ""
}
}
// nolint #funlen
func (c CmdCode) Desc() string {
switch c {
case CmdPut:
return "Sets the value for a key in the bucket."
case CmdPutWithTimestamp:
return "Sets the value for a key in the bucket but allow capabilities to custom the timestamp for ttl"
case CmdGet:
return "Retrieves the value for a key in the bucket"
case CmdGetAll:
return "Returns all keys and values of the bucket stored at given bucket"
case CmdRangeScan:
return "Query a range at given bucket, start and end slice."
case CmdPrefixScan:
return "Iterates over a key prefix at given bucket, prefix and limitNum. LimitNum will limit the number of entries return."
case CmdPrefixSearchScan:
return "Iterates over a key prefix at given bucket, prefix, match regular expression and limitNum. LimitNum will limit the number of entries return."
case CmdDelete:
return "Removes a key from the bucket at given bucket and key."
case CmdFindTxIDOnDisk:
return "Returns if txId on disk at given fid and txID."
case CmdFindOnDisk:
return "Returns entry on disk at given fID, rootOff and key."
case CmdFindLeafOnDisk:
return "Returns binary leaf node on disk at given fId, rootOff and key."
case CmdSAdd:
return "Adds the specified members to the set stored int the bucket at given bucket,key and items."
case CmdSRem:
return "Removes the specified members from the set stored int the bucket at given bucket,key and items."
case CmdSAreMembers:
return "Returns if the specified members are the member of the set int the bucket at given bucket,key and items."
case CmdSIsMember:
return "Returns if member is a member of the set stored int the bucket at given bucket,key and item."
case CmdSMembers:
return "Returns all the members of the set value stored int the bucket at given bucket and key."
case CmdSHasKey:
return "Returns if the set in the bucket at given bucket and key."
case CmdSPop:
return "Removes and returns one or more random elements from the set value store in the bucket at given bucket and key."
case CmdSCard:
return "Returns the set cardinality (number of elements) of the set stored in the bucket at given bucket and key."
case CmdSDiffByOneBucket:
return "Returns the members of the set resulting from the difference between the first set and all the successive sets in one bucket."
case CmdSDiffByTwoBuckets:
return "Returns the members of the set resulting from the difference between the first set and all the successive sets in two buckets."
case CmdSMoveByOneBucket:
return "Moves member from the set at source to the set at destination in one bucket."
case CmdSMoveByTwoBuckets:
return "Moves member from the set at source to the set at destination in two buckets."
case CmdSUnionByOneBucket:
return "The members of the set resulting from the union of all the given sets in one bucket."
case CmdSUnionByTwoBuckets:
return "The members of the set resulting from the union of all the given sets in two buckets."
case CmdRPop:
return "Removes and returns the last element of the list stored in the bucket at given bucket and key."
case CmdRPeek:
return "Returns the last element of the list stored in the bucket at given bucket and key."
case CmdRPush:
return "Inserts the values at the tail of the list stored in the bucket at given bucket,key and values."
case CmdLPush:
return "Inserts the values at the head of the list stored in the bucket at given bucket,key and values."
case CmdLPop:
return "Removes and returns the first element of the list stored in the bucket at given bucket and key."
case CmdLPeek:
return "Returns the first element of the list stored in the bucket at given bucket and key."
case CmdLSize:
return "Returns the size of key in the bucket in the bucket at given bucket and key."
case CmdLRange:
return "Returns the specified elements of the list stored in the bucket at given bucket,key, start and end. \n" +
"The offsets start and stop are zero-based indexes 0 being the first element of the list (the head of the list), 1 being the next element and so on. \n" +
"Start and end can also be negative numbers indicating offsets from the end of the list, where -1 is the last element of the list, -2 the penultimate element and so on."
case CmdLRem:
return "Removes the first count occurrences of elements equal to value from the list stored in the bucket at given bucket,key,count. \n" +
"The count argument influences the operation in the following ways: \n" +
"count > 0: Remove elements equal to value moving from head to tail. \n" +
"count < 0: Remove elements equal to value moving from tail to head. \n" +
"count = 0: Remove all elements equal to value."
case CmdLSet:
return "Sets the list element at index to value."
case CmdLTrim:
return "Trims an existing list so that it will contain only the specified range of elements specified. \n" +
"The offsets start and stop are zero-based indexes 0 being the first element of the list (the head of the list), 1 being the next element and so on. \n" +
"Start and end can also be negative numbers indicating offsets from the end of the list, where -1 is the last element of the list, -2 the penultimate element and so on."
case CmdZAdd:
return "Adds the specified member key with the specified score and specified val to the sorted set stored at bucket."
case CmdZMembers:
return "Returns all the members of the set value stored at bucket."
case CmdZCard:
return "Returns the sorted set cardinality (number of elements) of the sorted set stored at bucket."
case CmdZCount:
return "Returns the number of elements in the sorted set at bucket with a score between min and max and opts. \n" +
"Options includes the following parameters: \n" +
"Limit: (int) the max nodes to return. \n" +
"ExcludeStart: (bool) exclude start value, so it search in interval (start, end] or (start, end). \n" +
"ExcludeEnd: (bool) exclude end value, so it search in interval [start, end) or (start, end)."
case CmdZPopMax:
return "Removes and returns the member with the highest score in the sorted set stored at bucket."
case CmdZPopMin:
return "Removes and returns the member with the lowest score in the sorted set stored at bucket."
case CmdZPeekMax:
return "Returns the member with the highest score in the sorted set stored at bucket."
case CmdZPeekMin:
return "Returns the member with the lowest score in the sorted set stored at bucket."
case CmdZRangeByScore:
return "Returns all the elements in the sorted set at bucket with a score between min and max."
case CmdZRangeByRank:
return "Returns all the elements in the sorted set in one bucket and key with a rank between start and end (including elements with rank equal to start or end)."
case CmdZRem:
return "Removes the specified members from the sorted set stored in one bucket at given bucket and key."
case CmdZRemRangeByRank:
return "Removes all elements in the sorted set stored in one bucket at given bucket with rank between start and end. \n" +
"The rank is 1-based integer. Rank 1 means the first node; Rank -1 means the last node."
case CmdZRank:
return "Returns the rank of member in the sorted set stored in the bucket at given bucket and key, with the scores ordered from low to high."
case CmdZRevRank:
return "Returns the rank of member in the sorted set stored in the bucket at given bucket and key, with the scores ordered from high to low."
case CmdZScore:
return "Returns the score of member in the sorted set in the bucket at given bucket and key."
case CmdZGetByKey:
return "Returns node in the bucket at given bucket and key."
default:
return ""
}
}
// nolint #funlen
func (c CmdCode) Usage() string {
switch c {
case CmdPut:
return c.Name() + " <key> <value>"
case CmdPutWithTimestamp:
return "Sets the value for a key in the bucket but allow capabilities to custom the timestamp for ttl"
case CmdGet:
return "Retrieves the value for a key in the bucket"
case CmdGetAll:
return "Returns all keys and values of the bucket stored at given bucket"
case CmdRangeScan:
return "Query a range at given bucket, start and end slice."
case CmdPrefixScan:
return "Iterates over a key prefix at given bucket, prefix and limitNum. LimitNum will limit the number of entries return."
case CmdPrefixSearchScan:
return "Iterates over a key prefix at given bucket, prefix, match regular expression and limitNum. LimitNum will limit the number of entries return."
case CmdDelete:
return "Removes a key from the bucket at given bucket and key."
case CmdFindTxIDOnDisk:
return "Returns if txId on disk at given fid and txID."
case CmdFindOnDisk:
return "Returns entry on disk at given fID, rootOff and key."
case CmdFindLeafOnDisk:
return "Returns binary leaf node on disk at given fId, rootOff and key."
case CmdSAdd:
return "Adds the specified members to the set stored int the bucket at given bucket,key and items."
case CmdSRem:
return "Removes the specified members from the set stored int the bucket at given bucket,key and items."
case CmdSAreMembers:
return "Returns if the specified members are the member of the set int the bucket at given bucket,key and items."
case CmdSIsMember:
return "Returns if member is a member of the set stored int the bucket at given bucket,key and item."
case CmdSMembers:
return "Returns all the members of the set value stored int the bucket at given bucket and key."
case CmdSHasKey:
return "Returns if the set in the bucket at given bucket and key."
case CmdSPop:
return "Removes and returns one or more random elements from the set value store in the bucket at given bucket and key."
case CmdSCard:
return "Returns the set cardinality (number of elements) of the set stored in the bucket at given bucket and key."
case CmdSDiffByOneBucket:
return "Returns the members of the set resulting from the difference between the first set and all the successive sets in one bucket."
case CmdSDiffByTwoBuckets:
return "Returns the members of the set resulting from the difference between the first set and all the successive sets in two buckets."
case CmdSMoveByOneBucket:
return "Moves member from the set at source to the set at destination in one bucket."
case CmdSMoveByTwoBuckets:
return "Moves member from the set at source to the set at destination in two buckets."
case CmdSUnionByOneBucket:
return "The members of the set resulting from the union of all the given sets in one bucket."
case CmdSUnionByTwoBuckets:
return "The members of the set resulting from the union of all the given sets in two buckets."
case CmdRPop:
return "Removes and returns the last element of the list stored in the bucket at given bucket and key."
case CmdRPeek:
return "Returns the last element of the list stored in the bucket at given bucket and key."
case CmdRPush:
return "Inserts the values at the tail of the list stored in the bucket at given bucket,key and values."
case CmdLPush:
return "Inserts the values at the head of the list stored in the bucket at given bucket,key and values."
case CmdLPop:
return "Removes and returns the first element of the list stored in the bucket at given bucket and key."
case CmdLPeek:
return "Returns the first element of the list stored in the bucket at given bucket and key."
case CmdLSize:
return "Returns the size of key in the bucket in the bucket at given bucket and key."
case CmdLRange:
return "Returns the specified elements of the list stored in the bucket at given bucket,key, start and end. \n" +
"The offsets start and stop are zero-based indexes 0 being the first element of the list (the head of the list), 1 being the next element and so on. \n" +
"Start and end can also be negative numbers indicating offsets from the end of the list, where -1 is the last element of the list, -2 the penultimate element and so on."
case CmdLRem:
return "Removes the first count occurrences of elements equal to value from the list stored in the bucket at given bucket,key,count. \n" +
"The count argument influences the operation in the following ways: \n" +
"count > 0: Remove elements equal to value moving from head to tail. \n" +
"count < 0: Remove elements equal to value moving from tail to head. \n" +
"count = 0: Remove all elements equal to value."
case CmdLSet:
return "Sets the list element at index to value."
case CmdLTrim:
return "Trims an existing list so that it will contain only the specified range of elements specified. \n" +
"The offsets start and stop are zero-based indexes 0 being the first element of the list (the head of the list), 1 being the next element and so on. \n" +
"Start and end can also be negative numbers indicating offsets from the end of the list, where -1 is the last element of the list, -2 the penultimate element and so on."
case CmdZAdd:
return "Adds the specified member key with the specified score and specified val to the sorted set stored at bucket."
case CmdZMembers:
return "Returns all the members of the set value stored at bucket."
case CmdZCard:
return "Returns the sorted set cardinality (number of elements) of the sorted set stored at bucket."
case CmdZCount:
return "Returns the number of elements in the sorted set at bucket with a score between min and max and opts. \n" +
"Options includes the following parameters: \n" +
"Limit: (int) the max nodes to return. \n" +
"ExcludeStart: (bool) exclude start value, so it search in interval (start, end] or (start, end). \n" +
"ExcludeEnd: (bool) exclude end value, so it search in interval [start, end) or (start, end)."
case CmdZPopMax:
return "Removes and returns the member with the highest score in the sorted set stored at bucket."
case CmdZPopMin:
return "Removes and returns the member with the lowest score in the sorted set stored at bucket."
case CmdZPeekMax:
return "Returns the member with the highest score in the sorted set stored at bucket."
case CmdZPeekMin:
return "Returns the member with the lowest score in the sorted set stored at bucket."
case CmdZRangeByScore:
return "Returns all the elements in the sorted set at bucket with a score between min and max."
case CmdZRangeByRank:
return "Returns all the elements in the sorted set in one bucket and key with a rank between start and end (including elements with rank equal to start or end)."
case CmdZRem:
return "Removes the specified members from the sorted set stored in one bucket at given bucket and key."
case CmdZRemRangeByRank:
return "Removes all elements in the sorted set stored in one bucket at given bucket with rank between start and end. \n" +
"The rank is 1-based integer. Rank 1 means the first node; Rank -1 means the last node."
case CmdZRank:
return "Returns the rank of member in the sorted set stored in the bucket at given bucket and key, with the scores ordered from low to high."
case CmdZRevRank:
return "Returns the rank of member in the sorted set stored in the bucket at given bucket and key, with the scores ordered from high to low."
case CmdZScore:
return "Returns the score of member in the sorted set in the bucket at given bucket and key."
case CmdZGetByKey:
return "Returns node in the bucket at given bucket and key."
default:
return ""
}
}

112
nutsdb/config.go
View File

@@ -1,112 +0,0 @@
//go:build !386 && !arm && !mips && !mipsle
// +build !386,!arm,!mips,!mipsle
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package nutsdb
import (
"fmt"
moncfg "github.com/nabbar/golib/monitor/types"
libval "github.com/go-playground/validator/v10"
libclu "github.com/nabbar/golib/cluster"
liberr "github.com/nabbar/golib/errors"
"github.com/nutsdb/nutsdb"
)
type Config struct {
DB NutsDBOptions `mapstructure:"db" json:"db" yaml:"db" toml:"db" validate:""`
Cluster libclu.Config `mapstructure:"cluster" json:"cluster" yaml:"cluster" toml:"cluster" validate:""`
Directory NutsDBFolder `mapstructure:"directories" json:"directories" yaml:"directories" toml:"directories" validate:""`
Monitor moncfg.Config `mapstructure:"monitor" json:"monitor" yaml:"monitor" toml:"monitor" validate:""`
}
func (c Config) GetConfigFolder() NutsDBFolder {
return c.Directory
}
func (c Config) GetConfigDB() (nutsdb.Options, liberr.Error) {
if dir, err := c.Directory.GetDirectoryData(); err != nil {
return nutsdb.Options{}, err
} else {
return c.DB.GetNutsDBOptions(dir), nil
}
}
func (c Config) GetConfigCluster() (libclu.Config, liberr.Error) {
cfg := c.Cluster
if dir, err := c.Directory.GetDirectoryWal(); err != nil {
return cfg, err
} else {
cfg.Node.WALDir = dir
}
if dir, err := c.Directory.GetDirectoryHost(); err != nil {
return cfg, err
} else {
cfg.Node.NodeHostDir = dir
}
return cfg, nil
}
func (c Config) GetOptions() (Options, liberr.Error) {
return NewOptions(c.DB, c.Directory)
}
func (c Config) Validate() liberr.Error {
err := ErrorValidateConfig.Error(nil)
if er := libval.New().Struct(c); er != nil {
if e, ok := er.(*libval.InvalidValidationError); ok {
err.Add(e)
}
for _, e := range er.(libval.ValidationErrors) {
//nolint goerr113
err.Add(fmt.Errorf("config field '%s' is not validated by constraint '%s'", e.Namespace(), e.ActualTag()))
}
}
if err.HasParent() {
return err
}
return nil
}
func (c Config) ValidateDB() liberr.Error {
return c.DB.Validate()
}
func (c Config) ValidateCluster() liberr.Error {
return c.Cluster.Validate()
}

117
nutsdb/configDb.go
View File

@@ -1,117 +0,0 @@
//go:build !386 && !arm && !mips && !mipsle
// +build !386,!arm,!mips,!mipsle
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package nutsdb
import (
"fmt"
libval "github.com/go-playground/validator/v10"
liberr "github.com/nabbar/golib/errors"
nutsdb "github.com/nutsdb/nutsdb"
)
type NutsDBOptions struct {
// EntryIdxMode represents using which mode to index the entries.
EntryIdxMode nutsdb.EntryIdxMode `mapstructure:"entry_idx_mode" json:"entry_idx_mode" yaml:"entry_idx_mode" toml:"entry_idx_mode"`
// RWMode represents the read and write mode.
// RWMode includes two options: FileIO and MMap.
// FileIO represents the read and write mode using standard I/O.
// MMap represents the read and write mode using mmap.
RWMode nutsdb.RWMode `mapstructure:"rw_mode" json:"rw_mode" yaml:"rw_mode" toml:"rw_mode"`
// SegmentSize default value is 8 MBytes
SegmentSize int64 `mapstructure:"segment_size" json:"segment_size" yaml:"segment_size" toml:"segment_size"`
// SyncEnable represents if call Sync() function.
// if SyncEnable is false, high write performance but potential data loss likely.
// if SyncEnable is true, slower but persistent.
SyncEnable bool `mapstructure:"sync_enable" json:"sync_enable" yaml:"sync_enable" toml:"sync_enable"`
}
func (o NutsDBOptions) GetNutsDBOptions(dataDir string) nutsdb.Options {
d := nutsdb.DefaultOptions
if len(dataDir) < 1 {
d.RWMode = nutsdb.MMap
} else {
d.Dir = dataDir
//nolint #exhaustive
switch o.RWMode {
case nutsdb.MMap:
d.RWMode = nutsdb.MMap
default:
d.RWMode = nutsdb.FileIO
}
}
//nolint #exhaustive
switch o.EntryIdxMode {
case nutsdb.HintKeyAndRAMIdxMode:
d.EntryIdxMode = nutsdb.HintKeyAndRAMIdxMode
default:
d.EntryIdxMode = nutsdb.HintKeyValAndRAMIdxMode
}
if o.SegmentSize > 0 {
d.SegmentSize = o.SegmentSize
}
if o.SyncEnable {
d.SyncEnable = true
} else {
d.SyncEnable = false
}
return d
}
func (o NutsDBOptions) Validate() liberr.Error {
err := ErrorValidateConfig.Error(nil)
if er := libval.New().Struct(o); er != nil {
if e, ok := er.(*libval.InvalidValidationError); ok {
err.Add(e)
}
for _, e := range er.(libval.ValidationErrors) {
//nolint goerr113
err.Add(fmt.Errorf("config field '%s' is not validated by constraint '%s'", e.Namespace(), e.ActualTag()))
}
}
if err.HasParent() {
return err
}
return nil
}

203
nutsdb/configFs.go
View File

@@ -1,203 +0,0 @@
//go:build !386 && !arm && !mips && !mipsle
// +build !386,!arm,!mips,!mipsle
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package nutsdb
import (
"errors"
"fmt"
"os"
"path/filepath"
libval "github.com/go-playground/validator/v10"
liberr "github.com/nabbar/golib/errors"
)
const (
_DefaultFolderData = "data"
_DefaultFolderBackup = "backup"
_DefaultFolderWal = "wal"
_DefaultFolderHost = "host"
)
type NutsDBFolder struct {
// Working represents the main working folder witch will include sub directories : data, backup, temp...
// If the base directory is empty, all the sub directory will be absolute directories.
Base string `mapstructure:"base" json:"base" yaml:"base" toml:"base" validate:"dir,required"`
// Data represents the sub-dir for the opening database.
// By default, it will use `data` as sub folder.
Data string `mapstructure:"sub_data" json:"sub_data" yaml:"sub_data" toml:"sub_data" validate:"printascii,required"`
// Backup represents the sub-dir with all backup sub-folder.
// By default, it will use `backup` as sub folder.
Backup string `mapstructure:"sub_backup" json:"sub_backup" yaml:"sub_backup" toml:"sub_backup" validate:"printascii,required"`
// Temp represents the sub-dir for temporary file/dir.
// By default, it will use the system temporary folder.
Temp string `mapstructure:"sub_temp" json:"sub_temp" yaml:"sub_temp" toml:"sub_temp" validate:"printascii,required"`
// WalDir represents the sub-dir for cluster negociation.
// By default, it will use `wal` as sub folder.
WalDir string `mapstructure:"wal_dir" json:"wal_dir" yaml:"wal_dir" toml:"wal_dir" validate:"printascii,required"`
// HostDir represents the sub-dir for cluster storage.
// By default, it will use `host` as sub folder.
HostDir string `mapstructure:"host_dir" json:"host_dir" yaml:"host_dir" toml:"host_dir" validate:"printascii,required"`
// LimitNumberBackup represents how many backup will be keep.
LimitNumberBackup uint8 `mapstructure:"limit_number_backup" json:"limit_number_backup" yaml:"limit_number_backup" toml:"limit_number_backup"`
// Permission represents the permission apply to folder created.
// By default, it will use `0755` as permission.
Permission os.FileMode `mapstructure:"permission" json:"permission" yaml:"permission" toml:"permission"`
}
func (f NutsDBFolder) Validate() liberr.Error {
err := ErrorValidateConfig.Error(nil)
if er := libval.New().Struct(f); er != nil {
if e, ok := er.(*libval.InvalidValidationError); ok {
err.Add(e)
}
for _, e := range er.(libval.ValidationErrors) {
//nolint goerr113
err.Add(fmt.Errorf("config field '%s' is not validated by constraint '%s'", e.Namespace(), e.ActualTag()))
}
}
if err.HasParent() {
return err
}
return nil
}
func (f NutsDBFolder) getDirectory(base, dir string) (string, liberr.Error) {
if f.Permission == 0 {
f.Permission = 0770
}
var (
abs string
err error
)
if len(dir) < 1 {
return "", nil
}
if len(base) > 0 {
dir = filepath.Join(base, dir)
}
if abs, err = filepath.Abs(dir); err != nil {
return "", ErrorFolderCheck.Error(err)
}
if f.Permission == 0 {
f.Permission = 0755
}
if _, err = os.Stat(abs); err != nil && !errors.Is(err, os.ErrNotExist) {
return "", ErrorFolderCheck.Error(err)
} else if err != nil {
if err = os.MkdirAll(abs, f.Permission); err != nil {
return "", ErrorFolderCreate.Error(err)
}
}
return abs, nil
}
func (f NutsDBFolder) GetDirectoryBase() (string, liberr.Error) {
return f.getDirectory("", f.Base)
}
func (f NutsDBFolder) GetDirectoryData() (string, liberr.Error) {
if base, err := f.GetDirectoryBase(); err != nil {
return "", err
} else if fs, err := f.getDirectory(base, f.Data); err != nil {
return "", err
} else if fs == "" {
return f.getDirectory(base, _DefaultFolderData)
} else {
return fs, nil
}
}
func (f NutsDBFolder) GetDirectoryBackup() (string, liberr.Error) {
if base, err := f.GetDirectoryBase(); err != nil {
return "", err
} else if fs, err := f.getDirectory(base, f.Backup); err != nil {
return "", err
} else if fs == "" {
return f.getDirectory(base, _DefaultFolderBackup)
} else {
return fs, nil
}
}
func (f NutsDBFolder) GetDirectoryWal() (string, liberr.Error) {
if base, err := f.GetDirectoryBase(); err != nil {
return "", err
} else if fs, err := f.getDirectory(base, f.WalDir); err != nil {
return "", err
} else if fs == "" {
return f.getDirectory(base, _DefaultFolderWal)
} else {
return fs, nil
}
}
func (f NutsDBFolder) GetDirectoryHost() (string, liberr.Error) {
if base, err := f.GetDirectoryBase(); err != nil {
return "", err
} else if fs, err := f.getDirectory(base, f.HostDir); err != nil {
return "", err
} else if fs == "" {
return f.getDirectory(base, _DefaultFolderHost)
} else {
return fs, nil
}
}
func (f NutsDBFolder) GetDirectoryTemp() (string, liberr.Error) {
if base, err := f.GetDirectoryBase(); err != nil {
return "", err
} else if fs, err := f.getDirectory(base, f.Temp); err != nil {
return "", err
} else if fs == "" {
return f.getDirectory("", os.TempDir())
} else {
return fs, nil
}
}

279
nutsdb/entryKv.go
View File

@@ -1,279 +0,0 @@
//go:build !386 && !arm && !mips && !mipsle
// +build !386,!arm,!mips,!mipsle
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package nutsdb
import (
"context"
"fmt"
"reflect"
"runtime"
"strings"
"github.com/fxamacker/cbor/v2"
liberr "github.com/nabbar/golib/errors"
liblog "github.com/nabbar/golib/logger"
loglvl "github.com/nabbar/golib/logger/level"
"github.com/nutsdb/nutsdb"
)
const (
_MinSkipCaller = 2
)
type CommandRequest struct {
l liblog.FuncLog
Cmd CmdCode `mapstructure:"cmd" json:"cmd" yaml:"cmd" toml:"cmd" cbor:"cmd"`
Params []interface{} `mapstructure:"params" json:"params" yaml:"params" toml:"params" cbor:"params"`
}
type CommandResponse struct {
Error error `mapstructure:"error" json:"error" yaml:"error" toml:"error" cbor:"error"`
Value []interface{} `mapstructure:"value" json:"value" yaml:"value" toml:"value" cbor:"value"`
}
func NewCommand() *CommandRequest {
return &CommandRequest{
Cmd: CmdUnknown,
Params: make([]interface{}, 0),
}
}
func NewCommandByDecode(l liblog.FuncLog, p []byte) (*CommandRequest, liberr.Error) {
d := NewCommand()
if e := cbor.Unmarshal(p, d); e != nil {
return nil, ErrorCommandUnmarshal.Error(e)
}
d.SetLogger(l)
return d, nil
}
func NewCommandByCaller(params ...interface{}) *CommandRequest {
pc := make([]uintptr, 10) // at least 1 entry needed
runtime.Callers(_MinSkipCaller, pc)
f := runtime.FuncForPC(pc[0])
d := NewCommand()
fn := strings.Split(f.Name(), ".")
d.Cmd = CmdCodeFromName(fn[len(fn)-1])
if len(params) > 0 {
d.Params = params
}
return d
}
func (c *CommandRequest) SetLogger(l liblog.FuncLog) {
if l != nil {
c.l = l
}
}
func (c *CommandRequest) GetLogger() liblog.Logger {
if c.l != nil {
return c.l()
}
var log = liblog.New(context.Background)
c.l = func() liblog.Logger {
return log
}
return log
}
func (c *CommandRequest) InitParams(num int) {
c.Params = make([]interface{}, num)
}
func (c *CommandRequest) InitParamsCounter(num int) int {
c.InitParams(num)
return 0
}
func (c *CommandRequest) SetParams(num int, val interface{}) {
if num < len(c.Params) {
c.Params[num] = val
return
}
tmp := c.Params
c.Params = make([]interface{}, len(c.Params)+1)
if len(tmp) > 0 {
for i := 0; i < len(tmp); i++ {
c.Params[i] = tmp[i]
}
}
c.Params[num] = val
}
func (c *CommandRequest) SetParamsInc(num int, val interface{}) int {
c.SetParams(num, val)
num++
return num
}
func (c *CommandRequest) EncodeRequest() ([]byte, liberr.Error) {
if p, e := cbor.Marshal(c); e != nil {
return nil, ErrorCommandMarshal.Error(e)
} else {
return p, nil
}
}
func (c *CommandRequest) DecodeResult(p []byte) (*CommandResponse, liberr.Error) {
res := CommandResponse{}
if e := cbor.Unmarshal(p, &res); e != nil {
return nil, ErrorCommandResultUnmarshal.Error(e)
} else {
return &res, nil
}
}
func (c *CommandRequest) RunLocal(tx *nutsdb.Tx) (*CommandResponse, liberr.Error) {
if tx == nil {
return nil, ErrorTransactionClosed.Error(nil)
}
if c.Cmd == CmdUnknown {
return nil, ErrorClientCommandInvalid.Error(nil)
}
valTx := reflect.ValueOf(tx)
mtName := c.Cmd.Name()
method := valTx.MethodByName(mtName)
nbPrm := method.Type().NumIn()
if len(c.Params) != nbPrm {
//nolint #goerr113
return nil, ErrorClientCommandParamsBadNumber.Error(fmt.Errorf("%s need %d parameters", c.Cmd.Name(), nbPrm))
}
params := make([]reflect.Value, nbPrm)
for i := 0; i < nbPrm; i++ {
v := reflect.ValueOf(c.Params[i])
c.GetLogger().Entry(loglvl.DebugLevel, "Param %d : type %s - Val %v", i, v.Type().Name(), v.Interface()).Log()
if v.Type().Kind() == method.Type().In(i).Kind() {
params[i] = v
continue
}
if !v.Type().ConvertibleTo(method.Type().In(i)) {
//nolint #goerr113
return nil, ErrorClientCommandParamsMismatching.Error(fmt.Errorf("cmd: %s", mtName), fmt.Errorf("param num: %d", i), fmt.Errorf("param type: %s, avaitting type: %s", v.Type().Kind(), method.Type().In(i).Kind()))
}
//nolint #exhaustive
switch method.Type().In(i).Kind() {
case reflect.Bool:
params[i] = reflect.ValueOf(v.Bool())
case reflect.Int:
params[i] = reflect.ValueOf(int(v.Int()))
case reflect.Int8:
params[i] = reflect.ValueOf(int8(v.Int()))
case reflect.Int16:
params[i] = reflect.ValueOf(int8(v.Int()))
case reflect.Int32:
params[i] = reflect.ValueOf(int16(v.Int()))
case reflect.Int64:
params[i] = reflect.ValueOf(v.Int())
case reflect.Uintptr:
params[i] = reflect.ValueOf(v.UnsafeAddr())
case reflect.Uint:
params[i] = reflect.ValueOf(uint(v.Uint()))
case reflect.Uint8:
params[i] = reflect.ValueOf(uint8(v.Uint()))
case reflect.Uint16:
params[i] = reflect.ValueOf(uint16(v.Uint()))
case reflect.Uint32:
params[i] = reflect.ValueOf(uint32(v.Uint()))
case reflect.Uint64:
params[i] = reflect.ValueOf(v.Uint())
case reflect.Float32:
params[i] = reflect.ValueOf(float32(v.Float()))
case reflect.Float64:
params[i] = reflect.ValueOf(v.Float())
case reflect.Complex64:
params[i] = reflect.ValueOf(complex64(v.Complex()))
case reflect.Complex128:
params[i] = reflect.ValueOf(v.Complex())
case reflect.Interface:
params[i] = reflect.ValueOf(v.Interface())
case reflect.String:
params[i] = reflect.ValueOf(v.String())
}
c.GetLogger().Entry(loglvl.DebugLevel, "Change Param %d : type %s to %v", i, v.Type().Name(), params[i].Type().Name()).Log()
}
resp := method.Call(params)
ret := CommandResponse{
Error: nil,
Value: make([]interface{}, 0),
}
for i := 0; i < len(resp); i++ {
v := resp[i].Interface()
if e, ok := v.(error); ok {
ret.Error = e
} else {
ret.Value = append(ret.Value, v)
}
}
if ret.Error == nil && len(ret.Value) < 1 {
return nil, nil
}
return &ret, nil
}
func (c *CommandRequest) Run(tx *nutsdb.Tx) ([]byte, liberr.Error) {
if c.Cmd == CmdUnknown {
return nil, ErrorClientCommandInvalid.Error(nil)
}
if r, err := c.RunLocal(tx); err != nil {
return nil, err
} else if p, e := cbor.Marshal(r); e != nil {
return nil, ErrorCommandResultMarshal.Error(e)
} else {
return p, nil
}
}

164
nutsdb/errors.go
View File

@@ -1,164 +0,0 @@
//go:build !386 && !arm && !mips && !mipsle
// +build !386,!arm,!mips,!mipsle
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package nutsdb
import (
"fmt"
liberr "github.com/nabbar/golib/errors"
)
const pkgName = "golib/nutsdb"
const (
ErrorParamEmpty liberr.CodeError = iota + liberr.MinPkgNutsDB
ErrorParamMissing
ErrorParamMismatching
ErrorParamInvalid
ErrorParamInvalidNumber
ErrorValidateConfig
ErrorValidateNutsDB
ErrorClusterInit
ErrorFileTemp
ErrorFolderCheck
ErrorFolderCreate
ErrorFolderCopy
ErrorFolderDelete
ErrorFolderExtract
ErrorFolderArchive
ErrorFolderCompress
ErrorDatabaseClosed
ErrorDatabaseKeyInvalid
ErrorDatabaseBackup
ErrorDatabaseSnapshot
ErrorTransactionInit
ErrorTransactionClosed
ErrorTransactionCommit
ErrorTransactionPutKey
ErrorCommandInvalid
ErrorCommandUnmarshal
ErrorCommandMarshal
ErrorCommandResultUnmarshal
ErrorCommandResultMarshal
ErrorLogEntryAdd
ErrorClientCommandInvalid
ErrorClientCommandParamsBadNumber
ErrorClientCommandParamsMismatching
ErrorClientCommandCall
ErrorClientCommandCommit
ErrorClientCommandResponseInvalid
)
func init() {
if liberr.ExistInMapMessage(ErrorParamEmpty) {
panic(fmt.Errorf("error code collision %s", pkgName))
}
liberr.RegisterIdFctMessage(ErrorParamEmpty, getMessage)
}
func getMessage(code liberr.CodeError) (message string) {
switch code {
case liberr.UnknownError:
return liberr.NullMessage
case ErrorParamEmpty:
return "at least one given parameter is empty"
case ErrorParamMissing:
return "at least one given parameter is missing"
case ErrorParamMismatching:
return "at least one given parameter does not match the awaiting type"
case ErrorParamInvalid:
return "at least one given parameter is invalid"
case ErrorParamInvalidNumber:
return "the number of parameters is not matching the awaiting number"
case ErrorValidateConfig:
return "config seems to be invalid"
case ErrorValidateNutsDB:
return "database config seems to be invalid"
case ErrorClusterInit:
return "cannot start or join cluster"
case ErrorFileTemp:
return "error while trying to create new temp file"
case ErrorFolderCheck:
return "error while trying to check or stat folder"
case ErrorFolderCreate:
return "error while trying to create folder"
case ErrorFolderCopy:
return "error while trying to copy folder"
case ErrorFolderArchive:
return "error while trying to archive folder"
case ErrorFolderCompress:
return "error while trying to compress folder"
case ErrorFolderExtract:
return "error while trying to extract snapshot archive"
case ErrorDatabaseClosed:
return "database is closed"
case ErrorDatabaseKeyInvalid:
return "database key seems to be invalid"
case ErrorDatabaseBackup:
return "error occured while trying to backup database folder"
case ErrorDatabaseSnapshot:
return "error occured while trying to backup database to cluster members"
case ErrorTransactionInit:
return "cannot initialize new transaction from database"
case ErrorTransactionClosed:
return "transaction is closed"
case ErrorTransactionCommit:
return "cannot commit transaction writable into database"
case ErrorTransactionPutKey:
return "cannot send Put command into database transaction"
case ErrorCommandInvalid:
return "given query is not a valid DB command"
case ErrorCommandUnmarshal:
return "cannot unmarshall DB command"
case ErrorCommandMarshal:
return "cannot marshall DB command"
case ErrorCommandResultUnmarshal:
return "cannot unmarshall DB command result"
case ErrorCommandResultMarshal:
return "cannot marshall DB command result"
case ErrorLogEntryAdd:
return "cannot add key/value to database"
case ErrorClientCommandInvalid:
return "invalid command"
case ErrorClientCommandParamsBadNumber:
return "invalid number of parameters for client command"
case ErrorClientCommandParamsMismatching:
return "invalid type of parameter for client command"
case ErrorClientCommandCall:
return "error occured while running client command"
case ErrorClientCommandCommit:
return "error occured while commit client command"
case ErrorClientCommandResponseInvalid:
return "response of requested client command seems to be invalid"
}
return liberr.NullMessage
}

78
nutsdb/interface.go
View File

@@ -1,78 +0,0 @@
//go:build !386 && !arm && !mips && !mipsle
// +build !386,!arm,!mips,!mipsle
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package nutsdb
import (
"context"
"sync/atomic"
"time"
libclu "github.com/nabbar/golib/cluster"
libctx "github.com/nabbar/golib/context"
liberr "github.com/nabbar/golib/errors"
liblog "github.com/nabbar/golib/logger"
montps "github.com/nabbar/golib/monitor/types"
shlcmd "github.com/nabbar/golib/shell/command"
libver "github.com/nabbar/golib/version"
)
const LogLib = "NutsDB"
type NutsDB interface {
Listen() liberr.Error
Restart() liberr.Error
Shutdown() liberr.Error
ForceRestart()
ForceShutdown()
IsRunning() bool
IsReady(ctx context.Context) bool
IsReadyTimeout(parent context.Context, dur time.Duration) bool
WaitReady(ctx context.Context, tick time.Duration)
GetLogger() liblog.Logger
SetLogger(l liblog.FuncLog)
Monitor(ctx libctx.FuncContext, vrs libver.Version) (montps.Monitor, error)
Cluster() libclu.Cluster
Client(ctx context.Context, tickSync time.Duration) Client
ShellCommand(ctx func() context.Context, tickSync time.Duration) []shlcmd.Command
}
func New(c Config) NutsDB {
return &ndb{
c: c,
t: new(atomic.Value),
r: new(atomic.Value),
}
}

352
nutsdb/model.go
View File

@@ -1,352 +0,0 @@
//go:build !386 && !arm && !mips && !mipsle
// +build !386,!arm,!mips,!mipsle
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package nutsdb
import (
"context"
"sync"
"sync/atomic"
"time"
dgbstm "github.com/lni/dragonboat/v3/statemachine"
libclu "github.com/nabbar/golib/cluster"
liberr "github.com/nabbar/golib/errors"
liblog "github.com/nabbar/golib/logger"
shlcmd "github.com/nabbar/golib/shell/command"
)
type ndb struct {
m sync.Mutex
c Config
l liblog.FuncLog
e liberr.Error
t *atomic.Value // cluster
r *atomic.Value // status
}
func (n *ndb) createNodeMachine(node uint64, cluster uint64) dgbstm.IOnDiskStateMachine {
var (
err liberr.Error
opt Options
)
if opt, err = n.c.GetOptions(); err != nil {
panic(err)
}
return newNode(node, cluster, opt, n.setRunning)
}
func (n *ndb) newCluster() liberr.Error {
var (
clu libclu.Cluster
err liberr.Error
cfg libclu.Config
)
if c := n.Cluster(); c != nil {
if err = n.Shutdown(); err != nil {
return err
}
}
if cfg, err = n.c.GetConfigCluster(); err != nil {
return err
}
clu, err = libclu.NewCluster(cfg, nil)
if err != nil {
return err
}
clu.SetFctCreateSTMOnDisk(n.createNodeMachine)
n.setCluster(clu)
return nil
}
func (n *ndb) GetLogger() liblog.Logger {
n.m.Lock()
defer n.m.Unlock()
if n.l != nil {
return n.l()
}
var l = liblog.New(context.Background)
n.l = func() liblog.Logger {
return l
}
return l
}
func (n *ndb) SetLogger(l liblog.FuncLog) {
n.m.Lock()
defer n.m.Unlock()
n.l = l
}
func (n *ndb) IsRunning() bool {
n.m.Lock()
defer n.m.Unlock()
if i := n.r.Load(); i == nil {
return false
} else if b, ok := i.(bool); !ok {
return false
} else {
return b
}
}
func (n *ndb) setRunning(state bool) {
n.m.Lock()
defer n.m.Unlock()
if n == nil || n.r == nil {
return
} else {
n.r.Store(state)
}
}
func (n *ndb) IsReady(ctx context.Context) bool {
if m, e := n.Cluster().SyncGetClusterMembership(ctx); e != nil || m == nil || len(m.Nodes) < 1 {
return false
}
if _, ok, e := n.Cluster().GetLeaderID(); e != nil || !ok {
return false
} else {
return true
}
}
func (n *ndb) IsReadyTimeout(parent context.Context, dur time.Duration) bool {
ctx, cnl := context.WithTimeout(parent, dur)
defer cnl()
if n.IsRunning() && n.IsReady(ctx) {
return true
}
return false
}
func (n *ndb) WaitReady(ctx context.Context, tick time.Duration) {
for {
if n.IsRunning() && n.IsReady(ctx) {
return
}
time.Sleep(tick)
}
}
func (n *ndb) Listen() liberr.Error {
var (
c libclu.Cluster
e liberr.Error
)
if c = n.Cluster(); c == nil {
if e = n.newCluster(); e != nil {
n._SetError(e)
return e
} else if c = n.Cluster(); c == nil {
n._SetError(e)
return ErrorClusterInit.Error(nil)
}
}
if e = c.ClusterStart(len(n.c.Cluster.InitMember) < 1); e != nil {
n._SetError(e)
return e
}
n._SetError(nil)
n.setCluster(c)
return nil
}
func (n *ndb) Restart() liberr.Error {
return n.Listen()
}
func (n *ndb) ForceRestart() {
n.ForceShutdown()
_ = n.Listen()
}
func (n *ndb) Shutdown() liberr.Error {
if c := n.Cluster(); c == nil {
return nil
} else if !c.HasNodeInfo(0) {
return nil
} else if err := c.NodeStop(0); err != nil {
return err
} else {
n.setCluster(c)
return nil
}
}
func (n *ndb) ForceShutdown() {
if err := n.Shutdown(); err == nil {
return
} else if c := n.Cluster(); c == nil {
return
} else if !c.HasNodeInfo(0) {
return
} else {
_ = c.ClusterStop(true)
n.setCluster(c)
}
}
func (n *ndb) Cluster() libclu.Cluster {
n.m.Lock()
defer n.m.Unlock()
if i := n.t.Load(); i == nil {
return nil
} else if c, ok := i.(libclu.Cluster); !ok {
return nil
} else {
return c
}
}
func (n *ndb) setCluster(clu libclu.Cluster) {
n.m.Lock()
defer n.m.Unlock()
n.t.Store(clu)
}
func (n *ndb) Client(ctx context.Context, tickSync time.Duration) Client {
return &clientNutDB{
x: ctx,
t: tickSync,
c: n.Cluster,
w: n.WaitReady,
}
}
func (n *ndb) ShellCommand(ctx func() context.Context, tickSync time.Duration) []shlcmd.Command {
var (
res = make([]shlcmd.Command, 0)
cli func() Client
)
cli = func() Client {
x := ctx()
if x.Err() != nil {
return nil
}
return n.Client(x, tickSync)
}
res = append(res, newShellCommand(CmdPut, cli))
res = append(res, newShellCommand(CmdPutWithTimestamp, cli))
res = append(res, newShellCommand(CmdGet, cli))
res = append(res, newShellCommand(CmdGetAll, cli))
res = append(res, newShellCommand(CmdRangeScan, cli))
res = append(res, newShellCommand(CmdPrefixScan, cli))
res = append(res, newShellCommand(CmdPrefixSearchScan, cli))
res = append(res, newShellCommand(CmdDelete, cli))
res = append(res, newShellCommand(CmdFindTxIDOnDisk, cli))
res = append(res, newShellCommand(CmdFindOnDisk, cli))
res = append(res, newShellCommand(CmdFindLeafOnDisk, cli))
res = append(res, newShellCommand(CmdSAdd, cli))
res = append(res, newShellCommand(CmdSRem, cli))
res = append(res, newShellCommand(CmdSAreMembers, cli))
res = append(res, newShellCommand(CmdSIsMember, cli))
res = append(res, newShellCommand(CmdSMembers, cli))
res = append(res, newShellCommand(CmdSHasKey, cli))
res = append(res, newShellCommand(CmdSPop, cli))
res = append(res, newShellCommand(CmdSCard, cli))
res = append(res, newShellCommand(CmdSDiffByOneBucket, cli))
res = append(res, newShellCommand(CmdSDiffByTwoBuckets, cli))
res = append(res, newShellCommand(CmdSMoveByOneBucket, cli))
res = append(res, newShellCommand(CmdSMoveByTwoBuckets, cli))
res = append(res, newShellCommand(CmdSUnionByOneBucket, cli))
res = append(res, newShellCommand(CmdSUnionByTwoBuckets, cli))
res = append(res, newShellCommand(CmdRPop, cli))
res = append(res, newShellCommand(CmdRPeek, cli))
res = append(res, newShellCommand(CmdRPush, cli))
res = append(res, newShellCommand(CmdLPush, cli))
res = append(res, newShellCommand(CmdLPop, cli))
res = append(res, newShellCommand(CmdLPeek, cli))
res = append(res, newShellCommand(CmdLSize, cli))
res = append(res, newShellCommand(CmdLRange, cli))
res = append(res, newShellCommand(CmdLRem, cli))
res = append(res, newShellCommand(CmdLSet, cli))
res = append(res, newShellCommand(CmdLTrim, cli))
res = append(res, newShellCommand(CmdZAdd, cli))
res = append(res, newShellCommand(CmdZMembers, cli))
res = append(res, newShellCommand(CmdZCard, cli))
res = append(res, newShellCommand(CmdZCount, cli))
res = append(res, newShellCommand(CmdZPopMax, cli))
res = append(res, newShellCommand(CmdZPopMin, cli))
res = append(res, newShellCommand(CmdZPeekMax, cli))
res = append(res, newShellCommand(CmdZPeekMin, cli))
res = append(res, newShellCommand(CmdZRangeByScore, cli))
res = append(res, newShellCommand(CmdZRangeByRank, cli))
res = append(res, newShellCommand(CmdZRem, cli))
res = append(res, newShellCommand(CmdZRemRangeByRank, cli))
res = append(res, newShellCommand(CmdZRank, cli))
res = append(res, newShellCommand(CmdZRevRank, cli))
res = append(res, newShellCommand(CmdZScore, cli))
res = append(res, newShellCommand(CmdZGetByKey, cli))
return res
}
func (n *ndb) _SetError(e liberr.Error) {
n.m.Lock()
defer n.m.Unlock()
n.e = e
}
func (n *ndb) _GetError() liberr.Error {
n.m.Lock()
defer n.m.Unlock()
return n.e
}

110
nutsdb/monitor.go
View File

@@ -1,110 +0,0 @@
/*
* MIT License
*
* Copyright (c) 2022 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/
package nutsdb
import (
"context"
"fmt"
"runtime"
"time"
libctx "github.com/nabbar/golib/context"
libmon "github.com/nabbar/golib/monitor"
moninf "github.com/nabbar/golib/monitor/info"
montps "github.com/nabbar/golib/monitor/types"
libver "github.com/nabbar/golib/version"
)
const (
defaultNameMonitor = "NutsDB Server"
)
func (n *ndb) HealthCheck(ctx context.Context) error {
for i := 0; i < 5; i++ {
if n.IsRunning() {
if n.IsReadyTimeout(context.Background(), time.Second) {
return nil
}
}
time.Sleep(time.Second)
}
if e := n._GetError(); e != nil {
return e
}
return fmt.Errorf("node not ready")
}
func (n *ndb) Monitor(ctx libctx.FuncContext, vrs libver.Version) (montps.Monitor, error) {
var (
e error
inf moninf.Info
mon montps.Monitor
cfg Config
res = make(map[string]interface{}, 0)
)
n.m.Lock()
cfg = n.c
n.m.Unlock()
res["runtime"] = runtime.Version()[2:]
res["release"] = vrs.GetRelease()
res["build"] = vrs.GetBuild()
res["date"] = vrs.GetDate()
res["nodId"] = n.c.Cluster.Cluster.NodeID
if inf, e = moninf.New(defaultNameMonitor); e != nil {
return nil, e
} else {
inf.RegisterName(func() (string, error) {
return fmt.Sprintf("%s [%s]", defaultNameMonitor, cfg.Cluster.Node.RaftAddress), nil
})
inf.RegisterInfo(func() (map[string]interface{}, error) {
return res, nil
})
}
if mon, e = libmon.New(ctx, inf); e != nil {
return nil, e
}
mon.SetHealthCheck(n.HealthCheck)
if e = mon.SetConfig(ctx, cfg.Monitor); e != nil {
return nil, e
}
if e = mon.Start(ctx()); e != nil {
return nil, e
}
return mon, nil
}

427
nutsdb/node.go
View File

@@ -1,427 +0,0 @@
//go:build !386 && !arm && !mips && !mipsle
// +build !386,!arm,!mips,!mipsle
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package nutsdb
import (
"context"
"errors"
"io"
"strings"
"sync"
"sync/atomic"
liblog "github.com/nabbar/golib/logger"
dgbstm "github.com/lni/dragonboat/v3/statemachine"
liberr "github.com/nabbar/golib/errors"
"github.com/nutsdb/nutsdb"
)
const (
_RaftBucket = "_raft"
_RaftKeyAppliedIndex = "_raft_applied_index"
_WordByteTrue = 0xff
)
func newNode(node uint64, cluster uint64, opt Options, fct func(state bool)) dgbstm.IOnDiskStateMachine {
if fct == nil {
fct = func(state bool) {}
}
o := new(atomic.Value)
o.Store(opt)
return &nutsNode{
n: node,
c: cluster,
o: o,
r: fct,
d: new(atomic.Value),
}
}
type nutsNode struct {
m sync.Mutex // mutex for struct var
n uint64 // nodeId
c uint64 // clusterId
r func(state bool) // is running
o *atomic.Value // options nutsDB
d *atomic.Value // nutsDB database pointer
l liblog.FuncLog // logger
}
func (n *nutsNode) SetLogger(l liblog.FuncLog) {
n.m.Lock()
defer n.m.Unlock()
if l != nil {
n.l = l
}
}
func (n *nutsNode) GetLogger() liblog.Logger {
n.m.Lock()
defer n.m.Unlock()
if n.l != nil {
return n.l()
}
var l = liblog.New(context.Background)
n.l = func() liblog.Logger {
return l
}
return l
}
func (n *nutsNode) setRunning(state bool) {
n.m.Lock()
defer n.m.Unlock()
if n != nil && n.r != nil {
n.r(state)
}
}
func (n *nutsNode) newTx(writable bool) (*nutsdb.Tx, liberr.Error) {
if db := n.getDb(); db == nil {
return nil, ErrorDatabaseClosed.Error(nil)
} else if tx, e := db.Begin(writable); e != nil {
return nil, ErrorTransactionInit.Error(e)
} else {
return tx, nil
}
}
func (n *nutsNode) getRaftLogIndexLastApplied() (idxRaftlog uint64, err error) {
var (
tx *nutsdb.Tx
en *nutsdb.Entry
)
if tx, err = n.newTx(false); err != nil {
return 0, err
}
defer func() {
_ = tx.Rollback()
}()
if en, err = tx.Get(_RaftBucket, []byte(_RaftKeyAppliedIndex)); err != nil && errors.Is(err, nutsdb.ErrBucketNotFound) {
return 0, nil
} else if err != nil && errors.Is(err, nutsdb.ErrBucketEmpty) {
return 0, nil
} else if err != nil && errors.Is(err, nutsdb.ErrNotFoundKey) {
return 0, nil
} else if err != nil && errors.Is(err, nutsdb.ErrKeyNotFound) {
return 0, nil
} else if err != nil && errors.Is(err, nutsdb.ErrKeyEmpty) {
return 0, nil
} else if err != nil && strings.HasPrefix(err.Error(), "not found bucket") {
return 0, nil
} else if err != nil {
return 0, err
} else if en.IsZero() {
return 0, nil
} else {
return n.btoi64(en.Value), nil
}
}
func (n *nutsNode) i64tob(val uint64) []byte {
r := make([]byte, 8)
for i := uint64(0); i < 8; i++ {
r[i] = byte((val >> (i * 8)) & _WordByteTrue)
}
return r
}
func (n *nutsNode) btoi64(val []byte) uint64 {
r := uint64(0)
for i := uint64(0); i < 8; i++ {
r |= uint64(val[i]) << (8 * i)
}
return r
}
func (n *nutsNode) applyRaftLogIndexLastApplied(idx uint64) error {
var (
e error
tx *nutsdb.Tx
err liberr.Error
)
if tx, err = n.newTx(true); err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
if e = tx.Put(_RaftBucket, []byte(_RaftKeyAppliedIndex), n.i64tob(idx), 0); e != nil {
return ErrorTransactionPutKey.Error(e)
} else if e = tx.Commit(); e != nil {
return ErrorTransactionCommit.Error(e)
} else {
return nil
}
}
// Open @TODO : analyze channel role !!
func (n *nutsNode) Open(stopc <-chan struct{}) (idxRaftlog uint64, err error) {
var (
opt Options
db *nutsdb.DB
)
if opt = n.getOptions(); opt == nil {
return 0, ErrorValidateConfig.Error(nil)
}
if db, err = nutsdb.Open(opt.NutsDBOptions()); err != nil {
return 0, err
} else {
n.setDb(db)
n.setRunning(true)
if idxRaftlog, err = n.getRaftLogIndexLastApplied(); err != nil {
_ = n.Close()
}
return
}
}
func (n *nutsNode) Close() error {
defer n.setRunning(false)
if db := n.getDb(); db != nil {
err := db.Close()
n.setDb(db)
return err
}
return nil
}
func (n *nutsNode) Update(logEntry []dgbstm.Entry) ([]dgbstm.Entry, error) {
var (
e error
tx *nutsdb.Tx
kv *CommandRequest
err liberr.Error
res []byte
idx int
ent dgbstm.Entry
)
if tx, err = n.newTx(true); err != nil {
return nil, err
}
for idx, ent = range logEntry {
if kv, err = NewCommandByDecode(n.GetLogger, ent.Cmd); err != nil {
logEntry[idx].Result = dgbstm.Result{
Value: 0,
Data: nil,
}
_ = tx.Rollback()
return logEntry, err
}
if res, err = kv.Run(tx); err != nil {
logEntry[idx].Result = dgbstm.Result{
Value: 0,
Data: nil,
}
_ = tx.Rollback()
return logEntry, err
} else {
logEntry[idx].Result = dgbstm.Result{
Value: uint64(idx),
Data: res,
}
}
}
if e = tx.Commit(); e != nil {
_ = tx.Rollback()
return logEntry, ErrorTransactionCommit.Error(e)
}
return logEntry, n.applyRaftLogIndexLastApplied(logEntry[len(logEntry)-1].Index)
}
func (n *nutsNode) Lookup(query interface{}) (value interface{}, err error) {
var (
t *nutsdb.Tx
r *CommandResponse
c *CommandRequest
e liberr.Error
ok bool
)
if t, e = n.newTx(true); e != nil {
return nil, e
}
defer func() {
if t != nil {
_ = t.Rollback()
}
}()
if c, ok = query.(*CommandRequest); !ok {
return nil, ErrorCommandInvalid.Error(nil)
} else if r, e = c.RunLocal(t); e != nil {
return nil, e
} else {
return r, nil
}
}
func (n *nutsNode) Sync() error {
return nil
}
func (n *nutsNode) PrepareSnapshot() (interface{}, error) {
var sh = newSnap()
if opt := n.getOptions(); opt == nil {
return nil, ErrorValidateConfig.Error(nil)
} else if db := n.getDb(); db == nil {
sh.Finish()
return nil, ErrorDatabaseClosed.Error(nil)
} else if err := sh.Prepare(opt, db); err != nil {
sh.Finish()
return nil, ErrorDatabaseBackup.Error(err)
} else {
return sh, nil
}
}
func (n *nutsNode) SaveSnapshot(i interface{}, writer io.Writer, c <-chan struct{}) error {
if i == nil {
return ErrorParamEmpty.Error(nil)
} else if sh, ok := snapCast(i); !ok {
return ErrorParamMismatching.Error(nil)
} else if opt := n.getOptions(); opt == nil {
return ErrorValidateConfig.Error(nil)
} else if err := sh.Save(opt, writer); err != nil {
sh.Finish()
return err
} else {
sh.Finish()
}
return nil
}
func (n *nutsNode) RecoverFromSnapshot(reader io.Reader, c <-chan struct{}) error {
var (
sh = newSnap()
opt = n.getOptions()
)
defer sh.Finish()
if opt == nil {
return ErrorValidateConfig.Error(nil)
}
if err := sh.Load(opt, reader); err != nil {
return err
}
if db := n.getDb(); db != nil {
_ = db.Close()
n.setDb(db)
}
if err := sh.Apply(opt); err != nil {
return err
}
//@TODO : check channel is ok....
if _, err := n.Open(nil); err != nil {
return err
}
return nil
}
func (n *nutsNode) getDb() *nutsdb.DB {
n.m.Lock()
defer n.m.Unlock()
if n.d == nil {
return nil
} else if i := n.d.Load(); i == nil {
return nil
} else if db, ok := i.(*nutsdb.DB); !ok {
return nil
} else {
return db
}
}
func (n *nutsNode) setDb(db *nutsdb.DB) {
n.m.Lock()
defer n.m.Unlock()
if n.d == nil {
n.d = new(atomic.Value)
}
n.d.Store(db)
}
func (n *nutsNode) getOptions() Options {
n.m.Lock()
defer n.m.Unlock()
if n.o == nil {
return nil
} else if i := n.o.Load(); i == nil {
return nil
} else if opt, ok := i.(Options); !ok {
return nil
} else {
return opt
}
}

210
nutsdb/options.go
View File

@@ -1,210 +0,0 @@
//go:build !386 && !arm && !mips && !mipsle
// +build !386,!arm,!mips,!mipsle
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package nutsdb
import (
"bytes"
"fmt"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"
liberr "github.com/nabbar/golib/errors"
nutsdb "github.com/nutsdb/nutsdb"
xujufs "github.com/xujiajun/utils/filesystem"
)
type Options interface {
NutsDBOptions() nutsdb.Options
NewBackup(db *nutsdb.DB) (string, liberr.Error)
NewBackupTemp(db *nutsdb.DB) (string, liberr.Error)
NewTempFolder() (string, liberr.Error)
NewTempFilePattern(extension string) string
GetTempDir() string
CleanBackup() liberr.Error
Permission() os.FileMode
RestoreBackup(dir string) liberr.Error
}
func NewOptions(cfgNuts NutsDBOptions, cfgFs NutsDBFolder) (Options, liberr.Error) {
if _, err := cfgFs.GetDirectoryBase(); err != nil {
return nil, err
}
o := &options{
limit: cfgFs.LimitNumberBackup,
perm: cfgFs.Permission,
}
if fs, err := cfgFs.GetDirectoryData(); err != nil {
return nil, err
} else {
o.dirs.data = fs
}
if fs, err := cfgFs.GetDirectoryBackup(); err != nil {
return nil, err
} else {
o.dirs.backup = fs
}
if fs, err := cfgFs.GetDirectoryTemp(); err != nil {
return nil, err
} else {
o.dirs.temp = fs
}
o.nuts = cfgNuts.GetNutsDBOptions(o.dirs.data)
return o, nil
}
type options struct {
nuts nutsdb.Options
dirs struct {
data string
backup string
temp string
}
limit uint8
perm os.FileMode
}
func (o options) NutsDBOptions() nutsdb.Options {
return o.nuts
}
func (o options) NewBackup(db *nutsdb.DB) (string, liberr.Error) {
fld := o.getBackupDirName()
if e := os.MkdirAll(filepath.Join(o.dirs.backup, fld), o.perm); e != nil {
return "", ErrorFolderCreate.Error(e)
} else if err := o.newBackupDir(fld, db); err != nil {
return "", err
} else {
return fld, nil
}
}
func (o options) NewBackupTemp(db *nutsdb.DB) (string, liberr.Error) {
if fld, err := o.NewTempFolder(); err != nil {
return "", err
} else if err = o.newBackupDir(fld, db); err != nil {
return "", err
} else {
return fld, nil
}
}
func (o options) NewTempFolder() (string, liberr.Error) {
if p, e := os.MkdirTemp(o.dirs.temp, o.getTempPrefix()); e != nil {
return "", ErrorFolderCreate.Error(e)
} else {
_ = os.Chmod(p, o.perm)
return p, nil
}
}
func (o options) NewTempFilePattern(extension string) string {
pattern := o.getTempPrefix() + "-*"
if extension != "" {
pattern = pattern + "." + extension
}
return pattern
}
func (o options) GetTempDir() string {
return o.dirs.temp
}
func (o options) CleanBackup() liberr.Error {
panic("implement me")
}
func (o options) Permission() os.FileMode {
return o.perm
}
func (o options) RestoreBackup(dir string) liberr.Error {
if err := os.RemoveAll(o.dirs.data); err != nil {
return ErrorFolderDelete.Error(err)
} else if err = xujufs.CopyDir(dir, o.dirs.data); err != nil {
return ErrorFolderCopy.Error(err)
} else {
_ = os.Chmod(o.dirs.data, o.perm)
}
return nil
}
/// private
func (o options) newBackupDir(dir string, db *nutsdb.DB) liberr.Error {
if err := db.Backup(dir); err != nil {
return ErrorDatabaseBackup.Error(err)
}
return nil
}
func (o options) getTempPrefix() string {
b := make([]byte, 64)
b = b[:runtime.Stack(b, false)]
b = bytes.TrimPrefix(b, []byte("goroutine "))
b = b[:bytes.IndexByte(b, ' ')]
//nolint #nosec
/* #nosec */
n, _ := strconv.ParseUint(string(b), 10, 64)
return fmt.Sprintf("%d", n)
}
func (o options) getBackupDirName() string {
part := strings.Split(time.Now().Format(time.RFC3339), "T")
dt := part[0]
part = strings.Split(part[1], "Z")
tm := strings.Replace(part[0], ":", "-", -1)
tz := strings.Replace(part[1], ":", "", -1)
return dt + "T" + tm + "Z" + tz
}

152
nutsdb/shell.go
View File

@@ -1,152 +0,0 @@
//go:build !386 && !arm && !mips && !mipsle
// +build !386,!arm,!mips,!mipsle
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package nutsdb
import (
"encoding/json"
"fmt"
"io"
shlcmd "github.com/nabbar/golib/shell/command"
"github.com/nutsdb/nutsdb"
)
type shellCommand struct {
n CmdCode
c func() Client
}
func newShellCommand(code CmdCode, cli func() Client) shlcmd.Command {
if code == CmdUnknown {
return nil
}
return &shellCommand{
c: cli,
n: code,
}
}
func (s *shellCommand) Name() string {
return s.n.Name()
}
func (s *shellCommand) Describe() string {
return s.n.Desc()
}
func (s *shellCommand) Run(buf io.Writer, err io.Writer, args []string) {
if s.n == CmdUnknown {
_, _ = fmt.Fprintf(err, "error: %v\n", ErrorClientCommandInvalid.Error(nil).GetError())
}
var cli Client
if s.c == nil {
_, _ = fmt.Fprintf(err, "error: %v\n", ErrorClientCommandCommit.Error(nil).GetError())
} else if cli = s.c(); cli == nil {
_, _ = fmt.Fprintf(err, "error: %v\n", ErrorClientCommandCommit.Error(nil).GetError())
} else if r, e := cli.Run(s.n, args); e != nil {
_, _ = fmt.Fprintf(err, "error: %v\n", e)
} else if len(r.Value) < 1 {
_, _ = fmt.Fprintf(buf, "No result.\n")
} else {
for i, val := range r.Value {
if val == nil {
continue
}
s.parse(buf, err, r.Value[i])
}
}
}
func (s *shellCommand) json(buf, err io.Writer, val interface{}) {
if p, e := json.MarshalIndent(val, "", " "); e != nil {
_, _ = fmt.Fprintf(err, "error: %v\n", e)
} else {
_, _ = buf.Write(p)
}
}
func (s *shellCommand) parse(buf, err io.Writer, val interface{}) {
if values, ok := val.(*nutsdb.Entries); ok {
for _, v := range *values {
_, _ = fmt.Fprintf(buf, "Key: %s\n", string(v.Key))
_, _ = fmt.Fprintf(buf, "Val: %s\n", string(v.Value))
_, _ = fmt.Fprintf(buf, "\n")
}
return
}
if values, ok := val.(nutsdb.Entries); ok {
for _, v := range values {
_, _ = fmt.Fprintf(buf, "Key: %s\n", string(v.Key))
_, _ = fmt.Fprintf(buf, "Val: %s\n", string(v.Value))
_, _ = fmt.Fprintf(buf, "\n")
}
return
}
if values, ok := val.(*nutsdb.Entry); ok {
_, _ = fmt.Fprintf(buf, "Key: %s\n", string(values.Key))
_, _ = fmt.Fprintf(buf, "Val: %s\n", string(values.Value))
_, _ = fmt.Fprintf(buf, "\n")
return
}
if values, ok := val.(nutsdb.Entry); ok {
_, _ = fmt.Fprintf(buf, "Key: %s\n", string(values.Key))
_, _ = fmt.Fprintf(buf, "Val: %s\n", string(values.Value))
_, _ = fmt.Fprintf(buf, "\n")
return
}
if values, ok := val.(map[string]*nutsdb.SortedSetMember); ok {
for _, v := range values {
_, _ = fmt.Fprintf(buf, "Val: %s\n", string(v.Value))
_, _ = fmt.Fprintf(buf, "Score: %v\n", v.Score)
_, _ = fmt.Fprintf(buf, "\n")
}
return
}
if values, ok := val.([]*nutsdb.SortedSetMember); ok {
for _, v := range values {
_, _ = fmt.Fprintf(buf, "Val: %s\n", string(v.Value))
_, _ = fmt.Fprintf(buf, "Score: %v\n", v.Score)
_, _ = fmt.Fprintf(buf, "\n")
}
return
}
s.json(buf, err, val)
return
}

148
nutsdb/snap.go
View File

@@ -1,148 +0,0 @@
//go:build !386 && !arm && !mips && !mipsle
// +build !386,!arm,!mips,!mipsle
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package nutsdb
import (
"io"
"os"
"strings"
arcarc "github.com/nabbar/golib/archive/archive"
arctps "github.com/nabbar/golib/archive/archive/types"
libarc "github.com/nabbar/golib/archive"
arccmp "github.com/nabbar/golib/archive/compress"
liberr "github.com/nabbar/golib/errors"
"github.com/nutsdb/nutsdb"
)
type Snapshot interface {
Prepare(opt Options, db *nutsdb.DB) liberr.Error
Save(opt Options, writer io.Writer) liberr.Error
Load(opt Options, reader io.Reader) liberr.Error
Apply(opt Options) liberr.Error
Finish()
}
func newSnap() Snapshot {
return &snap{}
}
func snapCast(i interface{}) (Snapshot, bool) {
if sp, ok := i.(*snap); ok {
return sp, true
} else if ss, ok := i.(snap); ok {
return &ss, true
} else if si, ok := i.(Snapshot); ok {
return si, true
} else {
return nil, false
}
}
type snap struct {
path string
}
func (s *snap) Prepare(opt Options, db *nutsdb.DB) liberr.Error {
if dir, err := opt.NewBackupTemp(db); err != nil {
return err
} else {
s.path = dir
}
return nil
}
func (s *snap) Save(opt Options, writer io.Writer) liberr.Error {
var (
e error
g io.WriteCloser
t arctps.Writer
f = func(str string) string {
return strings.TrimPrefix(str, s.path)
}
)
defer func() {
if t != nil {
_ = t.Close()
}
if g != nil {
_ = g.Close()
}
}()
if g, e = arccmp.Gzip.Writer(libarc.NopWriteCloser(writer)); e != nil {
return ErrorDatabaseSnapshot.Error(e)
} else if t, e = arcarc.Tar.Writer(g); e != nil {
return ErrorDatabaseSnapshot.Error(e)
} else if e = t.FromPath(s.path, "", f); e != nil {
return ErrorDatabaseSnapshot.Error(e)
}
return nil
}
func (s *snap) Load(opt Options, reader io.Reader) liberr.Error {
var (
e error
d string
)
defer func() {
}()
if d, e = opt.NewTempFolder(); e != nil {
return ErrorDatabaseSnapshot.Error(e)
} else if e = libarc.ExtractAll(io.NopCloser(reader), "unknown", d); e != nil {
return ErrorDatabaseSnapshot.Error(e)
} else {
s.path = d
}
return nil
}
func (s *snap) Apply(opt Options) liberr.Error {
if e := opt.RestoreBackup(s.path); e != nil {
return ErrorDatabaseSnapshot.Error(e)
}
return nil
}
func (s *snap) Finish() {
_ = os.RemoveAll(s.path)
}

2
static/index.go
View File

@@ -27,7 +27,7 @@
package static
import (
"golang.org/x/exp/slices"
"slices"
)
func (s *staticHandler) SetIndex(group, route, pathFile string) {

2
status/listmandatory/model.go
View File

@@ -27,13 +27,13 @@
package listmandatory
import (
"slices"
"sort"
"sync"
"sync/atomic"
stsctr "github.com/nabbar/golib/status/control"
stsmdt "github.com/nabbar/golib/status/mandatory"
"golang.org/x/exp/slices"
)
type model struct {

2
status/mandatory/model.go
View File

@@ -27,10 +27,10 @@
package mandatory
import (
"slices"
"sync/atomic"
stsctr "github.com/nabbar/golib/status/control"
"golang.org/x/exp/slices"
)
type model struct {

2
status/model.go
View File

@@ -27,6 +27,7 @@
package status
import (
"slices"
"sync"
"time"
@@ -35,7 +36,6 @@ import (
monsts "github.com/nabbar/golib/monitor/status"
montps "github.com/nabbar/golib/monitor/types"
stsctr "github.com/nabbar/golib/status/control"
"golang.org/x/exp/slices"
)
type fctGetName func() string

422
test/test-nutsdb/main.go
View File

@@ -1,422 +0,0 @@
//go:build examples
// +build examples
/***********************************************************************************************************************
*
* MIT License
*
* Copyright (c) 2021 Nicolas JUHEL
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
**********************************************************************************************************************/
package main
import (
"bytes"
"context"
"fmt"
"os"
"runtime"
"strings"
"sync/atomic"
"time"
logcfg "github.com/nabbar/golib/logger/config"
loglvl "github.com/nabbar/golib/logger/level"
semtps "github.com/nabbar/golib/semaphore/types"
"github.com/c-bata/go-prompt"
"github.com/c-bata/go-prompt/completer"
libclu "github.com/nabbar/golib/cluster"
liberr "github.com/nabbar/golib/errors"
liblog "github.com/nabbar/golib/logger"
libndb "github.com/nabbar/golib/nutsdb"
libpwd "github.com/nabbar/golib/password"
libsem "github.com/nabbar/golib/semaphore"
libsh "github.com/nabbar/golib/shell"
libvrs "github.com/nabbar/golib/version"
"github.com/nutsdb/nutsdb"
"github.com/vbauerster/mpb/v8"
)
const (
BaseDirPattern = "/nutsdb/node-%d"
NbInstances = 3
NbEntries = 1000
LoggerFile = "/nutsdb/nutsdb.log"
AllowPut = false
AllowGet = true
)
var (
bg = new(atomic.Value)
bp = new(atomic.Value)
log = new(atomic.Value)
ctx context.Context
cnl context.CancelFunc
)
func init() {
ctx, cnl = context.WithCancel(context.Background())
liberr.SetModeReturnError(liberr.ErrorReturnCodeErrorTraceFull)
log.Store(liblog.New(func() context.Context {
return ctx
}))
initLogger()
}
type EmptyStruct struct{}
func main() {
defer func() {
if cnl != nil {
cnl()
}
}()
println(fmt.Sprintf("Running test with %d threads...", runtime.GOMAXPROCS(0)))
println(fmt.Sprintf("Init cluster..."))
tStart := time.Now()
cluster := Start(ctx)
getLogger().SetLevel(loglvl.WarnLevel)
defer func() {
Stop(ctx, cluster)
}()
if AllowPut || AllowGet {
println(strings.Join(Inject(ctx, tStart, cluster), "\n"))
os.Exit(0)
}
vs := libvrs.NewVersion(libvrs.License_MIT, "Test Raft DB Nuts", "Tools to test a raft cluster of nutsDB", time.Now().Format(time.RFC3339), "0000000", "v0.0.0", "No one", "pfx", EmptyStruct{}, 0)
vs.PrintLicense()
vs.PrintInfo()
_, _ = fmt.Fprintf(os.Stdout, "Please use `exit` or `Ctrl-D` to exit this program.\n")
sh := libsh.New()
sh.Add("", cluster[0].ShellCommand(func() context.Context {
return ctx
}, 200*time.Millisecond)...)
sh.RunPrompt(os.Stdout, os.Stderr,
prompt.OptionTitle(fmt.Sprintf("%s: %s", vs.GetPackage(), vs.GetDescription())),
prompt.OptionPrefix(">>> "),
prompt.OptionInputTextColor(prompt.Yellow),
prompt.OptionCompletionWordSeparator(completer.FilePathCompletionSeparator),
)
}
func Inject(ctx context.Context, tStart time.Time, cluster []libndb.NutsDB) []string {
getLogger().SetLevel(loglvl.WarnLevel)
tInit := time.Since(tStart)
mInit := fmt.Sprintf("Memory used after Init: \n%s", strings.Join(GetMemUsage(), "\n"))
runtime.GC()
println(fmt.Sprintf("Init done. \n"))
pgb := libsem.New(ctx, 0, true, mpb.WithWidth(64), mpb.WithRefreshRate(200*time.Millisecond))
barPut := pgb.BarNumber("PutEntry", "", int64(NbEntries), true, nil)
defer barPut.DeferMain()
tStart = time.Now()
for i := 0; i < NbEntries; i++ {
if e := barPut.NewWorker(); e != nil {
continue
}
go func(ctx context.Context, bar semtps.SemBar, clu libndb.NutsDB, num int) {
defer bar.DeferWorker()
if AllowPut {
Put(ctx, clu, fmt.Sprintf("key-%03d", num), fmt.Sprintf("val-%03d|%s|%s|%s", num, libpwd.Generate(50), libpwd.Generate(50), libpwd.Generate(50)))
}
}(ctx, barPut, cluster[i%NbInstances], i+1)
}
if e := barPut.WaitAll(); e != nil {
panic(e)
}
tPut := time.Since(tStart)
mPut := fmt.Sprintf("Memory used after Put entries: \n%s", strings.Join(GetMemUsage(), "\n"))
runtime.GC()
barGet := pgb.BarNumber("GetEntry", "", int64(NbEntries), true, nil)
defer barGet.DeferMain()
tStart = time.Now()
for i := 0; i < NbEntries; i++ {
if e := barGet.NewWorker(); e != nil {
continue
}
c := i%NbInstances + 1
if c == NbInstances {
c = 0
}
go func(ctx context.Context, bar semtps.SemBar, clu libndb.NutsDB, num int) {
defer bar.DeferWorker()
if AllowGet {
Get(ctx, clu, fmt.Sprintf("key-%03d", num), fmt.Sprintf("val-%03d", num))
}
}(ctx, barGet, cluster[c], i+1)
}
if e := barGet.WaitAll(); e != nil {
panic(e)
}
tGet := time.Since(tStart)
mGet := fmt.Sprintf("Memory used after Get entries: \n%s", strings.Join(GetMemUsage(), "\n"))
runtime.GC()
pgb = nil
res := []string{
fmt.Sprintf("Time for init cluster: %s", tInit.String()),
fmt.Sprintf("Time for %d Put in DB: %s ( %s by entry )", NbEntries, tPut.String(), (tPut / NbEntries).String()),
fmt.Sprintf("Time for %d Get in DB: %s ( %s by entry )", NbEntries, tGet.String(), (tGet / NbEntries).String()),
mInit,
mPut,
mGet,
}
runtime.GC()
getLogger().SetLevel(loglvl.InfoLevel)
getLogger().Info("Results testing: \n%s", nil, strings.Join(res, "\n"))
return res
}
func GetMemUsage() []string {
var m runtime.MemStats
runtime.ReadMemStats(&m)
return []string{
fmt.Sprintf("\t - Alloc = %v MiB", m.Alloc/1024/1024),
fmt.Sprintf("\t - TotalAlloc = %v MiB", m.TotalAlloc/1024/1024),
fmt.Sprintf("\t - Sys = %v MiB", m.Sys/1024/1024),
fmt.Sprintf("\t - NumGC = %v\n", m.NumGC),
}
}
func Put(ctx context.Context, c libndb.NutsDB, key, val string) {
_ = c.Client(ctx, 100*time.Microsecond).Put("myBucket", []byte(key), []byte(val), 0)
//res := c.Client(ctx, 100*time.Microsecond).Put("myBucket", []byte(key), []byte(val), 0)
//fmt.Printf("Cmd Put(%s, %s) : %v\n", key, val, res)
}
func Get(ctx context.Context, c libndb.NutsDB, key, val string) {
//_, _ = c.Client(ctx, 100*time.Microsecond).Get("myBucket", []byte(key))
v, e := c.Client(ctx, 100*time.Microsecond).Get("myBucket", []byte(key))
if e != nil {
getLogger().Error("Cmd Get for key '%s', error : %v", nil, key, e)
fmt.Printf("Cmd Get for key '%s', error : %v", key, e)
} else if !bytes.HasPrefix(v.Value, []byte(val)) {
getLogger().Error("Cmd Get for key '%s', awaiting value start with '%s', but find : %s", nil, key, val, string(v.Value))
fmt.Printf("Cmd Get for key '%s', awaiting value start with '%s', but find : %s", key, val, string(v.Value))
}
}
func Start(ctx context.Context) []libndb.NutsDB {
var clusters = make([]libndb.NutsDB, NbInstances)
for i := 0; i < NbInstances; i++ {
clusters[i] = initNutDB(i + 1)
clusters[i].SetLogger(func() liblog.Logger {
l := getLogger()
l.SetFields(l.GetFields().Add("lib", libndb.LogLib).Add("instance", i))
return l
})
getLogger().Info("Starting node ID #%d...", nil, i+1)
if err := clusters[i].Listen(); err != nil {
panic(err)
}
time.Sleep(5 * time.Second)
}
return clusters
}
func Stop(ctx context.Context, clusters []libndb.NutsDB) {
for i := 0; i < NbInstances; i++ {
getLogger().Info("Stopping node ID #%d...", nil, i+1)
if err := clusters[i].Shutdown(); err != nil {
panic(err)
}
time.Sleep(5 * time.Second)
}
}
func initNutDB(num int) libndb.NutsDB {
cfg := configNutDB()
cfg.Cluster.Cluster.NodeID = uint64(num)
cfg.Cluster.Node.RaftAddress = cfg.Cluster.InitMember[uint64(num)]
cfg.Directory.Base = fmt.Sprintf(BaseDirPattern, num)
return libndb.New(cfg)
}
func configNutDB() libndb.Config {
cfg := libndb.Config{
DB: libndb.NutsDBOptions{
EntryIdxMode: nutsdb.HintKeyAndRAMIdxMode,
RWMode: nutsdb.FileIO,
SegmentSize: 64 * 1024,
SyncEnable: true,
},
Cluster: libclu.Config{
Node: libclu.ConfigNode{
DeploymentID: 0,
WALDir: "",
NodeHostDir: "",
RTTMillisecond: 200,
RaftAddress: "",
AddressByNodeHostID: false,
ListenAddress: "",
MutualTLS: false,
CAFile: "",
CertFile: "",
KeyFile: "",
EnableMetrics: true,
MaxSendQueueSize: 0,
MaxReceiveQueueSize: 0,
MaxSnapshotSendBytesPerSecond: 0,
MaxSnapshotRecvBytesPerSecond: 0,
NotifyCommit: false,
Gossip: libclu.ConfigGossip{
BindAddress: "",
AdvertiseAddress: "",
Seed: nil,
},
Expert: libclu.ConfigExpert{
Engine: libclu.ConfigEngine{
ExecShards: 0,
CommitShards: 0,
ApplyShards: 0,
SnapshotShards: 0,
CloseShards: 0,
},
TestNodeHostID: 0,
TestGossipProbeInterval: 0,
},
},
Cluster: libclu.ConfigCluster{
NodeID: 0,
ClusterID: 1,
CheckQuorum: true,
ElectionRTT: 15,
HeartbeatRTT: 1,
SnapshotEntries: 10,
CompactionOverhead: 0,
OrderedConfigChange: false,
MaxInMemLogSize: 0,
SnapshotCompressionType: 0,
EntryCompressionType: 0,
DisableAutoCompactions: true,
IsObserver: false,
IsWitness: false,
Quiesce: false,
},
InitMember: map[uint64]string{
1: "localhost:9001",
2: "localhost:9002",
3: "localhost:9003",
4: "localhost:9004",
},
},
Directory: libndb.NutsDBFolder{
Base: "",
Data: "data",
Backup: "backup",
Temp: "temp",
LimitNumberBackup: 5,
Permission: 0770,
},
}
return cfg
}
func getLogger() liblog.Logger {
if log == nil {
return liblog.New(context.Background)
} else if i := log.Load(); i == nil {
return liblog.New(context.Background)
} else if l, ok := i.(liblog.Logger); !ok {
return liblog.New(context.Background)
} else {
return l
}
}
func getLoggerDgb() liblog.Logger {
l := getLogger()
l.SetFields(l.GetFields().Add("lib", libclu.LogLib))
return l
}
func setLogger(l liblog.Logger) {
if log == nil {
log = new(atomic.Value)
}
log.Store(l)
}
func initLogger() {
l := getLogger()
l.SetLevel(loglvl.InfoLevel)
if err := l.SetOptions(&logcfg.Options{
InheritDefault: false,
TraceFilter: "",
Stdout: nil,
LogFileExtend: false,
LogFile: []logcfg.OptionsFile{
{
LogLevel: []string{
"panic",
"fatal",
"error",
"warning",
"info",
"debug",
},
Filepath: LoggerFile,
Create: true,
CreatePath: true,
FileMode: 0644,
PathMode: 0755,
DisableStack: false,
DisableTimestamp: false,
EnableTrace: true,
},
},
LogSyslogExtend: false,
LogSyslog: nil,
}); err != nil {
panic(err)
}
setLogger(l)
libclu.SetLoggerFactory(getLoggerDgb)
}