mirror of
https://github.com/vishvananda/netlink.git
synced 2025-09-26 20:01:13 +08:00
rdma: support rdma metrics: resource and statistic
Signed-off-by: bingshen.wbs <bingshen.wbs@alibaba-inc.com>
This commit is contained in:
bingshen.wbs
committed by
Alessandro Boch
Alessandro Boch
parent
e9f11f77b9
commit
c4bb4f91d6
@@ -9,31 +9,41 @@ const (
|
||||
)
|
||||
|
||||
const (
|
||||
RDMA_NLDEV_CMD_GET = 1
|
||||
RDMA_NLDEV_CMD_SET = 2
|
||||
RDMA_NLDEV_CMD_NEWLINK = 3
|
||||
RDMA_NLDEV_CMD_DELLINK = 4
|
||||
RDMA_NLDEV_CMD_SYS_GET = 6
|
||||
RDMA_NLDEV_CMD_SYS_SET = 7
|
||||
RDMA_NLDEV_CMD_GET = 1
|
||||
RDMA_NLDEV_CMD_SET = 2
|
||||
RDMA_NLDEV_CMD_NEWLINK = 3
|
||||
RDMA_NLDEV_CMD_DELLINK = 4
|
||||
RDMA_NLDEV_CMD_SYS_GET = 6
|
||||
RDMA_NLDEV_CMD_SYS_SET = 7
|
||||
RDMA_NLDEV_CMD_RES_GET = 9
|
||||
RDMA_NLDEV_CMD_STAT_GET = 17
|
||||
)
|
||||
|
||||
const (
|
||||
RDMA_NLDEV_ATTR_DEV_INDEX = 1
|
||||
RDMA_NLDEV_ATTR_DEV_NAME = 2
|
||||
RDMA_NLDEV_ATTR_PORT_INDEX = 3
|
||||
RDMA_NLDEV_ATTR_CAP_FLAGS = 4
|
||||
RDMA_NLDEV_ATTR_FW_VERSION = 5
|
||||
RDMA_NLDEV_ATTR_NODE_GUID = 6
|
||||
RDMA_NLDEV_ATTR_SYS_IMAGE_GUID = 7
|
||||
RDMA_NLDEV_ATTR_SUBNET_PREFIX = 8
|
||||
RDMA_NLDEV_ATTR_LID = 9
|
||||
RDMA_NLDEV_ATTR_SM_LID = 10
|
||||
RDMA_NLDEV_ATTR_LMC = 11
|
||||
RDMA_NLDEV_ATTR_PORT_STATE = 12
|
||||
RDMA_NLDEV_ATTR_PORT_PHYS_STATE = 13
|
||||
RDMA_NLDEV_ATTR_DEV_NODE_TYPE = 14
|
||||
RDMA_NLDEV_ATTR_NDEV_NAME = 51
|
||||
RDMA_NLDEV_ATTR_LINK_TYPE = 65
|
||||
RDMA_NLDEV_SYS_ATTR_NETNS_MODE = 66
|
||||
RDMA_NLDEV_NET_NS_FD = 68
|
||||
RDMA_NLDEV_ATTR_DEV_INDEX = 1
|
||||
RDMA_NLDEV_ATTR_DEV_NAME = 2
|
||||
RDMA_NLDEV_ATTR_PORT_INDEX = 3
|
||||
RDMA_NLDEV_ATTR_CAP_FLAGS = 4
|
||||
RDMA_NLDEV_ATTR_FW_VERSION = 5
|
||||
RDMA_NLDEV_ATTR_NODE_GUID = 6
|
||||
RDMA_NLDEV_ATTR_SYS_IMAGE_GUID = 7
|
||||
RDMA_NLDEV_ATTR_SUBNET_PREFIX = 8
|
||||
RDMA_NLDEV_ATTR_LID = 9
|
||||
RDMA_NLDEV_ATTR_SM_LID = 10
|
||||
RDMA_NLDEV_ATTR_LMC = 11
|
||||
RDMA_NLDEV_ATTR_PORT_STATE = 12
|
||||
RDMA_NLDEV_ATTR_PORT_PHYS_STATE = 13
|
||||
RDMA_NLDEV_ATTR_DEV_NODE_TYPE = 14
|
||||
RDMA_NLDEV_ATTR_RES_SUMMARY = 15
|
||||
RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY = 16
|
||||
RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME = 17
|
||||
RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR = 18
|
||||
RDMA_NLDEV_ATTR_NDEV_NAME = 51
|
||||
RDMA_NLDEV_ATTR_LINK_TYPE = 65
|
||||
RDMA_NLDEV_SYS_ATTR_NETNS_MODE = 66
|
||||
RDMA_NLDEV_NET_NS_FD = 68
|
||||
RDMA_NLDEV_ATTR_STAT_HWCOUNTERS = 80
|
||||
RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY = 81
|
||||
RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME = 82
|
||||
RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE = 83
|
||||
)
|
||||
|
||||
@@ -18,6 +18,7 @@ type RdmaLinkAttrs struct {
|
||||
FirmwareVersion string
|
||||
NodeGuid string
|
||||
SysImageGuid string
|
||||
NumPorts uint32
|
||||
}
|
||||
|
||||
// Link represents a rdma device from netlink.
|
||||
@@ -69,6 +70,11 @@ func executeOneGetRdmaLink(data []byte) (*RdmaLink, error) {
|
||||
r := bytes.NewReader(value)
|
||||
binary.Read(r, nl.NativeEndian(), &sysGuid)
|
||||
link.Attrs.SysImageGuid = uint64ToGuidString(sysGuid)
|
||||
case nl.RDMA_NLDEV_ATTR_PORT_INDEX:
|
||||
var availablePort uint32
|
||||
r := bytes.NewReader(value)
|
||||
binary.Read(r, nl.NativeEndian(), &availablePort)
|
||||
link.Attrs.NumPorts = availablePort
|
||||
}
|
||||
if (len % 4) != 0 {
|
||||
// Skip pad bytes
|
||||
@@ -345,3 +351,212 @@ func (h *Handle) RdmaLinkAdd(linkName string, linkType string, netdev string) er
|
||||
_, err := req.Execute(unix.NETLINK_RDMA, 0)
|
||||
return err
|
||||
}
|
||||
|
||||
// RdmaResource represents a rdma device resource tracking summaries
|
||||
type RdmaResource struct {
|
||||
Index uint32
|
||||
Name string
|
||||
RdmaResourceSummaryEntries map[string]uint64
|
||||
}
|
||||
|
||||
// RdmaResourceList list rdma resource tracking information
|
||||
// Returns all rdma devices resource tracking summary on success or returns error
|
||||
// otherwise.
|
||||
// Equivalent to: `rdma resource'
|
||||
func RdmaResourceList() ([]*RdmaResource, error) {
|
||||
return pkgHandle.RdmaResourceList()
|
||||
}
|
||||
|
||||
// RdmaResourceList list rdma resource tracking information
|
||||
// Returns all rdma devices resource tracking summary on success or returns error
|
||||
// otherwise.
|
||||
// Equivalent to: `rdma resource'
|
||||
func (h *Handle) RdmaResourceList() ([]*RdmaResource, error) {
|
||||
proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_RES_GET)
|
||||
req := h.newNetlinkRequest(proto, unix.NLM_F_ACK|unix.NLM_F_DUMP)
|
||||
|
||||
msgs, err := req.Execute(unix.NETLINK_RDMA, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(msgs) == 0 {
|
||||
return nil, fmt.Errorf("No valid response from kernel")
|
||||
}
|
||||
var rdmaResources []*RdmaResource
|
||||
for _, msg := range msgs {
|
||||
res, err := executeOneGetRdmaResourceList(msg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rdmaResources = append(rdmaResources, res)
|
||||
}
|
||||
return rdmaResources, nil
|
||||
}
|
||||
|
||||
func parseRdmaCounters(counterType uint16, data []byte) (map[string]uint64, error) {
|
||||
var counterKeyType, counterValueType uint16
|
||||
switch counterType {
|
||||
case nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY:
|
||||
counterKeyType = nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME
|
||||
counterValueType = nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR
|
||||
case nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY:
|
||||
counterKeyType = nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_NAME
|
||||
counterValueType = nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY_VALUE
|
||||
default:
|
||||
return nil, fmt.Errorf("Invalid counter type: %d", counterType)
|
||||
}
|
||||
counters := make(map[string]uint64)
|
||||
reader := bytes.NewReader(data)
|
||||
|
||||
for reader.Len() >= 4 {
|
||||
_, attrType, _, value := parseNfAttrTLV(reader)
|
||||
if attrType != counterType {
|
||||
return nil, fmt.Errorf("Invalid resource summary entry type; %d", attrType)
|
||||
}
|
||||
|
||||
summaryReader := bytes.NewReader(value)
|
||||
for summaryReader.Len() >= 4 {
|
||||
_, attrType, len, value := parseNfAttrTLV(summaryReader)
|
||||
if attrType != counterKeyType {
|
||||
return nil, fmt.Errorf("Invalid resource summary entry name type; %d", attrType)
|
||||
}
|
||||
name := string(value[0 : len-1])
|
||||
// Skip pad bytes
|
||||
if (len % 4) != 0 {
|
||||
summaryReader.Seek(int64(4-(len%4)), seekCurrent)
|
||||
}
|
||||
_, attrType, len, value = parseNfAttrTLV(summaryReader)
|
||||
if attrType != counterValueType {
|
||||
return nil, fmt.Errorf("Invalid resource summary entry value type; %d", attrType)
|
||||
}
|
||||
counters[name] = native.Uint64(value)
|
||||
}
|
||||
}
|
||||
return counters, nil
|
||||
}
|
||||
|
||||
func executeOneGetRdmaResourceList(data []byte) (*RdmaResource, error) {
|
||||
var res RdmaResource
|
||||
reader := bytes.NewReader(data)
|
||||
for reader.Len() >= 4 {
|
||||
_, attrType, len, value := parseNfAttrTLV(reader)
|
||||
|
||||
switch attrType {
|
||||
case nl.RDMA_NLDEV_ATTR_DEV_INDEX:
|
||||
var Index uint32
|
||||
r := bytes.NewReader(value)
|
||||
binary.Read(r, nl.NativeEndian(), &Index)
|
||||
res.Index = Index
|
||||
case nl.RDMA_NLDEV_ATTR_DEV_NAME:
|
||||
res.Name = string(value[0 : len-1])
|
||||
case nl.RDMA_NLDEV_ATTR_RES_SUMMARY:
|
||||
var err error
|
||||
res.RdmaResourceSummaryEntries, err = parseRdmaCounters(nl.RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if (len % 4) != 0 {
|
||||
// Skip pad bytes
|
||||
reader.Seek(int64(4-(len%4)), seekCurrent)
|
||||
}
|
||||
}
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
// RdmaPortStatistic represents a rdma port statistic counter
|
||||
type RdmaPortStatistic struct {
|
||||
PortIndex uint32
|
||||
Statistics map[string]uint64
|
||||
}
|
||||
|
||||
// RdmaDeviceStatistic represents a rdma device statistic counter
|
||||
type RdmaDeviceStatistic struct {
|
||||
RdmaPortStatistics []*RdmaPortStatistic
|
||||
}
|
||||
|
||||
// RdmaStatistic get rdma device statistic counters
|
||||
// Returns rdma device statistic counters on success or returns error
|
||||
// otherwise.
|
||||
// Equivalent to: `rdma statistic show link [DEV]'
|
||||
func RdmaStatistic(link *RdmaLink) (*RdmaDeviceStatistic, error) {
|
||||
return pkgHandle.RdmaStatistic(link)
|
||||
}
|
||||
|
||||
// RdmaStatistic get rdma device statistic counters
|
||||
// Returns rdma device statistic counters on success or returns error
|
||||
// otherwise.
|
||||
// Equivalent to: `rdma statistic show link [DEV]'
|
||||
func (h *Handle) RdmaStatistic(link *RdmaLink) (*RdmaDeviceStatistic, error) {
|
||||
rdmaLinkStatistic := make([]*RdmaPortStatistic, 0)
|
||||
for portIndex := uint32(1); portIndex <= link.Attrs.NumPorts; portIndex++ {
|
||||
portStatistic, err := h.RdmaPortStatisticList(link, portIndex)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rdmaLinkStatistic = append(rdmaLinkStatistic, portStatistic)
|
||||
}
|
||||
return &RdmaDeviceStatistic{RdmaPortStatistics: rdmaLinkStatistic}, nil
|
||||
}
|
||||
|
||||
// RdmaPortStatisticList get rdma device port statistic counters
|
||||
// Returns rdma device port statistic counters on success or returns error
|
||||
// otherwise.
|
||||
// Equivalent to: `rdma statistic show link [DEV/PORT]'
|
||||
func RdmaPortStatisticList(link *RdmaLink, port uint32) (*RdmaPortStatistic, error) {
|
||||
return pkgHandle.RdmaPortStatisticList(link, port)
|
||||
}
|
||||
|
||||
// RdmaPortStatisticList get rdma device port statistic counters
|
||||
// Returns rdma device port statistic counters on success or returns error
|
||||
// otherwise.
|
||||
// Equivalent to: `rdma statistic show link [DEV/PORT]'
|
||||
func (h *Handle) RdmaPortStatisticList(link *RdmaLink, port uint32) (*RdmaPortStatistic, error) {
|
||||
proto := getProtoField(nl.RDMA_NL_NLDEV, nl.RDMA_NLDEV_CMD_STAT_GET)
|
||||
req := h.newNetlinkRequest(proto, unix.NLM_F_ACK|unix.NLM_F_REQUEST)
|
||||
b := make([]byte, 4)
|
||||
native.PutUint32(b, link.Attrs.Index)
|
||||
data := nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_DEV_INDEX, b)
|
||||
req.AddData(data)
|
||||
|
||||
b = make([]byte, 4)
|
||||
native.PutUint32(b, port)
|
||||
data = nl.NewRtAttr(nl.RDMA_NLDEV_ATTR_PORT_INDEX, b)
|
||||
req.AddData(data)
|
||||
|
||||
msgs, err := req.Execute(unix.NETLINK_RDMA, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(msgs) != 1 {
|
||||
return nil, fmt.Errorf("No valid response from kernel")
|
||||
}
|
||||
return executeOneGetRdmaPortStatistics(msgs[0])
|
||||
}
|
||||
|
||||
func executeOneGetRdmaPortStatistics(data []byte) (*RdmaPortStatistic, error) {
|
||||
var stat RdmaPortStatistic
|
||||
reader := bytes.NewReader(data)
|
||||
for reader.Len() >= 4 {
|
||||
_, attrType, len, value := parseNfAttrTLV(reader)
|
||||
|
||||
switch attrType {
|
||||
case nl.RDMA_NLDEV_ATTR_PORT_INDEX:
|
||||
var Index uint32
|
||||
r := bytes.NewReader(value)
|
||||
binary.Read(r, nl.NativeEndian(), &Index)
|
||||
stat.PortIndex = Index
|
||||
case nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTERS:
|
||||
var err error
|
||||
stat.Statistics, err = parseRdmaCounters(nl.RDMA_NLDEV_ATTR_STAT_HWCOUNTER_ENTRY, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if (len % 4) != 0 {
|
||||
// Skip pad bytes
|
||||
reader.Seek(int64(4-(len%4)), seekCurrent)
|
||||
}
|
||||
}
|
||||
return &stat, nil
|
||||
}
|
||||
|
||||
@@ -205,3 +205,30 @@ func TestRdmaLinkAddAndDel(t *testing.T) {
|
||||
|
||||
checkPresence(linkName, false)
|
||||
}
|
||||
|
||||
func TestRdmaLinkMetrics(t *testing.T) {
|
||||
minKernelRequired(t, 5, 1)
|
||||
setupRdmaKModule(t, "rdma_rxe")
|
||||
if err := RdmaLinkAdd(t.Name(), "rxe", "lo"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
link, err := RdmaLinkByName(t.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer RdmaLinkDel(t.Name())
|
||||
resources, err := RdmaResourceList()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, resource := range resources {
|
||||
t.Logf("resource: %+v", resource)
|
||||
}
|
||||
stats, err := RdmaStatistic(link)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, stat := range stats.RdmaPortStatistics {
|
||||
t.Logf("stat: %+v", stat)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user