Merge pull request #2292 from Creatone/creatone/extend-intelrdt

Add RDT Memory Bandwidth Monitoring (MBM) and Cache Monitoring Technology (CMT) statistics.

events.go

@@ -163,6 +163,12 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats {
 			s.IntelRdt.MemBwSchemaRoot = is.MemBwSchemaRoot
 			s.IntelRdt.MemBwSchema = is.MemBwSchema
 		}
+		if intelrdt.IsMBMEnabled() {
+			s.IntelRdt.MBMStats = is.MBMStats
+		}
+		if intelrdt.IsCMTEnabled() {
+			s.IntelRdt.CMTStats = is.CMTStats
+		}
 	}
 
 	s.NetworkInterfaces = ls.Interfaces

libcontainer/intelrdt/cmt.go

@@ -0,0 +1,22 @@
+package intelrdt
+
+var (
+	cmtEnabled bool
+)
+
+// Check if Intel RDT/CMT is enabled.
+func IsCMTEnabled() bool {
+	return cmtEnabled
+}
+
+func getCMTNumaNodeStats(numaPath string) (*CMTNumaNodeStats, error) {
+	stats := &CMTNumaNodeStats{}
+
+	llcOccupancy, err := getIntelRdtParamUint(numaPath, "llc_occupancy")
+	if err != nil {
+		return nil, err
+	}
+	stats.LLCOccupancy = llcOccupancy
+
+	return stats, nil
+}

libcontainer/intelrdt/cmt_test.go

@@ -0,0 +1,56 @@
+package intelrdt
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestGetCMTNumaNodeStats(t *testing.T) {
+	mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"}
+
+	mocksFilesToCreate := map[string]uint64{
+		"llc_occupancy": 9123911,
+	}
+
+	mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate)
+
+	defer func() {
+		err := os.RemoveAll(mockedL3_MON)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("Gather mbm", func(t *testing.T) {
+		enabledMonFeatures.llcOccupancy = true
+
+		stats := make([]CMTNumaNodeStats, 0, len(mocksNUMANodesToCreate))
+		for _, numa := range mocksNUMANodesToCreate {
+			other, err := getCMTNumaNodeStats(filepath.Join(mockedL3_MON, "mon_data", numa))
+			if err != nil {
+				t.Fatal(err)
+			}
+			stats = append(stats, *other)
+		}
+
+		expectedStats := CMTNumaNodeStats{
+			LLCOccupancy: mocksFilesToCreate["llc_occupancy"],
+		}
+
+		checkCMTStatCorrection(stats[0], expectedStats, t)
+		checkCMTStatCorrection(stats[1], expectedStats, t)
+	})
+}
+
+func checkCMTStatCorrection(got CMTNumaNodeStats, expected CMTNumaNodeStats, t *testing.T) {
+	if got.LLCOccupancy != expected.LLCOccupancy {
+		t.Fatalf("Wrong value of `llc_occupancy`. Expected: %v but got: %v",
+			expected.LLCOccupancy,
+			got.LLCOccupancy)
+	}
+}

libcontainer/intelrdt/intelrdt.go

@@ -55,6 +55,10 @@ import (
  * |   |   |-- cbm_mask
  * |   |   |-- min_cbm_bits
  * |   |   |-- num_closids
+ * |   |-- L3_MON
+ * |   |   |-- max_threshold_occupancy
+ * |   |   |-- mon_features
+ * |   |   |-- num_rmids
  * |   |-- MB
  * |       |-- bandwidth_gran
  * |       |-- delay_linear
@@ -191,8 +195,7 @@ type intelRdtData struct {
 // Check if Intel RDT sub-features are enabled in init()
 func init() {
 	// 1. Check if hardware and kernel support Intel RDT sub-features
-	// "cat_l3" flag for CAT and "mba" flag for MBA
+	flagsSet, err := parseCpuInfoFile("/proc/cpuinfo")
-	isCatFlagSet, isMbaFlagSet, err := parseCpuInfoFile("/proc/cpuinfo")
 	if err != nil {
 		return
 	}
@@ -207,7 +210,7 @@ func init() {
 	// "resource control" filesystem. Intel RDT sub-features can be
 	// selectively disabled or enabled by kernel command line
 	// (e.g., rdt=!l3cat,mba) in 4.14 and newer kernel
-	if isCatFlagSet {
+	if flagsSet.CAT {
 		if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3")); err == nil {
 			isCatEnabled = true
 		}
@@ -217,11 +220,23 @@ func init() {
 		// MBA should be enabled because MBA Software Controller
 		// depends on MBA
 		isMbaEnabled = true
-	} else if isMbaFlagSet {
+	} else if flagsSet.MBA {
 		if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "MB")); err == nil {
 			isMbaEnabled = true
 		}
 	}
+
+	if flagsSet.MBMTotal || flagsSet.MBMLocal {
+		if _, err := os.Stat(filepath.Join(intelRdtRoot, "info", "L3_MON")); err == nil {
+			mbmEnabled = true
+			cmtEnabled = true
+		}
+
+		enabledMonFeatures, err = getMonFeatures(intelRdtRoot)
+		if err != nil {
+			return
+		}
+	}
 }
 
 // Return the mount point path of Intel RDT "resource control" filesysem
@@ -298,13 +313,21 @@ func isIntelRdtMounted() bool {
 	return true
 }
 
-func parseCpuInfoFile(path string) (bool, bool, error) {
+type cpuInfoFlags struct {
-	isCatFlagSet := false
+	CAT bool // Cache Allocation Technology
-	isMbaFlagSet := false
+	MBA bool // Memory Bandwidth Allocation
+
+	// Memory Bandwidth Monitoring related.
+	MBMTotal bool
+	MBMLocal bool
+}
+
+func parseCpuInfoFile(path string) (cpuInfoFlags, error) {
+	infoFlags := cpuInfoFlags{}
 
 	f, err := os.Open(path)
 	if err != nil {
-		return false, false, err
+		return infoFlags, err
 	}
 	defer f.Close()
 
@@ -319,19 +342,23 @@ func parseCpuInfoFile(path string) (bool, bool, error) {
 			for _, flag := range flags {
 				switch flag {
 				case "cat_l3":
-					isCatFlagSet = true
+					infoFlags.CAT = true
 				case "mba":
-					isMbaFlagSet = true
+					infoFlags.MBA = true
+				case "cqm_mbm_total":
+					infoFlags.MBMTotal = true
+				case "cqm_mbm_local":
+					infoFlags.MBMLocal = true
 				}
 			}
-			return isCatFlagSet, isMbaFlagSet, nil
+			return infoFlags, nil
 		}
 	}
 	if err := s.Err(); err != nil {
-		return false, false, err
+		return infoFlags, err
 	}
 
-	return isCatFlagSet, isMbaFlagSet, nil
+	return infoFlags, nil
 }
 
 func parseUint(s string, base, bitSize int) (uint64, error) {
@@ -586,7 +613,8 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) {
 	schemaRootStrings := strings.Split(tmpRootStrings, "\n")
 
 	// The L3 cache and memory bandwidth schemata in 'container_id' group
-	tmpStrings, err := getIntelRdtParamString(m.GetPath(), "schemata")
+	containerPath := m.GetPath()
+	tmpStrings, err := getIntelRdtParamString(containerPath, "schemata")
 	if err != nil {
 		return nil, err
 	}
@@ -638,6 +666,11 @@ func (m *IntelRdtManager) GetStats() (*Stats, error) {
 		}
 	}
 
+	err = getMonitoringStats(containerPath, stats)
+	if err != nil {
+		return nil, err
+	}
+
 	return stats, nil
 }
 

libcontainer/intelrdt/mbm.go

@@ -0,0 +1,34 @@
+// +build linux
+
+package intelrdt
+
+var (
+	// The flag to indicate if Intel RDT/MBM is enabled
+	mbmEnabled bool
+)
+
+// Check if Intel RDT/MBM is enabled.
+func IsMBMEnabled() bool {
+	return mbmEnabled
+}
+
+func getMBMNumaNodeStats(numaPath string) (*MBMNumaNodeStats, error) {
+	stats := &MBMNumaNodeStats{}
+	if enabledMonFeatures.mbmTotalBytes {
+		mbmTotalBytes, err := getIntelRdtParamUint(numaPath, "mbm_total_bytes")
+		if err != nil {
+			return nil, err
+		}
+		stats.MBMTotalBytes = mbmTotalBytes
+	}
+
+	if enabledMonFeatures.mbmLocalBytes {
+		mbmLocalBytes, err := getIntelRdtParamUint(numaPath, "mbm_local_bytes")
+		if err != nil {
+			return nil, err
+		}
+		stats.MBMLocalBytes = mbmLocalBytes
+	}
+
+	return stats, nil
+}

libcontainer/intelrdt/mbm_test.go

@@ -0,0 +1,68 @@
+// +build linux
+
+package intelrdt
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestGetMBMNumaNodeStats(t *testing.T) {
+	mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"}
+
+	mocksFilesToCreate := map[string]uint64{
+		"mbm_total_bytes": 9123911,
+		"mbm_local_bytes": 2361361,
+	}
+
+	mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate)
+
+	defer func() {
+		err := os.RemoveAll(mockedL3_MON)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("Gather mbm", func(t *testing.T) {
+		enabledMonFeatures.mbmTotalBytes = true
+		enabledMonFeatures.mbmLocalBytes = true
+
+		stats := make([]MBMNumaNodeStats, 0, len(mocksNUMANodesToCreate))
+		for _, numa := range mocksNUMANodesToCreate {
+			other, err := getMBMNumaNodeStats(filepath.Join(mockedL3_MON, "mon_data", numa))
+			if err != nil {
+				t.Fatal(err)
+			}
+			stats = append(stats, *other)
+		}
+
+		expectedStats := MBMNumaNodeStats{
+			MBMTotalBytes: mocksFilesToCreate["mbm_total_bytes"],
+			MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"],
+		}
+
+		checkMBMStatCorrection(stats[0], expectedStats, t)
+		checkMBMStatCorrection(stats[1], expectedStats, t)
+	})
+}
+
+func checkMBMStatCorrection(got MBMNumaNodeStats, expected MBMNumaNodeStats, t *testing.T) {
+	if got.MBMTotalBytes != expected.MBMTotalBytes {
+		t.Fatalf("Wrong value of mbm_total_bytes. Expected: %v but got: %v",
+			expected.MBMTotalBytes,
+			got.MBMTotalBytes)
+	}
+
+	if got.MBMLocalBytes != expected.MBMLocalBytes {
+		t.Fatalf("Wrong value of mbm_local_bytes. Expected: %v but got: %v",
+			expected.MBMLocalBytes,
+			got.MBMLocalBytes)
+	}
+
+}

libcontainer/intelrdt/monitoring.go

@@ -0,0 +1,85 @@
+package intelrdt
+
+import (
+	"bufio"
+	"github.com/sirupsen/logrus"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+var (
+	enabledMonFeatures monFeatures
+)
+
+type monFeatures struct {
+	mbmTotalBytes bool
+	mbmLocalBytes bool
+	llcOccupancy  bool
+}
+
+func getMonFeatures(intelRdtRoot string) (monFeatures, error) {
+	file, err := os.Open(filepath.Join(intelRdtRoot, "info", "L3_MON", "mon_features"))
+	defer file.Close()
+	if err != nil {
+		return monFeatures{}, err
+	}
+	return parseMonFeatures(file)
+}
+
+func parseMonFeatures(reader io.Reader) (monFeatures, error) {
+	scanner := bufio.NewScanner(reader)
+
+	monFeatures := monFeatures{}
+
+	for scanner.Scan() {
+		switch feature := scanner.Text(); feature {
+		case "mbm_total_bytes":
+			monFeatures.mbmTotalBytes = true
+		case "mbm_local_bytes":
+			monFeatures.mbmLocalBytes = true
+		case "llc_occupancy":
+			monFeatures.llcOccupancy = true
+		default:
+			logrus.Warnf("Unsupported Intel RDT monitoring feature: %s", feature)
+		}
+	}
+
+	return monFeatures, scanner.Err()
+}
+
+func getMonitoringStats(containerPath string, stats *Stats) error {
+	numaFiles, err := ioutil.ReadDir(filepath.Join(containerPath, "mon_data"))
+	if err != nil {
+		return err
+	}
+
+	var mbmStats []MBMNumaNodeStats
+	var cmtStats []CMTNumaNodeStats
+
+	for _, file := range numaFiles {
+		if file.IsDir() {
+			numaPath := filepath.Join(containerPath, "mon_data", file.Name())
+			if IsMBMEnabled() {
+				numaMBMStats, err := getMBMNumaNodeStats(numaPath)
+				if err != nil {
+					return err
+				}
+				mbmStats = append(mbmStats, *numaMBMStats)
+			}
+			if IsCMTEnabled() {
+				numaCMTStats, err := getCMTNumaNodeStats(numaPath)
+				if err != nil {
+					return err
+				}
+				cmtStats = append(cmtStats, *numaCMTStats)
+			}
+		}
+	}
+
+	stats.MBMStats = &mbmStats
+	stats.CMTStats = &cmtStats
+
+	return err
+}

libcontainer/intelrdt/monitoring_test.go

@@ -0,0 +1,118 @@
+package intelrdt
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestParseMonFeatures(t *testing.T) {
+	t.Run("All features available", func(t *testing.T) {
+		parsedMonFeatures, err := parseMonFeatures(
+			strings.NewReader("mbm_total_bytes\nmbm_local_bytes\nllc_occupancy"))
+		if err != nil {
+			t.Errorf("Error while parsing mon features err = %v", err)
+		}
+
+		expectedMonFeatures := monFeatures{true, true, true}
+
+		if parsedMonFeatures != expectedMonFeatures {
+			t.Error("Cannot gather all features!")
+		}
+	})
+
+	t.Run("No features available", func(t *testing.T) {
+		parsedMonFeatures, err := parseMonFeatures(strings.NewReader(""))
+
+		if err != nil {
+			t.Errorf("Error while parsing mon features err = %v", err)
+		}
+
+		expectedMonFeatures := monFeatures{false, false, false}
+
+		if parsedMonFeatures != expectedMonFeatures {
+			t.Error("Expected no features available but there is any!")
+		}
+	})
+}
+
+func mockResctrlL3_MON(NUMANodes []string, mocks map[string]uint64) (string, error) {
+	testDir, err := ioutil.TempDir("", "rdt_mbm_test")
+	if err != nil {
+		return "", err
+	}
+	monDataPath := filepath.Join(testDir, "mon_data")
+
+	for _, numa := range NUMANodes {
+		numaPath := filepath.Join(monDataPath, numa)
+		err = os.MkdirAll(numaPath, os.ModePerm)
+		if err != nil {
+			return "", err
+		}
+
+		for fileName, value := range mocks {
+			err := ioutil.WriteFile(filepath.Join(numaPath, fileName), []byte(strconv.FormatUint(value, 10)), 777)
+			if err != nil {
+				return "", err
+			}
+		}
+
+	}
+
+	return testDir, nil
+}
+
+func TestGetMonitoringStats(t *testing.T) {
+	enabledMonFeatures.mbmTotalBytes = true
+	enabledMonFeatures.mbmLocalBytes = true
+	enabledMonFeatures.llcOccupancy = true
+	mbmEnabled = true
+	cmtEnabled = true
+
+	mocksNUMANodesToCreate := []string{"mon_l3_00", "mon_l3_01"}
+
+	mocksFilesToCreate := map[string]uint64{
+		"mbm_total_bytes": 9123911,
+		"mbm_local_bytes": 2361361,
+		"llc_occupancy":   123331,
+	}
+
+	mockedL3_MON, err := mockResctrlL3_MON(mocksNUMANodesToCreate, mocksFilesToCreate)
+
+	defer func() {
+		err := os.RemoveAll(mockedL3_MON)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("Gather monitoring stats", func(t *testing.T) {
+		var stats Stats
+		err := getMonitoringStats(mockedL3_MON, &stats)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		expectedMBMStats := MBMNumaNodeStats{
+			MBMTotalBytes: mocksFilesToCreate["mbm_total_bytes"],
+			MBMLocalBytes: mocksFilesToCreate["mbm_local_bytes"],
+		}
+
+		expectedCMTStats := CMTNumaNodeStats{LLCOccupancy: mocksFilesToCreate["llc_occupancy"]}
+
+		for _, gotMBMStat := range *stats.MBMStats {
+			checkMBMStatCorrection(gotMBMStat, expectedMBMStats, t)
+		}
+
+		for _, gotCMTStat := range *stats.CMTStats {
+			checkCMTStatCorrection(gotCMTStat, expectedCMTStats, t)
+		}
+	})
+}

libcontainer/intelrdt/stats.go

@@ -15,6 +15,19 @@ type MemBwInfo struct {
 	NumClosids    uint64 `json:"num_closids,omitempty"`
 }
 
+type MBMNumaNodeStats struct {
+	// The 'mbm_total_bytes' in 'container_id' group.
+	MBMTotalBytes uint64 `json:"mbm_total_bytes,omitempty"`
+
+	// The 'mbm_local_bytes' in 'container_id' group.
+	MBMLocalBytes uint64 `json:"mbm_local_bytes,omitempty"`
+}
+
+type CMTNumaNodeStats struct {
+	// The 'llc_occupancy' in 'container_id' group.
+	LLCOccupancy uint64 `json:"llc_occupancy,omitempty"`
+}
+
 type Stats struct {
 	// The read-only L3 cache information
 	L3CacheInfo *L3CacheInfo `json:"l3_cache_info,omitempty"`
@@ -33,6 +46,12 @@ type Stats struct {
 
 	// The memory bandwidth schema in 'container_id' group
 	MemBwSchema string `json:"mem_bw_schema,omitempty"`
+
+	// The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group
+	MBMStats *[]MBMNumaNodeStats `json:"mbm_stats,omitempty"`
+
+	// The cache monitoring technology statistics from NUMA nodes in 'container_id' group
+	CMTStats *[]CMTNumaNodeStats `json:"cmt_stats,omitempty"`
 }
 
 func NewStats() *Stats {

types/events.go

@@ -1,5 +1,7 @@
 package types
 
+import "github.com/opencontainers/runc/libcontainer/intelrdt"
+
 // Event struct for encoding the event data to json.
 type Event struct {
 	Type string      `json:"type"`
@@ -115,6 +117,12 @@ type IntelRdt struct {
 
 	// The memory bandwidth schema in 'container_id' group
 	MemBwSchema string `json:"mem_bw_schema,omitempty"`
+
+	// The memory bandwidth monitoring statistics from NUMA nodes in 'container_id' group
+	MBMStats *[]intelrdt.MBMNumaNodeStats `json:"mbm_stats,omitempty"`
+
+	// The cache monitoring technology statistics from NUMA nodes in 'container_id' group
+	CMTStats *[]intelrdt.CMTNumaNodeStats `json:"cmt_stats,omitempty"`
 }
 
 type NetworkInterface struct {