Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions internal/lib/statsserver/descriptors.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,16 @@ var (
Help: "Current working set in bytes.",
LabelKeys: baseLabelKeys,
}
containerMemoryTotalActiveFileBytes = &types.MetricDescriptor{
Name: "container_memory_total_active_file_bytes",
Help: "Current total active file in bytes.",
LabelKeys: baseLabelKeys,
}
containerMemoryTotalInactiveFileBytes = &types.MetricDescriptor{
Name: "container_memory_total_inactive_file_bytes",
Help: "Current total inactive file in bytes.",
LabelKeys: baseLabelKeys,
}
containerMemoryActiveAnonBytes = &types.MetricDescriptor{
Name: "container_memory_active_anon_bytes",
Help: "Current active anonymous memory in bytes.",
Expand Down
25 changes: 25 additions & 0 deletions internal/lib/statsserver/memory_metrics_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ func generateContainerMemoryMetrics(ctr *oci.Container, mem *cgroups.MemoryStats
workingSetBytes, rssBytes, pageFaults, majorPageFaults, _ := computeMemoryMetricValues(mem)
swapUsage := computeSwapUsageForMetrics(mem)
fileMapped := computeFileMapped(mem)
activeFile, inactiveFile := computeFileMemory(mem, node.CgroupIsV2())

memoryMetrics := []*containerMetric{
{
Expand Down Expand Up @@ -82,6 +83,18 @@ func generateContainerMemoryMetrics(ctr *oci.Container, mem *cgroups.MemoryStats
return metricValues{{value: workingSetBytes, metricType: types.MetricType_GAUGE}}
},
},
{
desc: containerMemoryTotalActiveFileBytes,
valueFunc: func() metricValues {
return metricValues{{value: activeFile, metricType: types.MetricType_GAUGE}}
},
},
{
desc: containerMemoryTotalInactiveFileBytes,
valueFunc: func() metricValues {
return metricValues{{value: inactiveFile, metricType: types.MetricType_GAUGE}}
},
},
{
desc: containerMemoryFailuresTotal,
valueFunc: func() metricValues {
Expand Down Expand Up @@ -241,6 +254,18 @@ func computeAnonMemory(
return memStats.Stats["total_active_anon"], memStats.Stats["total_inactive_anon"]
}

// computeFileMemory computes the active and inactive file (page cache) memory
// values. Both cgroup versions report these.
func computeFileMemory(
memStats *cgroups.MemoryStats, isCgroupV2 bool,
) (activeFile, inactiveFile uint64) {
if isCgroupV2 {
return memStats.Stats["active_file"], memStats.Stats["inactive_file"]
}

return memStats.Stats["total_active_file"], memStats.Stats["total_inactive_file"]
}

// computeTransparentHugepages computes the amount of memory backed by
// transparent hugepages, split by the kind of memory backing it. These keys
// only exist under cgroup v2; cgroup v1 has no equivalent, so all three are
Expand Down
2 changes: 2 additions & 0 deletions internal/lib/statsserver/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,8 @@ var availableMetricDescriptors = map[string][]*types.MetricDescriptor{
containerMemoryUsageBytes,
containerMemoryMaxUsageBytes,
containerMemoryWorkingSetBytes,
containerMemoryTotalActiveFileBytes,
containerMemoryTotalInactiveFileBytes,
containerMemoryFailuresTotal,
},
config.MemoryExtraMetrics: {
Expand Down
81 changes: 81 additions & 0 deletions internal/lib/statsserver/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,49 @@ func TestComputeAnonMemory(t *testing.T) {
}
}

// TestComputeFileMemory verifies that computeFileMemory reads the active and
// inactive file (page cache) keys for the given cgroup version.
func TestComputeFileMemory(t *testing.T) {
t.Parallel()

memStats := testMemoryStats()

tests := []struct {
name string
isCgroupV2 bool
wantActiveFile uint64
wantInactiveFile uint64
}{
{
name: "cgroup v2",
isCgroupV2: true,
wantActiveFile: memStats.Stats["active_file"],
wantInactiveFile: memStats.Stats["inactive_file"],
},
{
name: "cgroup v1",
isCgroupV2: false,
wantActiveFile: memStats.Stats["total_active_file"],
wantInactiveFile: memStats.Stats["total_inactive_file"],
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()

activeFile, inactiveFile := computeFileMemory(&memStats, tt.isCgroupV2)
if activeFile != tt.wantActiveFile {
t.Errorf("activeFile: got %d, want %d", activeFile, tt.wantActiveFile)
}

if inactiveFile != tt.wantInactiveFile {
t.Errorf("inactiveFile: got %d, want %d", inactiveFile, tt.wantInactiveFile)
}
})
}
}

// TestComputeTransparentHugepages verifies that computeTransparentHugepages
// reports the cgroup v2 THP counters and zero for cgroup v1, which has no THP
// accounting.
Expand Down Expand Up @@ -224,6 +267,41 @@ func TestContainerMemoryExtraMetricValues(t *testing.T) {
}
}

// TestContainerMemoryFileMetricValues verifies that
// generateContainerMemoryMetrics wires the file memory helper to the
// cAdvisor-compatible total_active/inactive_file metric names.
func TestContainerMemoryFileMetricValues(t *testing.T) {
t.Parallel()

ctr := newTestContainer(t)
memStats := testMemoryStats()

values := make(map[string]uint64)
for _, m := range generateContainerMemoryMetrics(ctr, &memStats) {
values[m.GetName()] = m.GetValue().GetValue()
}

activeFile, inactiveFile := computeFileMemory(&memStats, node.CgroupIsV2())

want := map[string]uint64{
"container_memory_total_active_file_bytes": activeFile,
"container_memory_total_inactive_file_bytes": inactiveFile,
}

for name, wantValue := range want {
got, ok := values[name]
if !ok {
t.Errorf("metric %q not generated", name)

continue
}

if got != wantValue {
t.Errorf("metric %q: got value %d, want %d", name, got, wantValue)
}
}
}

// descriptorLabelCounts builds a lookup from metric name to expected label count.
func descriptorLabelCounts() map[string]int {
m := make(map[string]int)
Expand Down Expand Up @@ -301,6 +379,9 @@ func testMemoryStats() cgroups.MemoryStats {
Stats: map[string]uint64{
"total_rss": 512 * 1024,
"total_inactive_file": 64 * 1024,
"total_active_file": 96 * 1024,
"inactive_file": 48 * 1024,
"active_file": 80 * 1024,
"total_mapped_file": 32 * 1024,
"pgfault": 100,
"pgmajfault": 5,
Expand Down
21 changes: 21 additions & 0 deletions test/cri-metrics.bats
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,25 @@ EOF
working_set_diff=$((metrics_memory_working_set - cgroup_memory_working_set))
[[ ${working_set_diff#-} -le $MEMORY_METRIC_TOLERANCE ]]

# assert container_memory_total_active_file_bytes /
# container_memory_total_inactive_file_bytes ==
# cgroup memory.stat:active_file/inactive_file (cgroup v2) or
# total_active_file/total_inactive_file (cgroup v1)
if is_cgroup_v2; then
cgroup_memory_active_file=$(grep -w active_file < "$CTR_CGROUP"/memory.stat | awk '{print $2}')
cgroup_memory_inactive_file=$(grep -w inactive_file < "$CTR_CGROUP"/memory.stat | awk '{print $2}')
else
cgroup_memory_active_file=$(grep -w total_active_file < "$CTR_CGROUP"/memory.stat | awk '{print $2}')
cgroup_memory_inactive_file=$(grep -w total_inactive_file < "$CTR_CGROUP"/memory.stat | awk '{print $2}')
fi
metrics=$(crictl metricsp)
metrics_memory_active_file=$(echo "$metrics" | jq '.podMetrics[0].containerMetrics[0].metrics[] | select(.name == "container_memory_total_active_file_bytes") | .value.value | tonumber')
metrics_memory_inactive_file=$(echo "$metrics" | jq '.podMetrics[0].containerMetrics[0].metrics[] | select(.name == "container_memory_total_inactive_file_bytes") | .value.value | tonumber')
active_file_diff=$((metrics_memory_active_file - cgroup_memory_active_file))
[[ ${active_file_diff#-} -le $MEMORY_METRIC_TOLERANCE ]]
inactive_file_diff=$((metrics_memory_inactive_file - cgroup_memory_inactive_file))
[[ ${inactive_file_diff#-} -le $MEMORY_METRIC_TOLERANCE ]]

# assert container_memory_rss == cgroup memory.stat:total_rss(cgroup v1) or memory.stat:anon(cgroup v2)
if is_cgroup_v2; then
# for cgroupv2, rss is memory.stat:anon
Expand Down Expand Up @@ -257,6 +276,8 @@ EOF
# assert the cAdvisor equivalent memory metrics are still included
grep -q "^container_memory_working_set_bytes$" <<< "$descs"
grep -q "^container_memory_usage_bytes$" <<< "$descs"
grep -q "^container_memory_total_active_file_bytes$" <<< "$descs"
grep -q "^container_memory_total_inactive_file_bytes$" <<< "$descs"

# assert the memoryExtra metrics are not included without the memoryExtra value
for metric in \
Expand Down