apiVersion: v1
kind: Pod
spec:
  containers:
  - name: busybox
    image: busybox
    resources:
      limits:
        cpu: 500m
        memory: "400Mi"
      requests:
        cpu: 250m
        memory: "300Mi"
    command: ["md5sum"]
    args: ["/dev/urandom"]

$ k describe node <node>
...
Capacity:
  cpu:                          48
  mem-hard-eviction-threshold:  500Mi
  mem-soft-eviction-threshold:  1536Mi
  memory:                       263192560Ki
  pods:                         256
Allocatable:
  cpu:                 46
  memory:              258486256Ki
  pods:                256
Allocated resources:
  (Total limits may be over 100 percent, i.e., overcommitted.)
  Resource            Requests     Limits
  --------            --------     ------
  cpu                 800m (1%)    7200m (15%)
  memory              1000Mi (0%)  7324Mi (2%)
  hugepages-1Gi       0 (0%)       0 (0%)
...

$ cat /etc/kubernetes/kubelet/config
...
systemReserved:
  cpu: "2"      # 这就是为什么上面 describe node 输出中， Allocatable 比 Capacity 少 2 个 CPU 的原因，
  memory: "4Gi" # 以及少 4GB 内存

$ mount | grep cgroup
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/net_cls type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls)

apiVersion: v1
kind: Pod
spec:
  containers:
  - name: busybox
    image: busybox
    resources:
      limits:
        cpu: 500m
        memory: "400Mi"
      requests:
        cpu: 250m
        memory: "300Mi"
    command: ["md5sum"]
    args: ["/dev/urandom"]

// pkg/kubelet/apis/config/types.go

// Enable QoS based Cgroup hierarchy: top level cgroups for QoS Classes
// And all Burstable and BestEffort pods are brought up under their specific top level QoS cgroup.
CgroupsPerQOS bool

# 计算 pod 的 CPU request，通过 cpu.shares 能实现最小值控制
pod<pod_id>/cpu.shares            = sum(pod.spec.containers.resources.requests[cpu])
# 计算 pod 的 CPU limit，通过 cpu.cfs_quota_us 能实现最大值控制
pod<pod_id>/cpu.cfs_quota_us      = sum(pod.spec.containers.resources.limits[cpu])

# 计算 pod 的 Memory limit
pod<pod_id>/memory.limit_in_bytes = sum(pod.spec.containers.resources.limits[memory])

/burstable/cpu.shares            = max(sum(Burstable pods cpu requests), MinShares) # MinShares == 2

burstableLimit                  := allocatable — qosMemoryRequests[PodQOSGuaranteed]*percentReserve/100
/burstable/memory.limit_in_bytes = burstableLimit

/besteffort/cpu.shares            = MinShares # MinShares == 2

bestEffortLimit                  := burstableLimit — qosMemoryRequests[PodQOSBurstable]*percentReserve/100
/besteffort/memory.limit_in_bytes = bestEffortLimit

containerManagerImpl.Start
  |-cm.cpuManager.Start
  |-cm.memoryManager.Start
  |
  |-cmd.setupNode(activePods)
  |  |-validateSystemRequirements(cm.mountUtil)
  |  |  |-expectedCgroups := sets.NewString("cpu", "cpuacct", "cpuset", "memory")
  |  |-cm.createNodeAllocatableCgroups()
  |  |-cm.qosContainerManager.Start(cm.GetNodeAllocatableAbsolute, activePods)
  |  |-cm.enforceNodeAllocatableCgroups()
  |  |-cm.manage system containers (not managed by kubelet) if needed
  |  |-cm.periodicTasks = append(cm.periodicTasks, func() { // manage kubelet itself if it's deploed as a container
  |  |   ensureProcessInContainerWithOOMScore
  |  |   cont := getContainer(os.Getpid()) // return the cgroup id field in "cat /proc/<pid>/cgroup" output
  |  |   cm.KubeletCgroupsName = cont      // this container's cgroup id
  |  |})
  |
  |-cm.deviceManager.Start

// https://github.com/kubernetes/kubernetes/blob/v1.26.0/pkg/kubelet/cm/container_manager_linux.go#L100

type containerManagerImpl struct {
    cadvisorInterface cadvisor.Interface
    systemContainers []*systemContainer        // External containers being managed.
    periodicTasks    []func()                  // Tasks that are run periodically
    subsystems       *CgroupSubsystems         // Holds all the mounted cgroup subsystems
    cgroupManager     CgroupManager            // Interface for cgroup management

    capacity         v1.ResourceList           // Capacity of this node.
    internalCapacity v1.ResourceList           // Capacity of this node, including internal resources.
    cgroupRoot       CgroupName                // Absolute cgroupfs path, e.g. /sys/fs/cgroup

    qosContainerManager QOSContainerManager     // Interface for QoS cgroup management
    deviceManager       devicemanager.Manager   // Interface for exporting and allocating devices reported by device plugins.
    cpuManager          cpumanager.Manager      // Interface for CPU affinity management.
    memoryManager       memorymanager.Manager   // Interface for memory affinity management.
    topologyManager     topologymanager.Manager // Interface for Topology resource co-ordination
    draManager          dra.Manager             // Interface for Dynamic Resource Allocation management.
}

func (cm *containerManagerImpl) Start(node *v1.Node, activePods ActivePodsFunc,
    sourcesReady config.SourcesReady, podStatusProvider status.PodStatusProvider,
    runtimeService internalapi.RuntimeService, localStorageCapacityIsolation bool) error {

    // Initialize CPU manager
    cm.cpuManager.Start(cpumanager.ActivePodsFunc(activePods), sourcesReady, podStatusProvider, runtimeService, containerMap)

    // Initialize memory manager
    containerMap := buildContainerMapFromRuntime(ctx, runtimeService)
    cm.memoryManager.Start(memorymanager.ActivePodsFunc(activePods), sourcesReady, podStatusProvider, runtimeService, containerMap)

    cm.nodeInfo = node

    if localStorageCapacityIsolation {
        rootfs := cm.cadvisorInterface.RootFsInfo()
        for rName, rCap := range cadvisor.EphemeralStorageCapacityFromFsInfo(rootfs) {
            cm.capacity[rName] = rCap
        }
    }

    cm.validateNodeAllocatable() // Ensure that node allocatable configuration (system/kube/eviction reserved) is valid.
    cm.setupNode(activePods) // Setup the node

    if hasEnsureStateFuncs { // Run ensure state functions every minute.
        go wait.Until(func() {
            for _, cont := range cm.systemContainers {
                cont.ensureStateFunc(cont.manager)
            }
        }, time.Minute, wait.NeverStop)
    }

    go wait.Until(func() {
        for _, task := range cm.periodicTasks { task() }
    }, 5*time.Minute, wait.NeverStop)

    cm.deviceManager.Start(devicemanager.ActivePodsFunc(activePods), sourcesReady)
}

func (cm *containerManagerImpl) setupNode(activePods ActivePodsFunc) error {
    validateSystemRequirements(cm.mountUtil)

    // Setup top level qos containers only if CgroupsPerQOS flag is true
    if cm.NodeConfig.CgroupsPerQOS {
        cm.createNodeAllocatableCgroups()
        cm.qosContainerManager.Start(cm.GetNodeAllocatableAbsolute, activePods)
    }

    // Enforce Node Allocatable (if required)
    cm.enforceNodeAllocatableCgroups()

    systemContainers := []*systemContainer{} // containers not managed by kubelet

    if cm.SystemCgroupsName != "" {
        cont := newSystemCgroups(cm.SystemCgroupsName)
        cont.ensureStateFunc = func(manager cgroups.Manager) error {
            return ensureSystemCgroups("/", manager)
        }
        systemContainers = append(systemContainers, cont)
    }

    if cm.KubeletCgroupsName != "" {
        cont := newSystemCgroups(cm.KubeletCgroupsName)
        cont.ensureStateFunc = func(_ cgroups.Manager) error {
            return ensureProcessInContainerWithOOMScore()
        }
        systemContainers = append(systemContainers, cont)
    } else {
        cm.periodicTasks = append(cm.periodicTasks, func() {
            ensureProcessInContainerWithOOMScore(os.Getpid(), cm.KubeletOOMScoreAdj)
            cont := getContainer(os.Getpid()) // return the cgroup id field in "cat /proc/<pid>/cgroup" output
            cm.KubeletCgroupsName = cont      // this container's cgroup id
        })
    }

    cm.systemContainers = systemContainers
}

// https://github.com/kubernetes/kubernetes/blob/v1.26.0/pkg/kubelet/cm/container_manager_linux.go#L161


// checks if the required cgroups subsystems are mounted.
// As of now, only 'cpu' and 'memory' are required.
// cpu quota is a soft requirement.
func validateSystemRequirements(mountUtil mount.Interface) (features, error) {
    ...
    expectedCgroups := sets.NewString("cpu", "cpuacct", "cpuset", "memory")
}

[email protected]: $ grep -i cgroup /etc/kubernetes/config
kubeletCgroups: ""
systemCgroups: ""
cgroupRoot: "" # if empty, default to "/" in the code
cgroupsPerQOS: true
cgroupDriver: cgroupfs

$ kubectl get --raw "/apis/metrics.k8s.io/v1beta1/namespaces/default/pods/smoke-pod-01" | jq -C .
{
  "kind": "PodMetrics",
  "apiVersion": "metrics.k8s.io/v1beta1",
  "metadata": {
    "name": "smoke-pod-01",
    "selfLink": "/apis/metrics.k8s.io/v1beta1/namespaces/default/pods/smoke-pod-01",
  },
  "containers": [
    {
      "name": "netperf",
      "usage": {
        "cpu": "0",
        "memory": "1808Ki"
      }
    }
  ]
}

Task stats for "/sys/fs/cgroup/cpu,cpuacct/kubepods/pod<pod_id>/<container_id>": {NrSleeping:1 NrRunning:0 NrStopped:0 NrUninterruptible:0 NrIoWait:0}

// vendor/github.com/google/cadvisor/utils/cpuload/netlink/reader.go

// Returns instantaneous number of running tasks in a group.
// * Caller can use historical data to calculate cpu load.
// * path is an absolute filesystem path for a container under the CPU cgroup hierarchy.
// NOTE: non-hierarchical load is returned. It does not include load for subcontainers.
func (r *NetlinkReader) GetCpuLoad(name string, path string) (info.LoadStats, error) {
	cfd := os.Open(path)
	defer cfd.Close()

	stats := getLoadStats(r.familyID, cfd, r.conn)
	klog.V(4).Infof("Task stats for %q: %+v", path, stats)
	return stats, nil
}

$ cat /proc/1606/cgroup
11:net_cls:    /kubepods/besteffort/pod<pod_id>/<ctn_id>
10:cpu,cpuacct:/kubepods/besteffort/pod<pod_id>/<ctn_id>
9:memory:      /kubepods/besteffort/pod<pod_id>/<ctn_id>
8:blkio:       /kubepods/besteffort/pod<pod_id>/<ctn_id>
7:devices:     /kubepods/besteffort/pod<pod_id>/<ctn_id>
6:hugetlb:     /kubepods/besteffort/pod<pod_id>/<ctn_id>
5:pids:        /kubepods/besteffort/pod<pod_id>/<ctn_id>
4:freezer:     /kubepods/besteffort/pod<pod_id>/<ctn_id>
3:cpuset:      /kubepods/besteffort/pod<pod_id>/<ctn_id>
2:perf_event:  /kubepods/besteffort/pod<pod_id>/<ctn_id>
1:name=systemd:/kubepods/besteffort/pod<pod_id>/<ctn_id>
0::/

$ cat /sys/fs/cgroup/memory/kubepods/burstable/pod<pod_id>/memory.limit_in_bytes
1073741824