从kubectl top看K8S监控原理

1[root@testcce-38725 ~]# kubectl top node
2NAME           CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%
3192.168.0.78   74m          0%     934Mi           6%
4[root@testcce-38725 ~]# kubectl top node 192.168.0.78
5NAME           CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%
6192.168.0.78   74m          0%     934Mi           6%

 1[root@testcce-38725 ~]# kubectl top pod
 2NAME                     CPU(cores)   MEMORY(bytes)
 3nginx-84f87fccc4-g8h24   0m           5Mi
 4nginx-84f87fccc4-pxnb9   2m           5Mi
 5[root@testcce-38725 ~]# kubectl top pod nginx-84f87fccc4-pxnb9
 6NAME                     CPU(cores)   MEMORY(bytes)
 7nginx-84f87fccc4-pxnb9   2m           5Mi
 8[root@testcce-38725 ~]# kubectl top pod nginx-84f87fccc4-pxnb9 --containers
 9POD                      NAME          CPU(cores)   MEMORY(bytes)
10nginx-84f87fccc4-pxnb9   container-0   0m           5Mi

 1type MemoryStats struct {
 2    // Current memory usage, this includes all memory regardless of when it was
 3    // accessed.
 4    // Units: Bytes.
 5    Usage uint64 `json:"usage"`
 6
 7    // Maximum memory usage recorded.
 8    // Units: Bytes.
 9    MaxUsage uint64 `json:"max_usage"`
10
11    // Number of bytes of page cache memory.
12    // Units: Bytes.
13    Cache uint64 `json:"cache"`
14
15    // The amount of anonymous and swap cache memory (includes transparent
16    // hugepages).
17    // Units: Bytes.
18    RSS uint64 `json:"rss"`
19
20    // The amount of swap currently used by the processes in this cgroup
21    // Units: Bytes.
22    Swap uint64 `json:"swap"`
23
24    // The amount of memory used for mapped files (includes tmpfs/shmem)
25    MappedFile uint64 `json:"mapped_file"`
26
27    // The amount of working set memory, this includes recently accessed memory,
28    // dirty memory, and kernel memory. Working set is 

 1type Manager interface {
 2    // Start the manager. Calling other manager methods before this returns
 3    // may produce undefined behavior.
 4    Start() error
 5
 6    // Stops the manager.
 7    Stop() error
 8
 9    //  information about a container.
10    GetContainerInfo(containerName string, query *info.ContainerInfoRequest) (*info.ContainerInfo, error)
11
12    // Get V2 information about a container.
13    // Recursive (subcontainer) requests are best-effort, and may return a partial result alongside an
14    // error in the partial failure case.
15    GetContainerInfoV2(containerName string, options v2.RequestOptions) (map[string]v2.ContainerInfo, error)
16
17    // Get information about all subcontainers of the specified container (includes self).
18    SubcontainersInfo(containerName string, query *info.ContainerInfoRequest) ([]*info.ContainerInfo, error)
19
20    // Gets all the Docker containers. Return is a map from full container name to ContainerInfo.
21    AllDockerContainers(query *info.ContainerInfoRequest) (map[string]info.ContainerInfo, error)
22
23    // Gets information about a specific Docker container. The specified name is within the Docker namespace.
24    DockerContainer(dockerName string, query *info.ContainerInfoRequest) (info.ContainerInfo, error)
25
26    // Gets spec for all containers based on request options.
27    GetContainerSpec(containerName string, options v2.RequestOptions) (map[string]v2.ContainerSpec, error)
28
29    // Gets summary stats for all containers based on request options.
30    GetDerivedStats(containerName string, options v2.RequestOptions) (map[string]v2.DerivedStats, error)
31
32    // Get info for all requested containers based on the request options.
33    GetRequestedContainersInfo(containerName string, options v2.RequestOptions) (map[string]*info.ContainerInfo, error)
34
35    // Returns true if the named container exists.
36    Exists(containerName string) bool
37
38    // Get information about the machine.
39    GetMachineInfo() (*info.MachineInfo, error)
40……

 1memory.usage_in_bytes        # show current usage for memory
 2                 (See 5.5 for details)
 3 memory.memsw.usage_in_bytes     # show current usage for memory+Swap
 4                 (See 5.5 for details)
 5 memory.limit_in_bytes       # set/show limit of memory usage
 6 memory.memsw.limit_in_bytes     # set/show limit of memory+Swap usage
 7 memory.failcnt          # show the number of memory usage hits limits
 8 memory.memsw.failcnt        # show the number of memory+Swap hits limits
 9 memory.max_usage_in_bytes   # show max memory usage recorded
10 memory.memsw.max_usage_in_bytes # show max memory+Swap usage recorded
11 memory.soft_limit_in_bytes  # set/show soft limit of memory usage
12 memory.stat             # show various statistics

 1cache       - # of bytes of page cache memory.
 2rss     - # of bytes of anonymous and swap cache memory (includes transparent hugepages).
 3rss_huge    - # of bytes of anonymous transparent hugepages.
 4mapped_file - # of bytes of mapped file (includes tmpfs/shmem)
 5pgpgin      - # of charging events to the memory cgroup. The charging
 6        event happens each time a page is accounted as either mapped
 7        anon page(RSS) or cache page(Page Cache) to the cgroup.
 8pgpgout     - # of uncharging events to the memory cgroup. The uncharging
 9        event happens each time a page is unaccounted from the cgroup.
10swap        - # of bytes of swap usage
11dirty       - # of bytes that are waiting to get written back to the disk.
12writeback   - # of bytes of file/anon cache that are queued for syncing to disk.
13inactive_anon   - # of bytes of anonymous and swap cache memory on inactive LRU list.
14active_anon - # of bytes of anonymous and swap cache memory on active LRU list.
15inactive_file   - # of bytes of file-backed memory on inactive LRU list.
16active_file - # of bytes of file-backed memory on active LRU list.
17unevictable - # of bytes of memory that cannot be reclaimed (mlocked etc).

 1func (sp *summaryProviderImpl) GetSystemContainersCPUAndMemoryStats(nodeConfig cm.NodeConfig, podStats []statsapi.PodStats, updateStats bool) (stats []statsapi.ContainerStats) {
 2    systemContainers := map[string]struct {
 3        name             string
 4        forceStatsUpdate bool
 5        startTime        metav1.Time
 6    }{
 7        statsapi.SystemContainerKubelet: {name: nodeConfig.KubeletCgroupsName, forceStatsUpdate: false, startTime: sp.kubeletCreationTime},
 8        statsapi.SystemContainerRuntime: {name: nodeConfig.RuntimeCgroupsName, forceStatsUpdate: false},
 9        statsapi.SystemContainerMisc:    {name: nodeConfig.SystemCgroupsName, forceStatsUpdate: false},
10        statsapi.SystemContainerPods:    {name: sp.provider.GetPodCgroupRoot(), forceStatsUpdate: updateStats},
11    }
12    for sys, cont := range systemContainers {
13        // skip if cgroup name is undefined (not all system containers are required)
14        if cont.name == "" {
15            continue
16        }
17        s, err := sp.provider.GetCgroupCPUAndMemoryStats(cont.name, cont.forceStatsUpdate)
18        if err != nil {
19            klog.ErrorS(err, "Failed to get system container stats", "containerName", cont.name)
20            continue
21        }
22        s.Name = sys
23
24        // if we know the start time of a system container, use that instead of the start time provided by cAdvisor
25        if !cont.startTime.IsZero() {
26            s.StartTime = cont.startTime
27        }
28        stats = append(stats, *s)
29    }

1workingSet := ret.Memory.Usage
2if v, ok := s.MemoryStats.Stats[inactiveFileKeyName]; ok {
3    if workingSet 

 1rss + cache = (in)active_anon + (in)active_file
 2
 35.5 usage_in_bytes
 4
 5For efficiency, as other kernel components, memory cgroup uses some optimization
 6to avoid unnecessary cacheline false sharing. usage_in_bytes is affected by the
 7method and doesn't show 'exact' value of memory (and swap) usage, it's a fuzz
 8value for efficient access. (Of course, when necessary, it's synchronized.)
 9If you want to know more exact memory usage, you should use RSS+CACHE(+SWAP)
10value in memory.stat(see 5.2).

1docker stats = container_memory_usage_bytes - container_memory_cache