【runc 源码分析】cgroup 源码分析

痛定思痛。 2022-06-10 08:50 525阅读 0赞

# **1 Manager 接口** #

###  ###

libcontainer/cgroups/cgroups.go 中接口 Manager 定义了操作 cgroup 的方法

type Manager interface {
    	// Applies cgroup configuration to the process with the specified pid
    	Apply(pid int) error
    
    	// Returns the PIDs inside the cgroup set
    	GetPids() ([]int, error)
    
    	// Returns the PIDs inside the cgroup set & all sub-cgroups
    	GetAllPids() ([]int, error)
    
    	// Returns statistics for the cgroup set
    	GetStats() (*Stats, error)
    
    	// Toggles the freezer cgroup according with specified state
    	Freeze(state configs.FreezerState) error
    
    	// Destroys the cgroup set
    	Destroy() error
    
    	// The option func SystemdCgroups() and Cgroupfs() require following attributes:
    	// 	Paths   map[string]string
    	// 	Cgroups *configs.Cgroup
    	// Paths maps cgroup subsystem to path at which it is mounted.
    	// Cgroups specifies specific cgroup settings for the various subsystems
    
    	// Returns cgroup paths to save in a state file and to be able to
    	// restore the object later.
    	GetPaths() map[string]string
    
    	// Sets the cgroup as configured.
    	Set(container *configs.Config) error
    }

# **2 cgroup raw 子系统** #

###  ###

libcontainer/cgroups 中有三个文件夹 fs，rootless，systemd，主要分析 fs 目录下的子系统，其中 libcontainer/cgroups/fs/apply\_raw.go 中定义了子系统，包括 cpu，cpuset，memory，pid 等等

var (
    	subsystems = subsystemSet{
    		&CpusetGroup{},
    		&DevicesGroup{},
    		&MemoryGroup{},
    		&CpuGroup{},
    		&CpuacctGroup{},
    		&PidsGroup{},
    		&BlkioGroup{},
    		&HugetlbGroup{},
    		&NetClsGroup{},
    		&NetPrioGroup{},
    		&PerfEventGroup{},
    		&FreezerGroup{},
    		&NameGroup{GroupName: "name=systemd", Join: true},
    	}
    	HugePageSizes, _ = cgroups.GetHugePageSize()
    )

### **    2.1 raw 子系统接口** ###

###  ###

libcontainer/cgroups/fs/apply\_raw.go 中定义接口 subsystem

type subsystem interface {
    	// Name returns the name of the subsystem.
    	Name() string
    	// Returns the stats, as 'stats', corresponding to the cgroup under 'path'.
    	GetStats(path string, stats *cgroups.Stats) error
    	// Removes the cgroup represented by 'cgroupData'.
    	Remove(*cgroupData) error
    	// Creates and joins the cgroup represented by 'cgroupData'.
    	Apply(*cgroupData) error
    	// Set the cgroup represented by cgroup.
    	Set(path string, cgroup *configs.Cgroup) error
    }

## ##

### **    2.2 raw 子系统数据结构** ###

###  ###

libcontainer/cgroups/fs/apply\_raw.go 中结构体 Manager 实现了第一章节 Manager 接口，其中 configs.Cgroup 定义在 libcontainer/configs/cgroup\_linux.go 中

type Manager struct {
    	mu       sync.Mutex
    	Cgroups  *configs.Cgroup
    	Rootless bool // ignore permission-related errors
    	Paths    map[string]string
    }

Cgroup 定义在 libcontainer/configs/cgroup\_linux.go 中

type Cgroup struct {
    	// Deprecated, use Path instead
    	Name string `json:"name,omitempty"`
    
    	// name of parent of cgroup or slice
    	// Deprecated, use Path instead
    	Parent string `json:"parent,omitempty"`
    
    	// Path specifies the path to cgroups that are created and/or joined by the container.
    	// The path is assumed to be relative to the host system cgroup mountpoint.
    	Path string `json:"path"`
    
    	// ScopePrefix describes prefix for the scope name
    	ScopePrefix string `json:"scope_prefix"`
    
    	// Paths represent the absolute cgroups paths to join.
    	// This takes precedence over Path.
    	Paths map[string]string
    
    	// Resources contains various cgroups settings to apply
    	*Resources
    }

type cgroupData struct {
    	root      string
    	innerPath string
    	config    *configs.Cgroup
    	pid       int
    }

### **    2.3 raw 子系统方法** ###

cgroupRoot 定义了 cgroup 的全局根路径，getCgroupRoot 返回根路径，如果全局变量存在则返回，不存在则调用 FindCgroupMountpointDir 在 /proc/self/mountinfo 查找并验证路径是否存在

var cgroupRoot string
    
    // Gets the cgroupRoot.
    func getCgroupRoot() (string, error) {
    	cgroupRootLock.Lock()
    	defer cgroupRootLock.Unlock()
    
    	if cgroupRoot != "" {
    		return cgroupRoot, nil
    	}
    
    	root, err := cgroups.FindCgroupMountpointDir()
    	if err != nil {
    		return "", err
    	}
    
    	if _, err := os.Stat(root); err != nil {
    		return "", err
    	}
    
    	cgroupRoot = root
    	return cgroupRoot, nil
    }

# **3 Apply函數** #

libcontainer/cgroups/fs/apply\_raw.go 中方法 Apply

###     3.1 getCgroupData: 获得 cgroup 根路径封装成 cgroupData 结构体 ###

func (m *Manager) Apply(pid int) (err error) {
    	if m.Cgroups == nil {
    		return nil
    	}
    	m.mu.Lock()
    	defer m.mu.Unlock()
    
    	var c = m.Cgroups
    
    	d, err := getCgroupData(m.Cgroups, pid)
    	if err != nil {
    		return err
    	}

###     3.2 m.Paths: 将 cgroup name 和 path 加入到 map 中 ###

m.Paths = make(map[string]string)
    	if c.Paths != nil {
    		for name, path := range c.Paths {
    			_, err := d.path(name)
    			if err != nil {
    				if cgroups.IsNotFound(err) {
    					continue
    				}
    				return err
    			}
    			m.Paths[name] = path
    		}
    		return cgroups.EnterPid(m.Paths, pid)
    	}

###    3.3 对各个子系统执行 Apply 方法，各个子系统稍后分析 ###

for _, sys := range subsystems {
    		// TODO: Apply should, ideally, be reentrant or be broken up into a separate
    		// create and join phase so that the cgroup hierarchy for a container can be
    		// created then join consists of writing the process pids to cgroup.procs
    		p, err := d.path(sys.Name())
    		if err != nil {
    			// The non-presence of the devices subsystem is
    			// considered fatal for security reasons.
    			if cgroups.IsNotFound(err) && sys.Name() != "devices" {
    				continue
    			}
    			return err
    		}
    		m.Paths[sys.Name()] = p
    
    		if err := sys.Apply(d); err != nil {
    			// In the case of rootless (including euid=0 in userns), where an explicit cgroup path hasn't
    			// been set, we don't bail on error in case of permission problems.
    			// Cases where limits have been set (and we couldn't create our own
    			// cgroup) are handled by Set.
    			if isIgnorableError(m.Rootless, err) && m.Cgroups.Path == "" {
    				delete(m.Paths, sys.Name())
    				continue
    			}
    			return err
    		}
    
    	}

*  blkio -- 这个子系统为块设备设定输入/输出限制，比如物理设备（磁盘，固态硬盘，USB 等）。
 *  cpu -- 这个子系统使用调度程序提供对 CPU 的 cgroup 任务访问。
 *  cpuacct -- 这个子系统自动生成 cgroup 中任务所使用的 CPU 报告。
 *  cpuset -- 这个子系统为 cgroup 中的任务分配独立 CPU（在多核系统）和内存节点。
 *  devices -- 这个子系统可允许或者拒绝 cgroup 中的任务访问设备。
 *  freezer -- 这个子系统挂起或者恢复 cgroup 中的任务。
 *  memory -- 这个子系统设定 cgroup 中任务使用的内存限制，并自动生成由那些任务使用的内存资源报告。

# **4.  cgroup cpu 子系统** #

libcontainer/cgroups/fs/cpu.go 文件中 cpu 子系统数据机构

type CpuGroup struct {
    }

### **    4.1 Apply - cpu** ###

libcontainer/cgroups/fs/cpu.go 中 Apply 获得 cpu 的路径并创建路径，将数据以及 pid 写入到相应的文件中

func (s *CpuGroup) Apply(d *cgroupData) error {
    	// We always want to join the cpu group, to allow fair cpu scheduling
    	// on a container basis
    	path, err := d.path("cpu")
    	if err != nil && !cgroups.IsNotFound(err) {
    		return err
    	}
    	return s.ApplyDir(path, d.config, d.pid)
    }

**      4.1.1 ApplyDir將pid吸入到cpu文件中**

func (s *CpuGroup) ApplyDir(path string, cgroup *configs.Cgroup, pid int) error {
    	// This might happen if we have no cpu cgroup mounted.
    	// Just do nothing and don't fail.
    	if path == "" {
    		return nil
    	}
    	if err := os.MkdirAll(path, 0755); err != nil {
    		return err
    	}
    	// We should set the real-Time group scheduling settings before moving
    	// in the process because if the process is already in SCHED_RR mode
    	// and no RT bandwidth is set, adding it will fail.
    	if err := s.SetRtSched(path, cgroup); err != nil {
    		return err
    	}
    	// because we are not using d.join we need to place the pid into the procs file
    	// unlike the other subsystems
    	return cgroups.WriteCgroupProc(path, pid)
    }

###     4.2 GetStats - cpu ###

GetStates 主要是读取 cpu.stat 文件，获得 nr\_periods，nr\_throtted，throttled\_time 三项数据

*  nr\_periods: 总共经过的周期
 *  nr\_throtted: 受限制的周期
 *  throttled\_time: 就是总共被控制组掐掉的 cpu 使用时间

func (s *CpuGroup) GetStats(path string, stats *cgroups.Stats) error {
    	f, err := os.Open(filepath.Join(path, "cpu.stat"))
    	if err != nil {
    		if os.IsNotExist(err) {
    			return nil
    		}
    		return err
    	}
    	defer f.Close()
    
    	sc := bufio.NewScanner(f)
    	for sc.Scan() {
    		t, v, err := getCgroupParamKeyValue(sc.Text())
    		if err != nil {
    			return err
    		}
    		switch t {
    		case "nr_periods":
    			stats.CpuStats.ThrottlingData.Periods = v
    
    		case "nr_throttled":
    			stats.CpuStats.ThrottlingData.ThrottledPeriods = v
    
    		case "throttled_time":
    			stats.CpuStats.ThrottlingData.ThrottledTime = v
    		}
    	}
    	return nil
    }

# **6.  cgroup memory 子系统** #

*  cgroup.clone\_children： subsystem会读取这个配置文件，如果这个的值是1(默认是0)，子cgroup才会继承父cgroup的配置
 *  cgroup.event\_control： 监视状态变化和分组删除事件的配置文件
 *  cgroup.procs： 属于分组的进程 PID 列表。仅包括多线程进程的线程 leader 的 TID，这点与 tasks 不同
 *  memory.failcnt:  申请内存失败次数计数
 *  memory.usage\_in\_bytes: 当前已用的内存
 *  memory.limit\_in\_bytes: 当前限制的内存额度
 *  memory.max\_usage\_in\_bytes: 历史内存最大使用量
 *  memory.memsw.limit\_in\_bytes: 设置内存加上交换分区的总量，通过设置这个值，可以防止进程将交换分区用完
 *  memory.swappiness: 该文件的值默认和全局的swappiness（/proc/sys/vm/swappiness）一样，修改该文件只对当前cgroup生效，其功能和全局的swappiness一样 (有一点和全局的swappiness不同，那就是如果这个文件被设置成0，就算系统配置的有交换空间，当前cgroup也不会使用交换空间。)

# **7. 调用函数**  #

libcontainer/process\_linux.go 中 start 函数在容器启功进程设置 cgroup 各个子系统，创建文件路径并将配置与 pid 写入相应的文件

func (p *initProcess) start() error {
           ......
           // Do this before syncing with child so that no children can escape the
           // cgroup. We don't need to worry about not doing this and not being root
           // because we'd be using the rootless cgroup manager in that case.
           if err := p.manager.Apply(p.pid()); err != nil {
                  return newSystemErrorWithCause(err, "applying cgroup configuration for process")
           }
           ......
           return nil

/var/run/runc/container-bbbb/state.json 中纪录运行容器的状态信息，其中 cgroup 子系统对应的路径

>  "cgroup\_paths": \{  
>         "blkio": "/sys/fs/cgroup/blkio/user.slice/container-bbbb",  
>         "cpu": "/sys/fs/cgroup/cpu,cpuacct/user.slice/container-bbbb",  
>         "cpuacct": "/sys/fs/cgroup/cpu,cpuacct/user.slice/container-bbbb",  
>         "cpuset": "/sys/fs/cgroup/cpuset/container-bbbb",  
>         "devices": "/sys/fs/cgroup/devices/user.slice/container-bbbb",  
>         "freezer": "/sys/fs/cgroup/freezer/container-bbbb",  
>         "hugetlb": "/sys/fs/cgroup/hugetlb/container-bbbb",  
>         "memory": "/sys/fs/cgroup/memory/user.slice/container-bbbb",  
>         "name=systemd": "/sys/fs/cgroup/systemd/user.slice/user-1000.slice/session-c2.scope/container-bbbb",  
>         "net\_cls": "/sys/fs/cgroup/net\_cls,net\_prio/container-bbbb",  
>         "net\_prio": "/sys/fs/cgroup/net\_cls,net\_prio/container-bbbb",  
>         "perf\_event": "/sys/fs/cgroup/perf\_event/container-bbbb",  
>         "pids": "/sys/fs/cgroup/pids/user.slice/user-1000.slice/container-bbbb"  
>     \}