同时使用cgroup1和cgroup2对系统资源进行限制

1、编辑启动项：

root@node115:~# vim.tiny /etc/default/grub
添加：GRUB_CMDLINE_LINUX="cgroup_no_v1=blkio"
root@node115:~# update-grub
root@node115:~# reboot

2、检查blkio在cgroup1上是否关闭(要重启两次)

root@node115:~# dmesg -T|grep group
[Wed Apr 27 12:03:27 2022] Command line: BOOT_IMAGE=/boot/vmlinuz-5.3.10-1-pve root=/dev/mapper/vcl-root ro cgroup_no_v1=blkio quiet
[Wed Apr 27 12:03:27 2022] Built 1 zonelists, mobility grouping on.  Total pages: 2064227
[Wed Apr 27 12:03:27 2022] Kernel command line: BOOT_IMAGE=/boot/vmlinuz-5.3.10-1-pve root=/dev/mapper/vcl-root ro cgroup_no_v1=blkio quiet
[Wed Apr 27 12:03:27 2022] Disabling io control group subsystem in v1 mounts
[Wed Apr 27 12:03:27 2022] *** VALIDATE cgroup1 ***
[Wed Apr 27 12:03:27 2022] *** VALIDATE cgroup2 ***
[Wed Apr 27 12:03:30 2022] cgroup: systemd (1) created nested cgroup for controller "memory" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.
[Wed Apr 27 12:03:30 2022] cgroup: "memory" requires setting use_hierarchy to 1 on the root
root@node115:~# reboot
root@node115:~# dmesg -T|grep group
[Tue Apr 26 09:23:21 2022] Command line: BOOT_IMAGE=/boot/vmlinuz-5.3.10-1-pve root=/dev/mapper/vcl-root ro cgroup_no_v1=blkio quiet
[Tue Apr 26 09:23:21 2022] Built 1 zonelists, mobility grouping on.  Total pages: 2064227
[Tue Apr 26 09:23:21 2022] Kernel command line: BOOT_IMAGE=/boot/vmlinuz-5.3.10-1-pve root=/dev/mapper/vcl-root ro cgroup_no_v1=blkio quiet
[Tue Apr 26 09:23:21 2022] Disabling io control group subsystem in v1 mounts
[Tue Apr 26 09:23:21 2022] *** VALIDATE cgroup1 ***
[Tue Apr 26 09:23:21 2022] *** VALIDATE cgroup2 ***

3、挂载cgroup2目录

root@node115:~# mount -t cgroup2 -o nosuid,nodev,noexec cgroup /sys/fs/cgroup/blkio/
root@node115:~# df -aT
Filesystem           Type        1K-blocks    Used Available Use% Mounted on
sysfs                sysfs               0       0         0    - /sys
proc                 proc                0       0         0    - /proc
udev                 devtmpfs      4051300       0   4051300   0% /dev
devpts               devpts              0       0         0    - /dev/pts
tmpfs                tmpfs          815176   17228    797948   3% /run
/dev/mapper/vcl-root ext4          7933384 3747364   3763316  50% /
securityfs           securityfs          0       0         0    - /sys/kernel/security
tmpfs                tmpfs         4075880   53040   4022840   2% /dev/shm
tmpfs                tmpfs            5120       0      5120   0% /run/lock
tmpfs                tmpfs         4075880       0   4075880   0% /sys/fs/cgroup
cgroup               cgroup              0       0         0    - /sys/fs/cgroup/systemd
pstore               pstore              0       0         0    - /sys/fs/pstore
cgroup               cgroup              0       0         0    - /sys/fs/cgroup/net_cls,net_prio
cgroup               cgroup              0       0         0    - /sys/fs/cgroup/memory
cgroup               cgroup              0       0         0    - /sys/fs/cgroup/rdma
cgroup               cgroup              0       0         0    - /sys/fs/cgroup/cpuset
cgroup               cgroup              0       0         0    - /sys/fs/cgroup/cpu,cpuacct
cgroup               cgroup              0       0         0    - /sys/fs/cgroup/pids
cgroup               cgroup              0       0         0    - /sys/fs/cgroup/perf_event
cgroup               cgroup              0       0         0    - /sys/fs/cgroup/freezer
cgroup               cgroup              0       0         0    - /sys/fs/cgroup/devices
cgroup               cgroup              0       0         0    - /sys/fs/cgroup/hugetlb
systemd-1            -                   -       -         -    - /proc/sys/fs/binfmt_misc
mqueue               mqueue              0       0         0    - /dev/mqueue
hugetlbfs            hugetlbfs           0       0         0    - /dev/hugepages
debugfs              debugfs             0       0         0    - /sys/kernel/debug
sunrpc               rpc_pipefs          0       0         0    - /run/rpc_pipefs
configfs             configfs            0       0         0    - /sys/kernel/config
fusectl              fusectl             0       0         0    - /sys/fs/fuse/connections
lxcfs                fuse.lxcfs          0       0         0    - /var/lib/lxcfs
/dev/fuse            fuse            30720      20     30700   1% /etc/pve
tmpfs                tmpfs          815176       0    815176   0% /run/user/0
binfmt_misc          binfmt_misc         0       0         0    - /proc/sys/fs/binfmt_misc
cgroup               cgroup2             0       0         0    - /sys/fs/cgroup/blkio

4、添加control项

root@node115:~# cd /sys/fs/cgroup/blkio
root@node115:/sys/fs/cgroup/blkio# ls -l
total 0
-r--r--r-- 1 root root 0 Apr 26 09:24 cgroup.controllers
-rw-r--r-- 1 root root 0 Apr 26 09:24 cgroup.max.depth
-rw-r--r-- 1 root root 0 Apr 26 09:24 cgroup.max.descendants
-rw-r--r-- 1 root root 0 Apr 26 09:24 cgroup.procs
-r--r--r-- 1 root root 0 Apr 26 09:24 cgroup.stat
-rw-r--r-- 1 root root 0 Apr 26 09:24 cgroup.subtree_control
-rw-r--r-- 1 root root 0 Apr 26 09:24 cgroup.threads
-rw-r--r-- 1 root root 0 Apr 26 09:24 cpu.pressure
-rw-r--r-- 1 root root 0 Apr 26 09:24 io.pressure
-rw-r--r-- 1 root root 0 Apr 26 09:24 memory.pressure
root@node115:/sys/fs/cgroup/blkio# cat cgroup.subtree_control
root@node115:/sys/fs/cgroup/blkio# echo "+io" > /sys/fs/cgroup/blkio/cgroup.subtree_control
root@node115:/sys/fs/cgroup/blkio# cat cgroup.subtree_control
io

5、限制磁盘带宽(cgroup2)

root@node115:/sys/fs/cgroup/blkio# mkdir user.slice
root@node115:/sys/fs/cgroup/blkio# cd user.slice/
root@node115:/sys/fs/cgroup/blkio/user.slice# cat io.max
root@node115:/sys/fs/cgroup/blkio/user.slice# echo "8:16 wbps=10485760" > io.max
root@node115:/sys/fs/cgroup/blkio/user.slice# echo "8:16 rbps=10485760" > io.max
root@node115:/sys/fs/cgroup/blkio/user.slice# cat io.max
8:16 rbps=10485760 wbps=10485760 riops=max wiops=max

6、测试磁盘读写带宽

root@node115:~# echo $$ > /sys/fs/cgroup/blkio/user.slice/cgroup.procs
root@node115:~# fio -filename=/mnt/sdb/testfile -direct=1 -iodepth 1 -thread -rw=read -ioengine=libaio -bs=64k -size=2G -numjobs=10 -runtime=60 -group_reporting -name=mytest
mytest: (g=0): rw=read, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
...
fio-2.16
Starting 10 threads
Jobs: 10 (f=10): [R(10)] [100.0% done] [10250KB/0KB/0KB /s] [160/0/0 iops] [eta 00m:00s]
mytest: (groupid=0, jobs=10): err= 0: pid=4225: Wed Apr 27 11:25:46 2022
  read : io=615040KB, bw=10249KB/s, iops=160, runt= 60009msec
    slat (usec): min=4, max=3142, avg=43.23, stdev=62.91
    clat (usec): min=266, max=129652, avg=62390.92, stdev=46852.06
     lat (usec): min=295, max=129675, avg=62434.15, stdev=46832.50
    clat percentiles (usec):
     |  1.00th=[  652],  5.00th=[  916], 10.00th=[ 1064], 20.00th=[ 1240],
     | 30.00th=[ 1832], 40.00th=[83456], 50.00th=[95744], 60.00th=[98816],
     | 70.00th=[98816], 80.00th=[99840], 90.00th=[100864], 95.00th=[105984],
     | 99.00th=[117248], 99.50th=[121344], 99.90th=[126464], 99.95th=[129536],
     | 99.99th=[129536]
    lat (usec) : 500=0.28%, 750=1.73%, 1000=5.44%
    lat (msec) : 2=23.45%, 4=2.23%, 10=2.39%, 20=1.65%, 50=0.40%
    lat (msec) : 100=49.81%, 250=12.61%
  cpu          : usr=0.04%, sys=0.11%, ctx=9792, majf=2, minf=179
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=9610/w=0/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
   READ: io=615040KB, aggrb=10249KB/s, minb=10249KB/s, maxb=10249KB/s, mint=60009msec, maxt=60009msec

Disk stats (read/write):
  sdb: ios=7981/2, merge=1675/0, ticks=24208/13, in_queue=10576, util=7.46%

root@node115:~# fio -filename=/mnt/sdb/testfile -direct=1 -iodepth 1 -thread -rw=write -ioengine=libaio -bs=64k -size=2G -numjobs=10 -runtime=60 -group_reporting -name=mytest
mytest: (g=0): rw=write, bs=64K-64K/64K-64K/64K-64K, ioengine=libaio, iodepth=1
...
fio-2.16
Starting 10 threads
Jobs: 10 (f=10): [W(10)] [100.0% done] [0KB/10240KB/0KB /s] [0/160/0 iops] [eta 00m:00s]
mytest: (groupid=0, jobs=10): err= 0: pid=2917: Wed Apr 27 11:23:05 2022
  write: io=616064KB, bw=10254KB/s, iops=160, runt= 60078msec
    slat (usec): min=7, max=6874, avg=56.35, stdev=77.24
    clat (usec): min=867, max=113648, avg=62344.94, stdev=46731.61
     lat (usec): min=952, max=113681, avg=62401.29, stdev=46709.69
    clat percentiles (usec):
     |  1.00th=[ 1464],  5.00th=[ 1720], 10.00th=[ 1832], 20.00th=[ 2008],
     | 30.00th=[ 2256], 40.00th=[95744], 50.00th=[97792], 60.00th=[98816],
     | 70.00th=[98816], 80.00th=[98816], 90.00th=[98816], 95.00th=[99840],
     | 99.00th=[102912], 99.50th=[104960], 99.90th=[111104], 99.95th=[112128],
     | 99.99th=[113152]
    lat (usec) : 1000=0.10%
    lat (msec) : 2=19.67%, 4=17.38%, 10=0.19%, 20=0.06%, 50=0.17%
    lat (msec) : 100=59.38%, 250=3.05%
  cpu          : usr=0.06%, sys=0.10%, ctx=9897, majf=0, minf=12
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued    : total=r=0/w=9626/d=0, short=r=0/w=0/d=0, drop=r=0/w=0/d=0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
  WRITE: io=616064KB, aggrb=10254KB/s, minb=10254KB/s, maxb=10254KB/s, mint=60078msec, maxt=60078msec

Disk stats (read/write):
  sdb: ios=72/8527, merge=0/1075, ticks=29/22506, in_queue=2244, util=8.25%

7、限制程序最大可用内存(cgroup1)

编写程序，循环分配600M内存.
root@node115:~/inode_test# vim.tiny memtest.c
#include<stdio.h>
#include<stdlib.h>
#include<string.h>

#define CHUNK_SIZE 1024 * 1024 * 100

int main()
{
    char *p;
    int i;

    for(i = 0; i < 6; i ++)
    {
        p = malloc(CHUNK_SIZE);
        if(p == NULL)
        {
            printf("malloc error!");
            return 0;
        }
        memset(p, 0, CHUNK_SIZE);
        printf("malloc memory %d MB\n", (i + 1) * 100);
        sleep(10);
    }
    while(1)
    {
        sleep(1);
    }
   return 0;
}
[root@node115 inode_test]# gcc memtest.c -o memtest
新开一个终端窗口，关闭虚拟内存：
root@node115:/sys/fs/cgroup/memory/user.slice# echo 0 > memory.swappiness
限制最大内存为200M：
root@node115:/sys/fs/cgroup/memory/user.slice# echo 209715200 > memory.limit_in_bytes
运行程序：
root@node115:~/inode_test# ./memtest
将程序的pid进程号加入cgroup的tasks中：
root@node115:/sys/fs/cgroup/memory/user.slice# echo `ps -ef | grep memtest | grep -v grep | awk '{print $2}'` > tasks
在程序运行的终端中查看进程会自动被kill掉：
root@node115:~/inode_test# ./memtest
malloc memory 100 MB
malloc memory 200 MB
Killed

可以看到cgroup2和cgroup1是同时生效的。