kata-containers Compile And Installed

kata-containers 编译安装

kata-runtime 编译安装

1 2 3 4 5 6 7 8 9 10 11 12

# download source code $ go get -d -u github.com/kata-containers/runtime $ cd ${GOPATH}/src/github.com/kata-containers/runtime # compile and install $ make $ make install # Install Dir /usr/libexec/kata-containers/kata-netmon /usr/local/bin/kata-runtime /usr/local/bin/containerd-shim-kata-v2 /usr/share/defaults/kata-containers/*

kata-shim 编译安装

1 2 3 4 5 6 7 8 9

# download source code $ go get -d -u github.com/kata-containers/shim $ cd ${GOTAH}/src/github.com/kata-containers/shim # compile and install $ makn $ make install # Install Dir /usr/libexec/kata-containers/kata-shim

kata-proxy 编译安装

1 2 3 4 5 6 7 8 9

# download source code $ go get -d -u github.com/kata-containers/proxy $ cd ${GOTAH}/src/github.com/kata-containers/proxy # compile and install $ make $ make install # Install Dir /usr/libexec/kata-containers/kata-proxy

编译 kata 所需的 kernel

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28

# download source code $ go get -d -u github.com/kata-containers/packaging $ cd ${GOTAH}/src/github.com/kata-containers/packaging/kernel # On Ubuntu20.04 should install some essential packages $ sudo apt install -y gcc make libncurses5-dev openssl libssl-dev build-essential pkg-config libc6-dev bison flex libelf-dev # Also you should install yq from github: https://github.com/mikefarah/yq # 注意：如果缺少依赖，会导致内核编译所需要的 .config 文件，无法主动生成，可以将 configs/ 和 configs/fragments 目录下对应文件拼接成完整文件。 $ ./build-kernel.sh -d setup # compile kernel $ ./build-kernel.sh -d build # Output File: ${GOPATH}/src/github.com/kata-containers/packaging/kernel/kata-linux-5.4.32-89/vmlinux # Install Dir: /usr/share/kata-containers/vmlinux

编译 agent (可选)

1 2	$ go get -d -u github.com/kata-containers/agent $ cd $GOPATH/src/github.com/kata-containers/agent && make

编译 rootfs 文件系统

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35

# download source code $ go get -d -u github.com/kata-containers/osbuilder # generate rootfs $ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/osbuilder/rootfs-builder/rootfs $ sudo rm -rf ${ROOTFS_DIR} $ cd $GOPATH/src/github.com/kata-containers/osbuilder/rootfs-builder # ${distro} 需要替换成具体的系统，推荐 centos # 此处增加额外的包，是为了后续便于进入虚拟机调试 #$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true EXTRA_PKGS="bash coreutils" ./rootfs.sh ${distro}' $ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true EXTRA_PKGS="bash coreutils vim net-tools procps curl iproute" http_proxy=http://{proxy}:{ip} https_proxy=http://{proxy}:{ip} ./rootfs.sh ${distro}' # 由于网络原因，可以构建时候添加 http_proxy 代理; $ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true EXTRA_PKGS="bash coreutils" http_proxt=http://{IP}:{PORT} ./rootfs.sh ${distro}' # Create a debug systemd service $ cat <<EOT | sudo tee ${ROOTFS_DIR}/lib/systemd/system/kata-debug.service [Unit] Description=Kata Containers debug console [Service] Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin StandardInput=tty StandardOutput=tty # Must be disabled to allow the job to access the real console PrivateDevices=no Type=simple ExecStart=/bin/bash Restart=always EOT # Add a dependency to start the debug console: $ sudo sed -i '$a Requires=kata-debug.service' ${ROOTFS_DIR}/lib/systemd/system/kata-containers.target # Output File: ${GOPATH}/src/github.com/kata-containers/osbuilder/rootfs-builder/rootfs

编译 rootfs.img

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19

# make sure rootfs is not MODIFIED!!! if you want to add new Agent # install agent (optional) $ sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/bin ../../agent/kata-agent $ sudo install -o root -g root -m 0440 ../../agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/ $ sudo install -o root -g root -m 0440 ../../agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/ # Compile $ cd $GOPATH/src/github.com/kata-containers/osbuilder/image-builder $ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}' # install $ commit=$(git log --format=%h -1 HEAD) $ date=$(date +%Y-%m-%d-%T.%N%z) $ image="kata-containers-${date}-${commit}" $ sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}" $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img) # Output File: $GOPATH/src/github.com/kata-containers/osbuilder/image-builder/kata-containers.img

编译 initrd.img

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17

# make sure rootfs is not MODIFIED!!! if you want to add new Agent # install agent(optional) $ sudo install -o root -g root -m 0550 -T ../../agent/kata-agent ${ROOTFS_DIR}/sbin/init # Compile $ cd $GOPATH/src/github.com/kata-containers/osbuilder/initrd-builder $ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./initrd_builder.sh ${ROOTFS_DIR}' # install $ commit=$(git log --format=%h -1 HEAD) $ date=$(date +%Y-%m-%d-%T.%N%z) $ image="kata-containers-initrd-${date}-${commit}" $ sudo install -o root -g root -m 0640 -D kata-containers-initrd.img "/usr/share/kata-containers/${image}" $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers-initrd.img) # Output File: $GOPATH/src/github.com/kata-containers/osbuilder/initrd-builder/kata-containers-initrd.img

编译 qemu on aarch64

1 2 3 4 5 6 7 8 9 10

# 下载代码 $ go get -d github.com/kata-containers/tests # 准备依赖 $ sudo apt install -y libcap-ng-dev libglib2.0-dev libpixman-1-dev librbd-dev libattr1-dev libcap-dev # 编译构建 $ script -fec 'sudo -E ${GOPATH}/src/github.com/kata-containers/tests/.ci/install_qemu.sh' # 注意：如果安装失败，清直接删除文件夹，然后重新跑升级脚本 $ sudo rm -rf ${GOPATH}/src/github.com/kata-containers/packaging $ sudo rm -rf ${GOPATH}/src/github.com/qemu

Docker 对接 kata-runtime

修改 Docker 配置文件/etc/docker/daemon.json

1 2 3 4 5 6 7 8 9

{ "debug": true, "default-runtime": "runc", # 可替换成 kata-runtime "runtimes": { "kata": { "path": "/usr/local/bin/kata-runtime" # 不支持直接配置成 containerd-shim-kata-v2 } } }

重启 docker 服务（必须）

验证修改生效

sudo docker run --rm --name test busybox:latest uname -a 与宿主机内核对比，验证是否生效

调试 kata-runtime

1 2 3 4 5 6

# docker 开启 debug: /etc/docker/daemon.json 添加参数 (需重启服务) { "debug": true } # kata配置文件/etc/kata-containers/configuration.toml，开启 enable_debug # 查看日志 $ journalctl -ft kata-runtime

Containerd 对接 containerd-shim-kata-v2

修改 containerd 的配置/etc/containerd/config.toml

containerd config default 生成当前版本默认配置

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81

root = "/var/lib/containerd" state = "/run/containerd" oom_score = 0 [grpc] address = "/run/containerd/containerd.sock" uid = 0 gid = 0 max_recv_message_size = 16777216 max_send_message_size = 16777216 [debug] address = "" uid = 0 gid = 0 level = "debug" # 开启 debug [metrics] address = "" grpc_histogram = false [cgroup] path = "" [plugins] [plugins.cgroups] no_prometheus = false [plugins.cri] stream_server_address = "127.0.0.1" stream_server_port = "0" enable_selinux = false sandbox_image = "docker.io/bevisy/pause:3.1" stats_collect_period = 10 systemd_cgroup = false enable_tls_streaming = false max_container_log_line_size = 16384 disable_proc_mount = false [plugins.cri.containerd] snapshotter = "overlayfs" no_pivot = false [plugins.cri.containerd.default_runtime] runtime_type = "io.containerd.runtime.v1.linux" runtime_engine = "" runtime_root = "" [plugins.cri.containerd.untrusted_workload_runtime] runtime_type = "" runtime_engine = "" runtime_root = "" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] # 新增 >>> [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata] runtime_type = "io.containerd.kata.v2" shim_debug = true # <<< [plugins.cri.cni] bin_dir = "/opt/cni/bin" conf_dir = "/etc/cni/net.d" conf_template = "" [plugins.cri.registry] [plugins.cri.registry.mirrors] [plugins.cri.registry.mirrors."docker.io"] endpoint = ["https://registry-1.docker.io"] [plugins.cri.x509_key_pair_streaming] tls_cert_file = "" tls_key_file = "" [plugins.diff-service] default = ["walking"] [plugins.linux] shim = "containerd-shim" runtime = "runc" runtime_root = "" no_shim = false shim_debug = false [plugins.opt] path = "/opt/containerd" [plugins.restart] interval = "10s" [plugins.scheduler] pause_threshold = 0.02 deletion_threshold = 0 mutation_threshold = 100 schedule_delay = "0s" startup_delay = "100ms"

注意： 确保containerd-shim-kata-v2 文件在 $PATH 目录下

验证：

1 2 3 4 5 6 7 8

# 创建容器 sudo ctr -n testns run --runtime io.containerd.kata.v2 -d --rm docker.io/library/busybox:latest busybox # 查看容器 id sudo ctr -n testns t ls # 进入容器 sudo ctr -n testns t exec -t --exec-id {ID} busybox sh # 查看内核版本 $ uname -a # 对比宿主机内核

调试 containerd-shim-runtime-v2

1 2 3 4 5

# containerd 配置开启 debug(需重启服务) # kata配置文件/etc/kata-containers/configuration.toml，开启 enable_debug (需重新创建安全容器) # 查看日志 $ journalctl -ft kata

附录

kata-containers 2.0 安装

1. 下载release包

   Release 地址

   下载 Kata Containers 2.0.0-alpha3

   解压后，拷贝至文件夹 /opt

2. 准备配置文件和containerd-shim-kata-v2

   1 2 3 4

   # 准备配置文件 $ cp /opt/kata/share/defaults/kata-containers/configuration-qemu.toml /etc/kata-containers/configuration.toml # 准备 containerd-shim-kata-v2 $ cp /opt/kata/bin/containerd-shim-kata-v2 /usr/local/bin/

   配置文件修改如下：

   1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38

   [hypervisor.qemu] path = "/opt/kata/bin/qemu-system-x86_64" kernel = "/opt/kata/share/kata-containers/vmlinuz.container" image = "/opt/kata/share/kata-containers/kata-containers.img" machine_type = "pc" kernel_params = "" firmware = "" machine_accelerators="" cpu_features="pmu=off" default_vcpus = 1 default_maxvcpus = 2 default_bridges = 1 default_memory = 2048 disable_block_device_use = false shared_fs = "virtio-9p" virtio_fs_daemon = "/opt/kata/bin/virtiofsd" virtio_fs_cache_size = 1024 virtio_fs_extra_args = [] virtio_fs_cache = "auto" block_device_driver = "virtio-scsi" enable_iothreads = false enable_vhost_user_store = false vhost_user_store_path = "/var/run/kata-containers/vhost-user" enable_debug = true [factory] [agent.kata] enable_debug = true kernel_modules=[] [netmon] path = "/opt/kata/libexec/kata-containers/kata-netmon" enable_debug = true [runtime] enable_debug = true internetworking_model="tcfilter" disable_guest_seccomp=true sandbox_cgroup_only=false experimental=[] EnablePprof = true

对接docker和containerd需要注意

注意：此版本无法与 docker配合使用，与containerd 使用正常。

In this article, you will learn how to use Kata Containers on IBM Power systems.

#Prerequisites:

You need to install golang version 1.8.3 or newer, make, gcc, qemu.

#Steps:

1.Build and install the Kata Containers runtime

$ go get -d -u github.com/kata-containers/runtime$ cd $GOPATH/src/github.com/kata-containers/runtime$ make && sudo -E PATH=$PATH make install

The build will create the following:

runtime binary: /usr/local/bin/kata-runtime

configuration file: /usr/share/defaults/kata-containers/configuration.toml

2. Check if your system is capable of creating a Kata Container:

$ sudo kata-runtime kata-check

If your system is not able to run Kata Containers, the previous command will error out and explain why.

3. Configure to use initrd image

$ sudo sed -i ‘s/^(image =.*)/# 1/g’ /usr/share/defaults/kata-containers/configuration.toml

4. Enable full debug

$ sudo sed -i -e ‘s/^# *(enable_debug).*=.*$/1 = true/g’ /usr/share/defaults/kata-containers/configuration.toml$ sudo sed -i -e ‘s/^kernel_params = “(.*)”/kernel_params = “1 agent.log=debug”/g’ /usr/share/defaults/kata-containers/configuration.toml

5. Build and install Kata proxy

$ go get -d -u github.com/kata-containers/proxy$ cd $GOPATH/src/github.com/kata-containers/proxy && make && sudo make install

6. Build and install Kata shim

$ go get -d -u github.com/kata-containers/shim$ cd $GOPATH/src/github.com/kata-containers/shim && make && sudo make install

7. Get the osbuilder

$ go get -d -u github.com/kata-containers/osbuilder

8. Build a custom Kata agent — OPTIONAL

$ go get -d -u github.com/kata-containers/agent$ cd $GOPATH/src/github.com/kata-containers/agent && make

9. Create an initrd image

$ export ROOTFS_DIR=”${GOPATH}/src/github.com/kata-containers/osbuilder/rootfs-builder/rootfs"$ sudo rm -rf ${ROOTFS_DIR}$ cd $GOPATH/src/github.com/kata-containers/osbuilder/rootfs-builder$ script -fec ‘sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true ./rootfs.sh ${distro}’

AGENT_INIT controls if the guest image uses kata agent as the guest init process. When you create an initrd image, always set AGENT_INIT to yes.

You MUST choose one of alpine, centos and fedora for ${distro}.

Optionally, add your custom agent binary to the rootfs with the following:

$ sudo install -o root -g root -m 0550 -T ../../agent/kata-agent ${ROOTFS_DIR}/sbin/init

10. Build an initrd image

$ cd $GOPATH/src/github.com/kata-containers/osbuilder/initrd-builder$ script -fec ‘sudo -E AGENT_INIT=yes USE_DOCKER=true ./initrd_builder.sh ${ROOTFS_DIR}’

11. Install the initrd image

$ commit=$(git log — format=%h -1 HEAD)$ date=$(date +%Y-%m-%d-%T.%N%z)$ image=”kata-containers-initrd-${date}-${commit}”$ sudo install -o root -g root -m 0640 -D kata-containers-initrd.img “/usr/share/kata-containers/${image}”$ (cd /usr/share/kata-containers && sudo ln -sf “$image” kata-containers-initrd.img)

12. Install guest kernel images

As a prerequisite, you need to install libelf-dev and bc. Otherwise, you will not be able to build the kernel from sources.

$ go get github.com/kata-containers/tests$ cd $GOPATH/src/github.com/kata-containers/tests/.ci$ kernel_arch=”$(./kata-arch.sh)”$ kernel_dir=”$(./kata-arch.sh — kernel)”$ tmpdir=”$(mktemp -d)”$ pushd “$tmpdir”$ curl -L https://raw.githubusercontent.com/kata-containers/packaging/master/kernel/configs/${kernel_arch}_kata_kvm_4.14.x -o .config$ kernel_version=$(grep “Linux/[${kernel_arch}]*” .config | cut -d’ ‘ -f3 | tail -1)$ kernel_tar_file=”linux-${kernel_version}.tar.xz”$ kernel_url=”https://cdn.kernel.org/pub/linux/kernel/v$(echo $kernel_version | cut -f1 -d.).x/${kernel_tar_file}”$ curl -LOk ${kernel_url}$ tar -xf ${kernel_tar_file}$ mv .config “linux-${kernel_version}”$ pushd “linux-${kernel_version}”$ make ARCH=${kernel_dir} -j$(nproc)$ kata_kernel_dir=”/usr/share/kata-containers”$ kata_vmlinuz=”${kata_kernel_dir}/kata-vmlinuz-${kernel_version}.container”$ [ $kernel_arch = ppc64le ] && kernel_file=”$(realpath ./vmlinux)” || kernel_file=”$(realpath arch/${kernel_arch}/boot/bzImage)”$ sudo install -o root -g root -m 0755 -D “${kernel_file}” “${kata_vmlinuz}”$ sudo ln -sf “${kata_vmlinuz}” “${kata_kernel_dir}/vmlinuz.container”$ kata_vmlinux=”${kata_kernel_dir}/kata-vmlinux-${kernel_version}”$ sudo install -o root -g root -m 0755 -D “$(realpath vmlinux)” “${kata_vmlinux}”$ sudo ln -sf “${kata_vmlinux}” “${kata_kernel_dir}/vmlinux.container”$ popd$ popd$ rm -rf “${tmpdir}”

13. Run Kata Containers with Docker

Update Docker configuration

$ dir=/etc/systemd/system/docker.service.d$ file=”$dir/kata-containers.conf”$ sudo mkdir -p “$dir”$ sudo test -e “$file” || echo -e “[Service]
Type=simple
ExecStart=
ExecStart=/usr/bin/dockerd -D — default-runtime runc” | sudo tee “$file”$ sudo grep -q “kata-runtime=” $file || sudo sed -i ‘s!^(ExecStart=[^$].*$)!1 — add-runtime kata-runtime=/usr/local/bin/kata-runtime!g’ “$file”$ sudo systemctl daemon-reload$ sudo systemctl restart docker

14. Create a Kata Container using Docker

$ sudo docker run -ti — runtime kata-runtime busybox sh

15. Check the logs in-case of failure

$ journalctl -q -o cat -a -t kata-runtime
$ journalctl -q -o cat -a -t kata-proxy