[root@localhost ~]# ps -elf | grep qemu 6 S root 49878 1 0 80 0 - 6068 poll_s 22:00 ? 00:00:00 /usr/libexec/qemu-kvm -name guest=vhuser-test1,debug-threads=on -S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-10-vhuser-test1/master-key.aes -machine virt-rhel7.6.0,accel=kvm,usb=off,dump-guest-core=off,gic-version=3 -cpu host -drive file=/usr/share/AAVMF/AAVMF_CODE.fd,if=pflash,format=raw,unit=0,readonly=on -drive file=/var/lib/libvirt/qemu/nvram/vhuser-test1_VARS.fd,if=pflash,format=raw,unit=1 -m 3072 -realtime mlock=on -smp 3,sockets=1,cores=3,threads=1 -object memory-backend-file,id=ram-node0,prealloc=yes,mem-path=/dev/hugepages/libvirt/qemu/10-vhuser-test1,share=yes,size=3221225472,host-nodes=0,policy=bind -numa node,nodeid=0,cpus=0-2,memdev=ram-node0 -uuid a412374f-2195-48b2-ac01-dcbddfb23c1d -display none -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=25,server,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown -boot strict=on -device pcie-root-port,port=0x8,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x1 -device pcie-root-port,port=0x9,chassis=2,id=pci.2,bus=pcie.0,addr=0x1.0x1 -device pcie-root-port,port=0xa,chassis=3,id=pci.3,bus=pcie.0,addr=0x1.0x2 -device pcie-root-port,port=0xb,chassis=4,id=pci.4,bus=pcie.0,addr=0x1.0x3 -device pcie-root-port,port=0xc,chassis=5,id=pci.5,bus=pcie.0,addr=0x1.0x4 -device pcie-root-port,port=0xd,chassis=6,id=pci.6,bus=pcie.0,addr=0x1.0x5 -device pcie-root-port,port=0xe,chassis=7,id=pci.7,bus=pcie.0,addr=0x1.0x6 -device qemu-xhci,p2=8,p3=8,id=usb,bus=pci.2,addr=0x0 -device virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0 -drive file=/data1/cloud_images/vhuser-test1.qcow2,format=qcow2,if=none,id=drive-virtio-disk0 -device virtio-blk-pci,scsi=off,bus=pci.4,addr=0x0,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 -netdev tap,fd=27,id=hostnet0,vhost=on,vhostfd=28 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=02:ca:fe:fa:ce:aa,bus=pci.1,addr=0x0 -chardev socket,id=charnet1,path=/tmp/vhost-user2,server -netdev vhost-user,chardev=charnet1,id=hostnet1 -device virtio-net-pci,rx_queue_size=256,netdev=hostnet1,id=net1,mac=02:ca:fe:fa:ce:aa,bus=pci.6,addr=0x0 -chardev pty,id=charserial0 -serial chardev:charserial0 -chardev socket,id=charchannel0,fd=29,server,nowait -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -object rng-random,id=objrng0,filename=/dev/urandom -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.5,addr=0x0 -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on 0 S root 49912 49849 0 80 0 - 1729 pipe_w 22:00 pts/2 00:00:00 grep --color=auto qemu [root@localhost ~]# pstack 49878 Thread 4 (Thread 0xffff7213cb10 (LWP 49905)): #0 syscall () at ../sysdeps/unix/sysv/linux/aarch64/syscall.S:38 #1 0x0000aaaae3b9c5a4 in qemu_event_wait () #2 0x0000aaaae3bacdf0 in call_rcu_thread () #3 0x0000ffff73477d38 in start_thread (arg=0xffff7213cb10) at pthread_create.c:309 #4 0x0000ffff733bf5f0 in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:91 Thread 3 (Thread 0xffff7192cb10 (LWP 49906)): #0 0x0000ffff733b4f00 in __GI_ppoll (fds=0xaaab1f5227c0, nfds=<optimized out>, timeout=<optimized out>, sigmask=<optimized out>) at ../sysdeps/unix/sysv/linux/ppoll.c:56 #1 0x0000aaaae3b982d8 in qemu_poll_ns () #2 0x0000aaaae3b9a104 in aio_poll () #3 0x0000aaaae39ca590 in iothread_run () #4 0x0000ffff73477d38 in start_thread (arg=0xffff7192cb10) at pthread_create.c:309 #5 0x0000ffff733bf5f0 in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:91 Thread 2 (Thread 0xffff7111cb10 (LWP 49907)): #0 do_sigwait (sig=0xffff7111c264, set=<optimized out>) at ../sysdeps/unix/sysv/linux/sigwait.c:63 #1 __sigwait (set=<optimized out>, sig=0xffff7111c264) at ../sysdeps/unix/sysv/linux/sigwait.c:95 #2 0x0000aaaae3b9a690 in sigwait_compat () #3 0x0000ffff73477d38 in start_thread (arg=0xffff7111cb10) at pthread_create.c:309 #4 0x0000ffff733bf5f0 in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:91 Thread 1 (Thread 0xffff72140a90 (LWP 49878)): #0 0x0000ffff733b4e24 in __GI___poll (fds=0xaaab1f521ba0, nfds=<optimized out>, timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/generic/poll.c:45 #1 0x0000ffff7376118c in g_main_context_iterate.isra.19 () from /lib64/libglib-2.0.so.0 #2 0x0000ffff73761538 in g_main_loop_run () from /lib64/libglib-2.0.so.0 #3 0x0000aaaae3b4e870 in qio_net_listener_wait_client () #4 0x0000aaaae3b3db40 in tcp_chr_wait_connected () #5 0x0000aaaae3b35a84 in qemu_chr_wait_connected () #6 0x0000aaaae3b3d4e8 in qmp_chardev_open_socket () #7 0x0000aaaae3b366a0 in qemu_chardev_new () #8 0x0000aaaae3b368d0 in qemu_chr_new_from_opts () #9 0x0000aaaae39cf07c in chardev_init_func () #10 0x0000aaaae3ba9d7c in qemu_opts_foreach () #11 0x0000aaaae389a280 in main () [root@localhost ~]#
root@ubuntu:/home/ubuntu# gdb attach 51365 GNU gdb (Ubuntu 8.1-0ubuntu3.2) 8.1.0.20180409-git Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>. Find the GDB manual and other documentation resources online at: <http://www.gnu.org/software/gdb/documentation/>. For help, type "help". Type "apropos word" to search for commands related to "word"... attach: No such file or directory. Attaching to process 51365 [New LWP 51366] [New LWP 51367] [New LWP 51370] warning: .dynamic section for "/lib/aarch64-linux-gnu/libc.so.6" is not at the expected address (wrong library or version mismatch?) warning: File "/lib/aarch64-linux-gnu/libthread_db-1.0.so" auto-loading has been declined by your `auto-load safe-path' set to "$debugdir:$datadir/auto-load". To enable execution of this file add add-auto-load-safe-path /lib/aarch64-linux-gnu/libthread_db-1.0.so line to your configuration file "/root/.gdbinit". To completely disable this security protection add set auto-load safe-path / line to your configuration file "/root/.gdbinit". For more information about this security protection see the "Auto-loading safe path" section in the GDB manual. E.g., run from the shell: info "(gdb)Auto-loading safe path" warning: Unable to find libthread_db matching inferior's thread library, thread debugging will not be available. warning: File "/lib/aarch64-linux-gnu/libthread_db-1.0.so" auto-loading has been declined by your `auto-load safe-path' set to "$debugdir:$datadir/auto-load". warning: Unable to find libthread_db matching inferior's thread library, thread debugging will not be available. 0x0000ffff9cd3f144 in fts_build (sp=0xffff9cdcb000 <__libio_codecvt+112>, type=265649664) at ../sysdeps/wordsize-64/../../io/fts.c:865 865 ../sysdeps/wordsize-64/../../io/fts.c: No such file or directory. (gdb) bt #0 0x0000ffff9cd3f144 in fts_build (sp=0xffff9cdcb000 <__libio_codecvt+112>, type=265649664) at ../sysdeps/wordsize-64/../../io/fts.c:865 #1 0x0000aaaad5661d50 in ppoll (__ss=0x0, __timeout=0xffffcb2a8ed8, __nfds=<optimized out>, __fds=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/poll2.h:77 #2 0x0000aaaad5661d50 in qemu_poll_ns (fds=<optimized out>, nfds=<optimized out>, timeout=timeout@entry=187650704736256) at util/qemu-timer.c:347 #3 0x0000aaaad565d19c in os_host_main_loop_wait (timeout=187650704736256) at util/main-loop.c:235 #4 0x0000aaaad565d19c in main_loop_wait (nonblocking=nonblocking@entry=0) at util/main-loop.c:516 #5 0x0000aaaad5197428 in qemu_main_loop () at /root/go/src/github.com/qemu/qemu/softmmu/vl.c:1676 #6 0x0000aaaad50458d0 in main (argc=<optimized out>, argv=<optimized out>, envp=<optimized out>) at /root/go/src/github.com/qemu/qemu/softmmu/main.c:49 (gdb) info thread Id Target Id Frame * 1 LWP 51365 "qemu-system-aar" 0x0000ffff9cd3f144 in fts_build (sp=0xffff9cdcb000 <__libio_codecvt+112>, type=265649664) at ../sysdeps/wordsize-64/../../io/fts.c:865 2 LWP 51366 "qemu-system-aar" getpass (prompt=0xffffcb2a8dd8 "x216*313377377") at getpass.c:86 3 LWP 51367 "qemu-system-aar" 0x0000ffff9ccab29c in __libc_siglongjmp (env=0xaaab0fd34f70, val=-1663258624) at longjmp.c:32 4 LWP 51370 "qemu-system-aar" 0x0000ffff9cd40b0c in __tcsetattr (fd=<optimized out>, optional_actions=<optimized out>, termios_p=0x0) at ../sysdeps/unix/sysv/linux/tcsetattr.c:103 (gdb) thread 1 [Switching to thread 1 (LWP 51365)] #0 0x0000ffff9cd3f144 in fts_build (sp=0xffff9cdcb000 <__libio_codecvt+112>, type=265649664) at ../sysdeps/wordsize-64/../../io/fts.c:865 865 in ../sysdeps/wordsize-64/../../io/fts.c (gdb) bt #0 0x0000ffff9cd3f144 in fts_build (sp=0xffff9cdcb000 <__libio_codecvt+112>, type=265649664) at ../sysdeps/wordsize-64/../../io/fts.c:865 #1 0x0000aaaad5661d50 in ppoll (__ss=0x0, __timeout=0xffffcb2a8ed8, __nfds=<optimized out>, __fds=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/poll2.h:77 #2 0x0000aaaad5661d50 in qemu_poll_ns (fds=<optimized out>, nfds=<optimized out>, timeout=timeout@entry=187650704736256) at util/qemu-timer.c:347 #3 0x0000aaaad565d19c in os_host_main_loop_wait (timeout=187650704736256) at util/main-loop.c:235 #4 0x0000aaaad565d19c in main_loop_wait (nonblocking=nonblocking@entry=0) at util/main-loop.c:516 #5 0x0000aaaad5197428 in qemu_main_loop () at /root/go/src/github.com/qemu/qemu/softmmu/vl.c:1676 #6 0x0000aaaad50458d0 in main (argc=<optimized out>, argv=<optimized out>, envp=<optimized out>) at /root/go/src/github.com/qemu/qemu/softmmu/main.c:49 (gdb) thread 2 [Switching to thread 2 (LWP 51366)] #0 getpass (prompt=0xffffcb2a8dd8 "x216*313377377") at getpass.c:86 86 getpass.c: No such file or directory. (gdb) bt #0 0x0000ffff9cd448e0 in getpass (prompt=0xffffcb2a8dd8 "x216*313377377") at getpass.c:86 #1 0x0000aaaad5651cf4 in call_rcu_thread (opaque=<optimized out>) at util/rcu.c:258 (gdb) thread 3 [Switching to thread 3 (LWP 51367)] #0 0x0000ffff9ccab29c in __libc_siglongjmp (env=0xaaab0fd34f70, val=-1663258624) at longjmp.c:32 32 longjmp.c: No such file or directory. (gdb) bt #0 0x0000ffff9ccab29c in __libc_siglongjmp (env=0xaaab0fd34f70, val=-1663258624) at longjmp.c:32 #1 0x0000ffff9cde3fac in __libc_msgsnd (msqid=<optimized out>, msgp=0xffff933e8ab8, msgsz=0, msgflg=0) at ../sysdeps/unix/sysv/linux/msgsnd.c:27 #2 0x0000aaaad566b5e8 in sigwait_compat (opaque=0xaaab0fd34f70) at util/compatfd.c:37 #3 0x0000aaaad5648438 in qemu_thread_start (args=<optimized out>) at util/qemu-thread-posix.c:521 #4 0x0000ffff9cdd9088 in start_thread (arg=0xffffcb2a8adf) at pthread_create.c:463 #5 0x0000ffff9cd484dc in __GI___get_nprocs () at ../sysdeps/unix/sysv/linux/getsysstats.c:178 (gdb) thread 4 [Switching to thread 4 (LWP 51370)] #0 0x0000ffff9cd40b0c in __tcsetattr (fd=<optimized out>, optional_actions=<optimized out>, termios_p=0x0) at ../sysdeps/unix/sysv/linux/tcsetattr.c:103 103 ../sysdeps/unix/sysv/linux/tcsetattr.c: No such file or directory. (gdb) bt #0 0x0000ffff9cd40b0c in __tcsetattr (fd=<optimized out>, optional_actions=<optimized out>, termios_p=0x0) at ../sysdeps/unix/sysv/linux/tcsetattr.c:103 #1 0x0000aaaad509d620 in kvm_cpu_exec (cpu=0xaaaad5990000, cpu@entry=0xaaab0fee2060) at /root/go/src/github.com/qemu/qemu/accel/kvm/kvm-all.c:2468 #2 0x0000aaaad518788c in qemu_kvm_cpu_thread_fn (arg=0xaaab0fee2060) at /root/go/src/github.com/qemu/qemu/softmmu/cpus.c:1188 #3 0x0000aaaad5648438 in qemu_thread_start (args=<optimized out>) at util/qemu-thread-posix.c:521 #4 0x0000ffff9cdd9088 in start_thread (arg=0xffffcb2a871f) at pthread_create.c:463 #5 0x0000ffff9cd484dc in __GI___get_nprocs () at ../sysdeps/unix/sysv/linux/getsysstats.c:178 (gdb) quit A debugging session is active. Inferior 1 [process 51365] will be detached. Quit anyway? (y or n) y Detaching from program: /usr/share/kata-containers/binary/qemu-system-aarch64, process 51365 root@ubuntu:/home/ubuntu#
QEMU
其中, machine->init会创建vcpu, 用一个线程去模拟vcpu, 该线程执行的函数为qemu_kvm_cpu_thread_fn, 并且该线程最终调用kvm_cpu_exec, 该函数调用kvm_vcpu_ioctl切换到kvm中,下次从kvm中返回时,会接着执行kvm_vcpu_ioctl之后的代码
之前创建的线程中有一个大循环，执行kvm_cpu_exec()
do {
if (cpu_can_run(cpu)) {
r = kvm_cpu_exec(cpu);
if (r == EXCP_DEBUG) {
cpu_handle_guest_debug(cpu);
}
}
qemu_wait_io_event(cpu);
} while (!cpu->unplug || cpu_can_run(cpu));
kvm_cpu_exec()函数会调用ioctl让kvm执行客户机代码，一直到在需要进行IO模拟等操作。此时从内核返回，由qemu处理这些操作，之后返回kvm继续执行。
一句话总结
实例化一个vcpu就是在hostOS中创建了一个线程，线程里有个while循环，循环里不停的调用kvm_cpu_exec方法，kvm_cpu_exec方法调用通过kvm_vcpu_ioctl(cpu, KVM_RUN, 0)使得kvm切换为no-root模式（qemu处理）。在no-root模式下处理特权指令的时候，会退回root模式，然后一步步返回到kvm_cpu_exec中根据不同原因，处理返回异常。
如此一个轮回结束，周而复始，vcpu。
再补充说一点，内存中申请一块内存，根模式和非根模式切换的时候，先把当前寄存器值放到这块内存中，然后设置物理cpu使得进入对应模式，这块内存叫vmcs。
背景
vcpu初始化的时候（qemu_init_vcpu）是启动了一个线程，也就是说vcpu其实就是一个线程.线程运行方法是qemu_kvm_cpu_thread_fn
kvm_init_vcpu调用KVM_CREATE_VCPU创建了vcpu返回vm_fdvcpu的运行是在kvm_cpu_exec里面的，这里调用如下命令进入kvm
run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
进入KVM后，KVM会切入Guest OS，假如Guest OS运行运行，需要访问IO等也就是说要访问physical device，那么Qemu与KVM就要进行emulate。如果是KVM emulate的则由KVM emulate，然后切回Guest OS。如果是Qemu emulate的，则从KVM中进入Qemu，等Qemu中的device model执行完emulate之后，再次在Qemu中调用kvm_vcpu_ioctl(vcpu_fd, KVM_RUN, xxx)进入KVM运行，然后再切回Guest OS