root@cloud:~# docker exec -it test ping 8.8.8.8 PING 8.8.8.8 (8.8.8.8): 56 data bytes 64 bytes from 8.8.8.8: seq=0 ttl=42 time=56.182 ms 64 bytes from 8.8.8.8: seq=1 ttl=42 time=55.559 ms 64 bytes from 8.8.8.8: seq=2 ttl=42 time=77.515 ms 64 bytes from 8.8.8.8: seq=3 ttl=42 time=55.752 ms 64 bytes from 8.8.8.8: seq=4 ttl=42 time=55.839 ms 64 bytes from 8.8.8.8: seq=5 ttl=42 time=83.197 ms
bluepill
// bluepill enters guest mode.
func bluepill(*vCPU)
// See bluepill.go.
TEXT ·bluepill(SB),NOSPLIT,$0
begin:
MOVQ vcpu+0(FP), AX
LEAQ VCPU_CPU(AX), BX
BYTE CLI;
check_vcpu:
MOVQ ENTRY_CPU_SELF(GS), CX
CMPQ BX, CX
JE right_vCPU
wrong_vcpu:
CALL ·redpill(SB)
JMP begin
right_vCPU:
RET
bluepillHandler
//go:nosplit func bluepillHandler(context unsafe.Pointer) { // Sanitize the registers; interrupts must always be disabled. c := bluepillArchEnter(bluepillArchContext(context)) // Mark this as guest mode. switch atomic.SwapUint32(&c.state, vCPUGuest|vCPUUser) { case vCPUUser: // Expected case. case vCPUUser | vCPUWaiter: c.notify() default: throw("invalid state") } for { _, _, errno := syscall.RawSyscall(syscall.SYS_IOCTL, uintptr(c.fd), _KVM_RUN, 0) // escapes: no. switch errno { case 0: // Expected case. case syscall.EINTR: // First, we process whatever pending signal // interrupted KVM. Since we're in a signal handler // currently, all signals are masked and the signal // must have been delivered directly to this thread. timeout := syscall.Timespec{} sig, _, errno := syscall.RawSyscall6( // escapes: no. syscall.SYS_RT_SIGTIMEDWAIT, uintptr(unsafe.Pointer(&bounceSignalMask)), 0, // siginfo. uintptr(unsafe.Pointer(&timeout)), // timeout. 8, // sigset size. 0, 0) if errno == syscall.EAGAIN { continue } if errno != 0 { throw("error waiting for pending signal") } if sig != uintptr(bounceSignal) { throw("unexpected signal") } // Check whether the current state of the vCPU is ready // for interrupt injection. Because we don't have a // PIC, we can't inject an interrupt while they are // masked. We need to request a window if it's not // ready. if bluepillReadyStopGuest(c) { // Force injection below; the vCPU is ready. c.runData.exitReason = _KVM_EXIT_IRQ_WINDOW_OPEN } else { c.runData.requestInterruptWindow = 1 continue // Rerun vCPU. } case syscall.EFAULT: // If a fault is not serviceable due to the host // backing pages having page permissions, instead of an // MMIO exit we receive EFAULT from the run ioctl. We // always inject an NMI here since we may be in kernel // mode and have interrupts disabled. bluepillSigBus(c) continue // Rerun vCPU. case syscall.ENOSYS: bluepillHandleEnosys(c) continue default: throw("run failed") } switch c.runData.exitReason { case _KVM_EXIT_EXCEPTION: c.die(bluepillArchContext(context), "exception") return case _KVM_EXIT_IO: c.die(bluepillArchContext(context), "I/O") return case _KVM_EXIT_INTERNAL_ERROR: // An internal error is typically thrown when emulation // fails. This can occur via the MMIO path below (and // it might fail because we have multiple regions that // are not mapped). We would actually prefer that no // emulation occur, and don't mind at all if it fails. case _KVM_EXIT_HYPERCALL: c.die(bluepillArchContext(context), "hypercall") return case _KVM_EXIT_DEBUG: c.die(bluepillArchContext(context), "debug") return case _KVM_EXIT_HLT: bluepillGuestExit(c, context) return case _KVM_EXIT_MMIO: physical := uintptr(c.runData.data[0]) if getHypercallID(physical) == _KVM_HYPERCALL_VMEXIT { bluepillGuestExit(c, context) return } // Increment the fault count. atomic.AddUint32(&c.faults, 1) // For MMIO, the physical address is the first data item. physical = uintptr(c.runData.data[0]) virtual, ok := handleBluepillFault(c.machine, physical, physicalRegions, _KVM_MEM_FLAGS_NONE) if !ok { c.die(bluepillArchContext(context), "invalid physical address") return } // We now need to fill in the data appropriately. KVM // expects us to provide the result of the given MMIO // operation in the runData struct. This is safe // because, if a fault occurs here, the same fault // would have occurred in guest mode. The kernel should // not create invalid page table mappings. data := (*[8]byte)(unsafe.Pointer(&c.runData.data[1])) length := (uintptr)((uint32)(c.runData.data[2])) write := (uint8)(((c.runData.data[2] >> 32) & 0xff)) != 0 for i := uintptr(0); i < length; i++ { b := bytePtr(uintptr(virtual) + i) if write { // Write to the given address. *b = data[i] } else { // Read from the given address. data[i] = *b } } case _KVM_EXIT_IRQ_WINDOW_OPEN: bluepillStopGuest(c) case _KVM_EXIT_SHUTDOWN: c.die(bluepillArchContext(context), "shutdown") return case _KVM_EXIT_FAIL_ENTRY: c.die(bluepillArchContext(context), "entry failed") return default: bluepillArchHandleExit(c, context) return } } }
root@cloud:/mycontainer# dlv attach 928771 Type 'help' for list of commands. (dlv) b bluepillHandler Breakpoint 1 set at 0x87b300 for gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:91 (dlv) c > gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:91 (hits goroutine(276):1 total:1) (PC: 0x87b300) Warning: debugging optimized function (dlv) bt 0 0x000000000087b300 in gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler at pkg/sentry/platform/kvm/bluepill_unsafe.go:91 1 0x0000000000881bec in ??? at ?:-1 2 0x0000ffff82bfe598 in ??? at ?:-1 3 0x000000000087f514 in gvisor.dev/gvisor/pkg/sentry/platform/kvm.(*vCPU).SwitchToUser at pkg/sentry/platform/kvm/machine_arm64_unsafe.go:249 4 0x00000000009e5f89 in ??? at ?:-1 5 0x000000000087bb1c in gvisor.dev/gvisor/pkg/sentry/platform/kvm.(*context).Switch at pkg/sentry/platform/kvm/context.go:75 6 0x00000040005fc000 in ??? at ?:-1 7 0x0000000000517d9c in gvisor.dev/gvisor/pkg/sentry/kernel.(*Task).run at pkg/sentry/kernel/task_run.go:97 8 0x00000040007a73f0 in ??? at ?:-1 9 0x0000000000077c84 in runtime.goexit at src/runtime/asm_arm64.s:1136 error: Undefined return address at 0x77c84 (truncated) (dlv) s > gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:58 (PC: 0x87b30c) Warning: debugging optimized function (dlv) list > gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:58 (PC: 0x87b30c) Warning: debugging optimized function Command failed: open pkg/sentry/platform/kvm/bluepill_unsafe.go: no such file or directory (dlv) quit Would you like to kill the process? [Y/n] n root@cloud:/mycontainer# dlv attach 928771 Type 'help' for list of commands. (dlv) b bluepill_unsafe.go:105 Breakpoint 1 set at 0x87b34c for gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:105 (dlv) c > gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:105 (hits goroutine(252):1 total:1) (PC: 0x87b34c) Warning: debugging optimized function (dlv) bt 0 0x000000000087b34c in gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler at pkg/sentry/platform/kvm/bluepill_unsafe.go:105 1 0x0000000000881bec in ??? at ?:-1 2 0x0000ffff82bfe598 in ??? at ?:-1 3 0x000000000087f514 in gvisor.dev/gvisor/pkg/sentry/platform/kvm.(*vCPU).SwitchToUser at pkg/sentry/platform/kvm/machine_arm64_unsafe.go:249 (dlv) p errno Command failed: could not find symbol value for errno (dlv) n > gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:106 (PC: 0x87b370) Warning: debugging optimized function (dlv) p errno 0 (dlv) p c.runData.exitReason 6 (dlv) p c.runData *gvisor.dev/gvisor/pkg/sentry/platform/kvm.runData { requestInterruptWindow: 0, _: [7]uint8 [0,0,0,0,0,0,0], exitReason: 6, readyForInterruptInjection: 0, ifFlag: 0, _: [2]uint8 [0,0], cr8: 0, apicBase: 0, data: [32]uint64 [8882176,0,4294967304,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],} (dlv) clearall Breakpoint 1 cleared at 0x87b34c for gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:105 (dlv) quit Would you like to kill the process? [Y/n] y root@cloud:/mycontainer#
exitReason: 6
// KVM exit reasons. const ( _KVM_EXIT_EXCEPTION = 0x1 _KVM_EXIT_IO = 0x2 _KVM_EXIT_HYPERCALL = 0x3 _KVM_EXIT_DEBUG = 0x4 _KVM_EXIT_HLT = 0x5 _KVM_EXIT_MMIO = 0x6 _KVM_EXIT_IRQ_WINDOW_OPEN = 0x7 _KVM_EXIT_SHUTDOWN = 0x8 _KVM_EXIT_FAIL_ENTRY = 0x9 _KVM_EXIT_INTERNAL_ERROR = 0x11 _KVM_EXIT_SYSTEM_EVENT = 0x18 _KVM_EXIT_ARM_NISV = 0x1c )
root@cloud:/mycontainer# dlv attach 928771 could not attach to pid 928771: no such process root@cloud:/mycontainer# dlv attach 929197 Type 'help' for list of commands. (dlv) b bluepill_unsafe.go:105 Breakpoint 1 set at 0x87b34c for gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:105 (dlv) c > gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:105 (hits goroutine(224):1 total:1) (PC: 0x87b34c) Warning: debugging optimized function (dlv) p context unsafe.Pointer(0x40006a0e20) (dlv) p *context Command failed: expression "context" (unsafe.Pointer) can not be dereferenced (dlv) p c *gvisor.dev/gvisor/pkg/sentry/platform/kvm.vCPU { CPU: gvisor.dev/gvisor/pkg/sentry/platform/ring0.CPU { self: *(*"gvisor.dev/gvisor/pkg/sentry/platform/ring0.CPU")(0x40004ff000), kernel: *(*"gvisor.dev/gvisor/pkg/sentry/platform/ring0.Kernel")(0x4000323a18), CPUArchState: (*"gvisor.dev/gvisor/pkg/sentry/platform/ring0.CPUArchState")(0x40004ff010), registers: (*"gvisor.dev/gvisor/pkg/sentry/arch.Registers")(0x40004ff258), hooks: gvisor.dev/gvisor/pkg/sentry/platform/ring0.Hooks(*gvisor.dev/gvisor/pkg/sentry/platform/kvm.vCPU) ...,}, id: 0, fd: 7, tid: 21, userExits: 14, guestExits: 11, faults: 0, state: 3, runData: *gvisor.dev/gvisor/pkg/sentry/platform/kvm.runData { requestInterruptWindow: 0, _: [7]uint8 [0,0,0,0,0,0,0], exitReason: 6, readyForInterruptInjection: 0, ifFlag: 0, _: [2]uint8 [0,0], cr8: 0, apicBase: 0, data: [32]uint64 [8882176,0,4294967304,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],}, machine: *gvisor.dev/gvisor/pkg/sentry/platform/kvm.machine { fd: 23, nextSlot: 17, upperSharedPageTables: *(*"gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables.PageTables")(0x4000124280), kernel: (*"gvisor.dev/gvisor/pkg/sentry/platform/ring0.Kernel")(0x4000323a18), mu: (*"gvisor.dev/gvisor/pkg/sync.RWMutex")(0x4000323a28), available: (*sync.Cond)(0x4000323a40), vCPUsByTID: map[uint64]*gvisor.dev/gvisor/pkg/sentry/platform/kvm.vCPU [...], vCPUsByID: []*gvisor.dev/gvisor/pkg/sentry/platform/kvm.vCPU len: 512, cap: 512, [ *(*"gvisor.dev/gvisor/pkg/sentry/platform/kvm.vCPU")(0x40004ff000), *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, ...+448 more ], maxVCPUs: 512, maxSlots: 512, usedSlots: []uintptr len: 512, cap: 512, [65536,10092544,1096804425728,1096804954112,1097341366272,1097408417792,1097416798208,21823488,274877906944,1096766996480,1096804950016,1097341362176,1097408413696,1097416794112,1097417842688,1099348434944,1093521661952,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...+448 more], nextID: 1,}, active: gvisor.dev/gvisor/pkg/sentry/platform/kvm.atomicAddressSpace {pointer: unsafe.Pointer(0x400021c5e0)}, vCPUArchState: gvisor.dev/gvisor/pkg/sentry/platform/kvm.vCPUArchState { PCIDs: *(*"gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables.PCIDs")(0x400014cba0), floatingPointState: *0,}, dieState: gvisor.dev/gvisor/pkg/sentry/platform/kvm.dieState { message: "", guestRegs: (*"gvisor.dev/gvisor/pkg/sentry/platform/kvm.userRegs")(0x40004ff3e8),},} (dlv) p c.state 3 (dlv) b bluepill_unsafe.go:105-6 Command failed: Malformed breakpoint location "bluepill_unsafe.go:105-6" at 19: line offset negative or not a number (dlv) b bluepill_unsafe.go:106 Breakpoint 2 set at 0x87b370 for gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:106 (dlv) c > gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:106 (hits goroutine(224):1 total:1) (PC: 0x87b370) Warning: debugging optimized function (dlv) p c.state 3 (dlv) p c *gvisor.dev/gvisor/pkg/sentry/platform/kvm.vCPU { CPU: gvisor.dev/gvisor/pkg/sentry/platform/ring0.CPU { self: *(*"gvisor.dev/gvisor/pkg/sentry/platform/ring0.CPU")(0x40004ff000), kernel: *(*"gvisor.dev/gvisor/pkg/sentry/platform/ring0.Kernel")(0x4000323a18), CPUArchState: (*"gvisor.dev/gvisor/pkg/sentry/platform/ring0.CPUArchState")(0x40004ff010), registers: (*"gvisor.dev/gvisor/pkg/sentry/arch.Registers")(0x40004ff258), hooks: gvisor.dev/gvisor/pkg/sentry/platform/ring0.Hooks(*gvisor.dev/gvisor/pkg/sentry/platform/kvm.vCPU) ...,}, id: 0, fd: 7, tid: 21, userExits: 14, guestExits: 11, faults: 0, state: 3, runData: *gvisor.dev/gvisor/pkg/sentry/platform/kvm.runData { requestInterruptWindow: 0, _: [7]uint8 [0,0,0,0,0,0,0], exitReason: 6, readyForInterruptInjection: 0, ifFlag: 0, _: [2]uint8 [0,0], cr8: 0, apicBase: 0, data: [32]uint64 [8882176,0,4294967304,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],}, machine: *gvisor.dev/gvisor/pkg/sentry/platform/kvm.machine { fd: 23, nextSlot: 17, upperSharedPageTables: *(*"gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables.PageTables")(0x4000124280), kernel: (*"gvisor.dev/gvisor/pkg/sentry/platform/ring0.Kernel")(0x4000323a18), mu: (*"gvisor.dev/gvisor/pkg/sync.RWMutex")(0x4000323a28), available: (*sync.Cond)(0x4000323a40), vCPUsByTID: map[uint64]*gvisor.dev/gvisor/pkg/sentry/platform/kvm.vCPU [...], vCPUsByID: []*gvisor.dev/gvisor/pkg/sentry/platform/kvm.vCPU len: 512, cap: 512, [ *(*"gvisor.dev/gvisor/pkg/sentry/platform/kvm.vCPU")(0x40004ff000), *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, *nil, ...+448 more ], maxVCPUs: 512, maxSlots: 512, usedSlots: []uintptr len: 512, cap: 512, [65536,10092544,1096804425728,1096804954112,1097341366272,1097408417792,1097416798208,21823488,274877906944,1096766996480,1096804950016,1097341362176,1097408413696,1097416794112,1097417842688,1099348434944,1093521661952,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...+448 more], nextID: 1,}, active: gvisor.dev/gvisor/pkg/sentry/platform/kvm.atomicAddressSpace {pointer: unsafe.Pointer(0x400021c5e0)}, vCPUArchState: gvisor.dev/gvisor/pkg/sentry/platform/kvm.vCPUArchState { PCIDs: *(*"gvisor.dev/gvisor/pkg/sentry/platform/ring0/pagetables.PCIDs")(0x400014cba0), floatingPointState: *0,}, dieState: gvisor.dev/gvisor/pkg/sentry/platform/kvm.dieState { message: "", guestRegs: (*"gvisor.dev/gvisor/pkg/sentry/platform/kvm.userRegs")(0x40004ff3e8),},} (dlv) clearall Breakpoint 1 cleared at 0x87b34c for gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:105 Breakpoint 2 cleared at 0x87b370 for gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:106 (dlv) quit Would you like to kill the process? [Y/n] n root@cloud:/mycontainer#
root@cloud:/mycontainer# dlv attach 929197 Type 'help' for list of commands. (dlv) b pkg/sentry/kernel/task_run.go:271 Breakpoint 1 set at 0x51867c for gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:271 (dlv) c > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:271 (hits goroutine(233):1 total:1) (PC: 0x51867c) Warning: debugging optimized function (dlv) n > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:272 (PC: 0x51870c) Warning: debugging optimized function (dlv) p info *gvisor.dev/gvisor/pkg/sentry/arch.SignalInfo { Signo: 11, Errno: 0, Code: 1, _: 0, Fields: [112]uint8 [96,64,209,165,116,105,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...+48 more],} (dlv) p at Command failed: could not find symbol value for at (dlv) p err error(*errors.errorString) *{ s: "interrupted by signal",} (dlv) n > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:275 (PC: 0x518738) Warning: debugging optimized function (dlv) list > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:275 (PC: 0x518738) Warning: debugging optimized function Command failed: open pkg/sentry/kernel/task_run.go: no such file or directory (dlv) n > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:280 (PC: 0x518740) Warning: debugging optimized function (dlv) n > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:284 (PC: 0x518748) Warning: debugging optimized function (dlv) n > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:289 (PC: 0x51875c) Warning: debugging optimized function (dlv) n > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:299 (PC: 0x518770) Warning: debugging optimized function (dlv) n > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:303 (PC: 0x5188ec) Warning: debugging optimized function (dlv) c > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:271 (hits goroutine(233):2 total:2) (PC: 0x51867c) Warning: debugging optimized function (dlv) p info Command failed: could not find symbol value for info (dlv) n > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:272 (PC: 0x51870c) Warning: debugging optimized function (dlv) p info *gvisor.dev/gvisor/pkg/sentry/arch.SignalInfo { Signo: 11, Errno: 0, Code: 1, _: 0, Fields: [112]uint8 [0,94,2,37,107,250,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,...+48 more],} (dlv) p err error(*errors.errorString) *{ s: "interrupted by signal",} (dlv) b pkg/sentry/kernel/task_run.go:282 Breakpoint 2 set at 0x518df4 for gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:282 (dlv) c > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:271 (hits goroutine(233):3 total:3) (PC: 0x51867c) Warning: debugging optimized function (dlv) c > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:271 (hits goroutine(233):4 total:4) (PC: 0x51867c) Warning: debugging optimized function (dlv) c > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:271 (hits goroutine(233):5 total:5) (PC: 0x51867c) Warning: debugging optimized function (dlv) c > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:271 (hits goroutine(233):6 total:6) (PC: 0x51867c) Warning: debugging optimized function (dlv) c > gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:282 (hits goroutine(233):1 total:1) (PC: 0x518df4) Warning: debugging optimized function (dlv) clearall Breakpoint 1 cleared at 0x51867c for gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:271 Breakpoint 2 cleared at 0x518df4 for gvisor.dev/gvisor/pkg/sentry/kernel.(*runApp).execute() pkg/sentry/kernel/task_run.go:282 (dlv) c received SIGINT, stopping process (will not forward signal) > runtime.futex() src/runtime/sys_linux_arm64.s:565 (PC: 0x78e3c) Warning: debugging optimized function (dlv) quit Would you like to kill the process? [Y/n] n root@cloud:/mycontainer#
(dlv) p err error(*errors.errorString) *{ s: "interrupted by signal",} (dlv) n
// SwitchToUser unpacks architectural-details. func (c *vCPU) SwitchToUser(switchOpts ring0.SwitchOpts, info *arch.SignalInfo) (usermem.AccessType, error) { // Check for canonical addresses. if regs := switchOpts.Registers; !ring0.IsCanonical(regs.Rip) { return nonCanonical(regs.Rip, int32(syscall.SIGSEGV), info) } else if !ring0.IsCanonical(regs.Rsp) { return nonCanonical(regs.Rsp, int32(syscall.SIGBUS), info) } else if !ring0.IsCanonical(regs.Fs_base) { return nonCanonical(regs.Fs_base, int32(syscall.SIGBUS), info) } else if !ring0.IsCanonical(regs.Gs_base) { return nonCanonical(regs.Gs_base, int32(syscall.SIGBUS), info) } // Assign PCIDs. if c.PCIDs != nil { var requireFlushPCID bool // Force a flush? switchOpts.UserPCID, requireFlushPCID = c.PCIDs.Assign(switchOpts.PageTables) switchOpts.KernelPCID = fixedKernelPCID switchOpts.Flush = switchOpts.Flush || requireFlushPCID } // See below. var vector ring0.Vector // Past this point, stack growth can cause system calls (and a break // from guest mode). So we need to ensure that between the bluepill // call here and the switch call immediately below, no additional // allocations occur. entersyscall() bluepill(c) vector = c.CPU.SwitchToUser(switchOpts) exitsyscall() switch vector { case ring0.Syscall, ring0.SyscallInt80: // Fast path: system call executed. return usermem.NoAccess, nil ---------------------系统调用 case ring0.PageFault: return c.fault(int32(syscall.SIGSEGV), info) case ring0.Debug, ring0.Breakpoint: *info = arch.SignalInfo{ Signo: int32(syscall.SIGTRAP), Code: 1, // TRAP_BRKPT (breakpoint). } info.SetAddr(switchOpts.Registers.Rip) // Include address. return usermem.AccessType{}, platform.ErrContextSignal case ring0.GeneralProtectionFault, ring0.SegmentNotPresent, ring0.BoundRangeExceeded, ring0.InvalidTSS, ring0.StackSegmentFault: *info = arch.SignalInfo{ Signo: int32(syscall.SIGSEGV), Code: arch.SignalInfoKernel, } info.SetAddr(switchOpts.Registers.Rip) // Include address. if vector == ring0.GeneralProtectionFault { // When CPUID faulting is enabled, we will generate a #GP(0) when // userspace executes a CPUID instruction. This is handled above, // because we need to be able to map and read user memory. return usermem.AccessType{}, platform.ErrContextSignalCPUID } return usermem.AccessType{}, platform.ErrContextSignal case ring0.InvalidOpcode: *info = arch.SignalInfo{ Signo: int32(syscall.SIGILL), Code: 1, // ILL_ILLOPC (illegal opcode). } info.SetAddr(switchOpts.Registers.Rip) // Include address. return usermem.AccessType{}, platform.ErrContextSignal case ring0.DivideByZero: *info = arch.SignalInfo{ Signo: int32(syscall.SIGFPE), Code: 1, // FPE_INTDIV (divide by zero). } info.SetAddr(switchOpts.Registers.Rip) // Include address. return usermem.AccessType{}, platform.ErrContextSignal case ring0.Overflow: *info = arch.SignalInfo{ Signo: int32(syscall.SIGFPE), Code: 2, // FPE_INTOVF (integer overflow). } info.SetAddr(switchOpts.Registers.Rip) // Include address. return usermem.AccessType{}, platform.ErrContextSignal case ring0.X87FloatingPointException, ring0.SIMDFloatingPointException: *info = arch.SignalInfo{ Signo: int32(syscall.SIGFPE), Code: 7, // FPE_FLTINV (invalid operation). } info.SetAddr(switchOpts.Registers.Rip) // Include address. return usermem.AccessType{}, platform.ErrContextSignal case ring0.Vector(bounce): // ring0.VirtualizationException return usermem.NoAccess, platform.ErrContextInterrupt case ring0.AlignmentCheck: *info = arch.SignalInfo{ Signo: int32(syscall.SIGBUS), Code: 2, // BUS_ADRERR (physical address does not exist). } return usermem.NoAccess, platform.ErrContextSignal case ring0.NMI: // An NMI is generated only when a fault is not servicable by // KVM itself, so we think some mapping is writeable but it's // really not. This could happen, e.g. if some file is // truncated (and would generate a SIGBUS) and we map it // directly into the instance. return c.fault(int32(syscall.SIGBUS), info) case ring0.DeviceNotAvailable, ring0.DoubleFault, ring0.CoprocessorSegmentOverrun, ring0.MachineCheck, ring0.SecurityException: fallthrough default: panic(fmt.Sprintf("unexpected vector: 0x%x", vector)) } }
https://github.com/google/gvisor/issues/25
runtime: newstack at runtime.printlock+0x76 sp=0xc4204cd9f8 stack=[0xc420051000, 0xc420052000] morebuf={pc:0x4293d3 sp:0xc4204cda00 lr:0x0} sched={pc:0x429d66 sp:0xc4204cd9f8 lr:0x0 ctxt:0x0} runtime.throw(0xb2537f, 0x8) GOROOT/src/runtime/panic.go:610 +0x13 fp=0xc4204cda20 sp=0xc4204cda00 pc=0x4293d3 gvisor.googlesource.com/gvisor/pkg/sentry/platform/kvm.bluepillHandler(0xc4204cdac0) pkg/sentry/platform/kvm/bluepill_unsafe.go:168 +0x38a fp=0xc4204cdab0 sp=0xc4204cda20 pc=0x81e79a gvisor.googlesource.com/gvisor/pkg/sentry/platform/kvm.sighandler(0x7, 0x0, 0xc4204c6000, 0x0, 0x8000, 0xc42008c418, 0xc42008c498, 0x1, 0xc42008c410, 0xc42008c410, ...) pkg/sentry/platform/kvm/bluepill_amd64.s:79 +0x24 fp=0xc4204cdac0 sp=0xc4204cdab0 pc=0x82c254 runtime: unexpected return pc for runtime.sigreturn called from 0x7 stack: frame={sp:0xc4204cdac0, fp:0xc4204cdac8} stack=[0xc420051000,0xc420052000) runtime.sigreturn(0x0, 0xc4204c6000, 0x0, 0x8000, 0xc42008c418, 0xc42008c498, 0x1, 0xc42008c410, 0xc42008c410, 0x0, ...) bazel-out/k8-fastbuild/bin/external/io_bazel_rules_go/linux_amd64_pure_stripped/stdlib~/src/runtime/sys_linux_amd64.s:444 fp=0xc4204cdac8 sp=0xc4204cdac0 pc=0x457340 created by gvisor.googlesource.com/gvisor/pkg/sentry/kernel.(*Task).Start pkg/sentry/kernel/task_start.go:251 +0x100 fatal error: runtime: stack split at bad time runtime stack: runtime.throw(0xb35d4b, 0x20) GOROOT/src/runtime/panic.go:616 +0x81 fp=0xc4204b3e48 sp=0xc4204b3e28 pc=0x429441 runtime.newstack() GOROOT/src/runtime/stack.go:954 +0xb61 fp=0xc4204b3fd8 sp=0xc4204b3e48 pc=0x442821 runtime.morestack() bazel-out/k8-fastbuild/bin/external/io_bazel_rules_go/linux_amd64_pure_stripped/stdlib~/src/runtime/asm_amd64.s:480 +0x89 fp=0xc4204b3fe0 sp=0xc4204b3fd8 pc=0x453669 goroutine 46 [syscall, locked to thread]: runtime.throw(0xb2537f, 0x8) GOROOT/src/runtime/panic.go:610 +0x13 fp=0xc4204cda20 sp=0xc4204cda00 pc=0x4293d3 gvisor.googlesource.com/gvisor/pkg/sentry/platform/kvm.bluepillHandler(0xc4204cdac0) pkg/sentry/platform/kvm/bluepill_unsafe.go:168 +0x38a fp=0xc4204cdab0 sp=0xc4204cda20 pc=0x81e79a gvisor.googlesource.com/gvisor/pkg/sentry/platform/kvm.sighandler(0x7, 0x0, 0xc4204c6000, 0x0, 0x8000, 0xc42008c418, 0xc42008c498, 0x1, 0xc42008c410, 0xc42008c410, ...) pkg/sentry/platform/kvm/bluepill_amd64.s:79 +0x24 fp=0xc4204cdac0 sp=0xc4204cdab0 pc=0x82c254 runtime: unexpected return pc for runtime.sigreturn called from 0x7 stack: frame={sp:0xc4204cdac0, fp:0xc4204cdac8} stack=[0xc420051000,0xc420052000) runtime.sigreturn(0x0, 0xc4204c6000, 0x0, 0x8000, 0xc42008c418, 0xc42008c498, 0x1, 0xc42008c410, 0xc42008c410, 0x0, ...) bazel-out/k8-fastbuild/bin/external/io_bazel_rules_go/linux_amd64_pure_stripped/stdlib~/src/runtime/sys_linux_amd64.s:444 fp=0xc4204cdac8 sp=0xc4204cdac0 pc=0x457340 created by gvisor.googlesource.com/gvisor/pkg/sentry/kernel.(*Task).Start pkg/sentry/kernel/task_start.go:251 +0x100
bluepill --> bluepillHandler -->c.CPU.SwitchToUser 执行顺序
root@cloud:~# dlv attach 953821 Type 'help' for list of commands. (dlv) b pkg/sentry/platform/kvm/machine_arm64_unsafe.go:249 Breakpoint 1 set at 0x87f508 for gvisor.dev/gvisor/pkg/sentry/platform/kvm.(*vCPU).SwitchToUser() pkg/sentry/platform/kvm/machine_arm64_unsafe.go:249 (dlv) b pkg/sentry/platform/kvm/machine_arm64_unsafe.go:250 Breakpoint 2 set at 0x87f514 for gvisor.dev/gvisor/pkg/sentry/platform/kvm.(*vCPU).SwitchToUser() pkg/sentry/platform/kvm/machine_arm64_unsafe.go:250 (dlv) b bluepillHandler Breakpoint 3 set at 0x87b300 for gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:91 (dlv) c > gvisor.dev/gvisor/pkg/sentry/platform/kvm.(*vCPU).SwitchToUser() pkg/sentry/platform/kvm/machine_arm64_unsafe.go:249 (hits goroutine(237):1 total:1) (PC: 0x87f508) Warning: debugging optimized function (dlv) c > gvisor.dev/gvisor/pkg/sentry/platform/kvm.bluepillHandler() pkg/sentry/platform/kvm/bluepill_unsafe.go:91 (hits goroutine(237):1 total:1) (PC: 0x87b300) Warning: debugging optimized function (dlv) c > gvisor.dev/gvisor/pkg/sentry/platform/kvm.(*vCPU).SwitchToUser() pkg/sentry/platform/kvm/machine_arm64_unsafe.go:250 (hits goroutine(237):1 total:1) (PC: 0x87f514) Warning: debugging optimized function (dlv) quit Would you like to kill the process? [Y/n] n