gdb) bt #0 posix_quic::Event::Trigger (event=4, this=0x839080) at /root/posix_quic/src/event.cpp:55 #1 posix_quic::Event::SetWritable (this=this@entry=0x839080, b=b@entry=true) at /root/posix_quic/src/event.cpp:117 #2 0x0000000000416660 in posix_quic::QuicSocketEntry::OnCryptoHandshakeComplete (this=0x839080) at /root/posix_quic/src/socket_entry.cpp:457 #3 0x00000000004cb940 in net::QuicCryptoClientHandshaker::DoReceiveSHLO(net::CryptoHandshakeMessage const*, net::QuicCryptoClientConfig::CachedState*) () #4 0x00000000004cc1c8 in net::QuicCryptoClientHandshaker::DoHandshakeLoop(net::CryptoHandshakeMessage const*) () #5 0x00000000004cc95c in net::QuicCryptoClientHandshaker::OnHandshakeMessage(net::CryptoHandshakeMessage const&) () #6 0x000000000048bf1c in net::CryptoFramer::Process(base::BasicStringPiece<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, net::Perspective) () #7 0x000000000048c70c in net::CryptoFramer::ProcessInput(base::BasicStringPiece<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, net::Perspective) () Backtrace stopped: previous frame identical to this frame (corrupt stack?)
class QuicEpollerEntry : public FdBase { public: struct quic_epoll_event { short int events; short int revents; epoll_data_t data; }; typedef std::unordered_map< int, std::pair<EntryWeakPtr, std::shared_ptr<quic_epoll_event>> > FdContainer; // 引用计数 typedef std::unordered_map<int, long> UdpContainer; struct EpollTrigger : public Event::EventTrigger { QuicEpollerEntry * epollEntry; void OnTrigger(short int event) override; void OnClose(Event* ) override; };
int QuicEpollerEntry::AddInner(int fd, struct epoll_event * event) { std::unique_lock<std::mutex> lock(mtx_); auto itr = fds_.find(fd); if (itr != fds_.end()) { EntryPtr entry = itr->second.first.lock(); if (entry && entry->Fd() == fd) { errno = EEXIST; return -1; } fds_.erase(itr); } EntryPtr entry = EntryBase::GetFdManager().Get(fd); if (!entry || entry->Fd() != fd) { errno = EBADF; return -1; } std::shared_ptr<quic_epoll_event> qev(new quic_epoll_event); qev->events = Epoll2Poll(event->events); qev->data = event->data; qev->revents = 0; Event::EventWaiter waiter = { &qev->events, &qev->revents }; if (!entry->StartWait(waiter, &trigger_)) { errno = EBADF; return -1; } fds_[fd] = std::make_pair(EntryWeakPtr(entry), qev); // listen udp std::shared_ptr<int> udpSocket = entry->NativeUdpFd(); if (!udpSocket) { // QuicSocket必须已经有了对应的udp socket才能加入Epoller. // 自行创建的QuicSocket, Bind后就可以了. entry->StopWait(&trigger_); errno = EINVAL; return -1; } if (entry->Category() == EntryCategory::Socket) { if (!((QuicSocketEntry*)entry.get())->GetQuicTaskRunnerProxy()->Link(&taskRunner_)) { entry->StopWait(&trigger_); errno = EBUSY; return -1; } } auto udpItr = udps_.find(*udpSocket); if (udps_.end() == udpItr) { struct epoll_event udpEv; udpEv.events = EPOLLIN; udpEv.data.fd = *udpSocket; int res = epoll_ctl(Fd(), EPOLL_CTL_ADD, *udpSocket, &udpEv); if (res < 0) { return -1; } udps_[*udpSocket] = 1; DebugPrint(dbg_epoll, "Add udp socket = %d, quicFd = %d", *udpSocket, fd); } else { udpItr->second++; DebugPrint(dbg_epoll, "Ref udp socket = %d, quicFd = %d", *udpSocket, fd); } return 0;
(gdb) bt #0 posix_quic::QuicEpollerEntry::AddInner (this=this@entry=0x835950, fd=fd@entry=2, event=event@entry=0xffffffff90b8) at /root/posix_quic/src/epoller_entry.cpp:115 #1 0x0000000000408a40 in posix_quic::QuicEpollerEntry::Add (this=this@entry=0x835950, fd=fd@entry=2, event=event@entry=0xffffffff90b8) at /root/posix_quic/src/epoller_entry.cpp:110 #2 0x00000000004155d8 in posix_quic::QuicEpollCtl (epfd=3, epfd@entry=3224376, op=op@entry=1, quicFd=quicFd@entry=2, event=0xffffffff90b8, event@entry=0xffffffff9148) at /root/posix_quic/src/quic_socket.cpp:462 #3 0x00000000004020e0 in doLoop (ep=3224376, ep@entry=3) at /root/posix_quic/test/client/src/client.cpp:61 #4 0x00000000004009c8 in main () at /root/posix_quic/test/client/src/client.cpp:149
udp socket
Thread 1 "client" hit Breakpoint 1, doLoop (ep=3749433, ep@entry=3) at /root/posix_quic/test/client/src/client.cpp:61 61 res = QuicEpollCtl(ep, EPOLL_CTL_ADD, stream, &ev); (gdb) p stream $1 = 2 (gdb) p fd $2 = 1 (gdb) s 59 ev.events = EPOLLIN; (gdb) list 54 QuicStream stream = QuicCreateStream(fd); 55 assert(stream > 0); 56 57 struct epoll_event ev; 58 ev.data.fd = stream; 59 ev.events = EPOLLIN; 60 //res = QuicEpollCtl(ep, EPOLL_CTL_MOD, stream, &ev); 61 res = QuicEpollCtl(ep, EPOLL_CTL_ADD, stream, &ev); 62 CHECK_RES(res, "epoll_ctl"); 63 (gdb) b /epoller_entry.cpp:148 No source file named /epoller_entry.cpp. Make breakpoint pending on future shared library load? (y or [n]) y Breakpoint 2 (/epoller_entry.cpp:148) pending. (gdb) b epoller_entry.cpp:148 Breakpoint 3 at 0x4083e8: file /root/posix_quic/src/epoller_entry.cpp, line 148. (gdb) c Continuing. Thread 1 "client" hit Breakpoint 3, posix_quic::QuicEpollerEntry::AddInner (this=this@entry=0x835950, fd=1718229288, fd@entry=2, event=event@entry=0xffffffff90b8) at /root/posix_quic/src/epoller_entry.cpp:148 148 std::shared_ptr<int> udpSocket = entry->NativeUdpFd(); (gdb) n 149 if (!udpSocket) { (gdb) p *udpSocket Could not find operator*. (gdb) p udpSocket->get() Could not find operator->. (gdb) p udpSocket $3 = {<std::__shared_ptr<int, (__gnu_cxx::_Lock_policy)2>> = {_M_ptr = 0x839350, _M_refcount = {_M_pi = 0x839370}}, <No data fields>} (gdb) p *(0x839350) $4 = 7 //udp socket 的句柄 (gdb) p *(0x839350)
QuicEpollerEntry::Poll
(gdb) bt #0 posix_quic::QuicEpollerEntry::Poll (this=this@entry=0x835950, events=0xffffffff9108, events@entry=0x41ef58 <posix_quic::QuicStreamEntry::GetQuartcStream()+128>, maxevents=1024, maxevents@entry=65535) at /root/posix_quic/src/epoller_entry.cpp:385 #1 0x0000000000406544 in posix_quic::QuicEpollerEntry::Wait (this=this@entry=0x835950, events=0x41ef58 <posix_quic::QuicStreamEntry::GetQuartcStream()+128>, events@entry=0xffffffff9108, maxevents=65535, maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/epoller_entry.cpp:263 #2 0x000000000041586c in posix_quic::QuicEpollWait (epfd=3, epfd@entry=3749376, events=events@entry=0xffffffff9108, maxevents=maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/quic_socket.cpp:494 #3 0x0000000000401dfc in doLoop (ep=3749376, ep@entry=3) at /root/posix_quic/test/client/src/client.cpp:37 #4 0x00000000004009c8 in main () at /root/posix_quic/test/client/src/client.cpp:149 (gdb)
int QuicEpollerEntry::Poll(struct epoll_event *events, int maxevents)
{
std::unique_lock<std::mutex> lock(mtx_);
int i = 0;
for (auto & kv : fds_) {
if (i >= maxevents) break;
quic_epoll_event & qev = *(kv.second.second);
short int event = qev.events | POLLERR;
// DebugPrint(dbg_event, "fd = %d, qev.revents = %s, event = %s",
// kv.first, PollEvent2Str(qev.revents), PollEvent2Str(event));
short int revents = __atomic_fetch_and(&qev.revents, ~event, std::memory_order_seq_cst);
revents &= event;
DebugPrint(dbg_event, "after __atomic_fetch_and fd = %d, qev.revents = %s, revents = %s",
kv.first, PollEvent2Str(qev.revents), PollEvent2Str(revents));
if (revents == 0) continue;
struct epoll_event & ev = events[i++];
ev.data = qev.data;
ev.events = Poll2Epoll(revents);
}
DebugPrint(dbg_epoll, "QuicEpollerEntry::Poll returns %d", i);
return i;
}
Event::EventTrigger::Trigger & wait
void Event::EventTrigger::Trigger(short int event) { { std::unique_lock<std::mutex> lock(cvMtx); triggered = true; cv.notify_one(); //条件变量 } OnTrigger(event); }
Thread 1 "client" hit Breakpoint 1, posix_quic::Event::TriggerWithoutLock (event=4, this=0x839080) at /root/posix_quic/src/event.cpp:82 82 trigger->Trigger(POLLOUT); (gdb) bt #0 posix_quic::Event::TriggerWithoutLock (event=4, this=0x839080) at /root/posix_quic/src/event.cpp:82 #1 posix_quic::Event::Trigger (event=4, this=0x839080) at /root/posix_quic/src/event.cpp:55 #2 posix_quic::Event::SetWritable (this=this@entry=0x839080, b=b@entry=true) at /root/posix_quic/src/event.cpp:117 #3 0x00000000004166a0 in posix_quic::QuicSocketEntry::OnCryptoHandshakeComplete (this=0x839080) at /root/posix_quic/src/socket_entry.cpp:457 #4 0x00000000004ca980 in net::QuicCryptoClientHandshaker::DoReceiveSHLO(net::CryptoHandshakeMessage const*, net::QuicCryptoClientConfig::CachedState*) () #5 0x00000000004cb208 in net::QuicCryptoClientHandshaker::DoHandshakeLoop(net::CryptoHandshakeMessage const*) () #6 0x00000000004cb99c in net::QuicCryptoClientHandshaker::OnHandshakeMessage(net::CryptoHandshakeMessage const&) () #7 0x000000000048af5c in net::CryptoFramer::Process(base::BasicStringPiece<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, net::Perspective) () #8 0x000000000048b74c in net::CryptoFramer::ProcessInput(base::BasicStringPiece<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, net::Perspective) () Backtrace stopped: previous frame identical to this frame (corrupt stack?) (gdb) c Continuing. [16:35:54.777784]client.cpp:47:(doLoop) [C=0] QuicEpoller trigger: fd = 1, category = Socket, events = EPOLLOUT [16:35:54.777874]client.cpp:51:(doLoop) [C=0] Connected. Thread 1 "client" hit Breakpoint 1, posix_quic::Event::TriggerWithoutLock (event=1, this=0x84bbc0) at /root/posix_quic/src/event.cpp:71 71 trigger->Trigger(POLLIN); (gdb) bt #0 posix_quic::Event::TriggerWithoutLock (event=1, this=0x84bbc0) at /root/posix_quic/src/event.cpp:71 #1 posix_quic::Event::Trigger (event=1, this=0x84bbc0) at /root/posix_quic/src/event.cpp:55 #2 posix_quic::Event::SetReadable (this=this@entry=0x84bbc0, b=b@entry=true) at /root/posix_quic/src/event.cpp:111 #3 0x000000000041e9dc in posix_quic::QuicStreamEntry::OnReceived (this=0x84bbc0, stream=<optimized out>, data=0x853970 "Hello quic!", size=11) at /root/posix_quic/src/stream_entry.cpp:152 #4 0x0000000000465494 in net::QuartcStream::OnDataAvailable() () #5 0x000000000045696c in net::QuicStreamSequencer::OnStreamFrame(net::QuicStreamFrame const&) () #6 0x0000000000450bb8 in net::QuicStream::OnStreamFrame(net::QuicStreamFrame const&) () #7 0x000000000044cb14 in net::QuicSession::OnStreamFrame(net::QuicStreamFrame const&) () #8 0x000000000043210c in net::QuicConnection::OnStreamFrame(net::QuicStreamFrame const&) () #9 0x000000000043fd64 in net::QuicFramer::ProcessFrameData(net::QuicDataReader*, net::QuicPacketHeader const&) [clone .part.162] [clone .constprop.172] () #10 0x00000000004402f8 in net::QuicFramer::ProcessDataPacket(net::QuicDataReader*, net::QuicPacketHeader*, net::QuicEncryptedPacket const&, char*, unsigned long) [clone .part.163] [clone .constprop.166] () #11 0x0000000000440724 in net::QuicFramer::ProcessPacket(net::QuicEncryptedPacket const&) () #12 0x00000000004349f8 in net::QuicConnection::ProcessUdpPacket(net::QuicSocketAddress const&, net::QuicSocketAddress const&, net::QuicReceivedPacket const&) () #13 0x0000000000417298 in posix_quic::QuicSocketEntry::ProcessUdpPacket (this=this@entry=0x839080, self_address=..., peer_address=..., packet=...) at /root/posix_quic/src/socket_entry.cpp:415 #14 0x0000000000407090 in posix_quic::QuicEpollerEntry::Wait (this=this@entry=0x835950, events=0x461830 <net::QuicSocketAddressImpl::QuicSocketAddressImpl(sockaddr_storage const&)+48>, events@entry=0xffffffff9118, maxevents=65535, maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/epoller_entry.cpp:376 #15 0x000000000041586c in posix_quic::QuicEpollWait (epfd=3, epfd@entry=3421952, events=events@entry=0xffffffff9118, maxevents=maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/quic_socket.cpp:494 #16 0x0000000000401dfc in doLoop (ep=3421952, ep@entry=3) at /root/posix_quic/test/client/src/client.cpp:37 #17 0x00000000004009c8 in main () at /root/posix_quic/test/client/src/client.cpp:149 (gdb) c
(gdb) bt #0 posix_quic::Event::TriggerWithoutLock (event=8, this=0x84bbc0) at /root/posix_quic/src/event.cpp:92 #1 posix_quic::Event::Trigger (event=<optimized out>, this=<optimized out>) at /root/posix_quic/src/event.cpp:55 #2 posix_quic::Event::SetError (this=0x84bbc0, err=<optimized out>, quicErr=<optimized out>) at /root/posix_quic/src/event.cpp:126 #3 0x000000000044b704 in net::QuicSession::CloseStreamInner(unsigned int, bool) () #4 0x0000000000465514 in net::QuartcStream::OnDataAvailable() () #5 0x000000000045625c in net::QuicStreamSequencer::MaybeCloseStream() () #6 0x0000000000456400 in net::QuicStreamSequencer::OnStreamFrame(net::QuicStreamFrame const&) () #7 0x0000000000450bb8 in net::QuicStream::OnStreamFrame(net::QuicStreamFrame const&) () #8 0x000000000044cb14 in net::QuicSession::OnStreamFrame(net::QuicStreamFrame const&) () #9 0x000000000043210c in net::QuicConnection::OnStreamFrame(net::QuicStreamFrame const&) () #10 0x000000000043fd64 in net::QuicFramer::ProcessFrameData(net::QuicDataReader*, net::QuicPacketHeader const&) [clone .part.162] [clone .constprop.172] () #11 0x00000000004402f8 in net::QuicFramer::ProcessDataPacket(net::QuicDataReader*, net::QuicPacketHeader*, net::QuicEncryptedPacket const&, char*, unsigned long) [clone .part.163] [clone .constprop.166] () #12 0x0000000000440724 in net::QuicFramer::ProcessPacket(net::QuicEncryptedPacket const&) () #13 0x00000000004349f8 in net::QuicConnection::ProcessUdpPacket(net::QuicSocketAddress const&, net::QuicSocketAddress const&, net::QuicReceivedPacket const&) () #14 0x0000000000417298 in posix_quic::QuicSocketEntry::ProcessUdpPacket (this=this@entry=0x839080, self_address=..., peer_address=..., packet=...) at /root/posix_quic/src/socket_entry.cpp:415 #15 0x0000000000407090 in posix_quic::QuicEpollerEntry::Wait (this=this@entry=0x835950, events=0x461830 <net::QuicSocketAddressImpl::QuicSocketAddressImpl(sockaddr_storage const&)+48>, events@entry=0xffffffff9118, maxevents=65535, maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/epoller_entry.cpp:376 #16 0x000000000041586c in posix_quic::QuicEpollWait (epfd=3, epfd@entry=3356724, events=events@entry=0xffffffff9118, maxevents=maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/quic_socket.cpp:494 #17 0x0000000000401dfc in doLoop (ep=3356724, ep@entry=3) at /root/posix_quic/test/client/src/client.cpp:37 #18 0x00000000004009c8 in main () at /root/posix_quic/test/client/src/client.cpp:149 (gdb) c
QuicEpollerEntry::Notify
(gdb) b QuicEpollerEntry::Notify Breakpoint 1 at 0x404164: QuicEpollerEntry::Notify. (2 locations) (gdb) r Starting program: /root/posix_quic/test/client/client [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". [New Thread 0xffffbf6fe010 (LWP 47445)] [New Thread 0xffffbeefd010 (LWP 47446)] Thread 1 "client" hit Breakpoint 1, posix_quic::QuicEpollerEntry::EpollTrigger::OnTrigger (this=0x835a30, event=4) at /root/posix_quic/src/epoller_entry.cpp:15 15 epollEntry->Notify(); (gdb) bt #0 posix_quic::QuicEpollerEntry::EpollTrigger::OnTrigger (this=0x835a30, event=4) at /root/posix_quic/src/epoller_entry.cpp:15 #1 0x000000000040d43c in posix_quic::Event::EventTrigger::Trigger (event=4, this=0x835a30) at /root/posix_quic/src/event.cpp:33 #2 posix_quic::Event::TriggerWithoutLock (event=4, this=0x839080) at /root/posix_quic/src/event.cpp:82 #3 posix_quic::Event::Trigger (event=4, this=0x839080) at /root/posix_quic/src/event.cpp:55 #4 posix_quic::Event::SetWritable (this=this@entry=0x839080, b=b@entry=true) at /root/posix_quic/src/event.cpp:117 #5 0x00000000004166a0 in posix_quic::QuicSocketEntry::OnCryptoHandshakeComplete (this=0x839080) at /root/posix_quic/src/socket_entry.cpp:457 #6 0x00000000004ca980 in net::QuicCryptoClientHandshaker::DoReceiveSHLO(net::CryptoHandshakeMessage const*, net::QuicCryptoClientConfig::CachedState*) () #7 0x00000000004cb208 in net::QuicCryptoClientHandshaker::DoHandshakeLoop(net::CryptoHandshakeMessage const*) () #8 0x00000000004cb99c in net::QuicCryptoClientHandshaker::OnHandshakeMessage(net::CryptoHandshakeMessage const&) () #9 0x000000000048af5c in net::CryptoFramer::Process(base::BasicStringPiece<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, net::Perspective) () #10 0x000000000048b74c in net::CryptoFramer::ProcessInput(base::BasicStringPiece<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, net::Perspective) () Backtrace stopped: previous frame identical to this frame (corrupt stack?) (gdb) c Continuing. [Switching to Thread 0xffffbf6fe010 (LWP 47445)] Thread 2 "client" hit Breakpoint 1, posix_quic::QuicEpollerEntry::Notify (this=0x835950) at /root/posix_quic/src/epoller_entry.cpp:19 19 { (gdb) bt #0 posix_quic::QuicEpollerEntry::Notify (this=0x835950) at /root/posix_quic/src/epoller_entry.cpp:19 #1 0x0000000000406364 in posix_quic::QuicEpollerEntry::<lambda()>::operator() (__closure=<optimized out>) at /root/posix_quic/src/epoller_entry.cpp:60 #2 std::_Bind_simple<posix_quic::QuicEpollerEntry::QuicEpollerEntry()::<lambda()>()>::_M_invoke<> (this=<optimized out>) at /usr/include/c++/5/functional:1531 #3 std::_Bind_simple<posix_quic::QuicEpollerEntry::QuicEpollerEntry()::<lambda()>()>::operator() (this=<optimized out>) at /usr/include/c++/5/functional:1520 #4 std::thread::_Impl<std::_Bind_simple<posix_quic::QuicEpollerEntry::QuicEpollerEntry()::<lambda()>()> >::_M_run(void) (this=0x835f70) at /usr/include/c++/5/thread:115 #5 0x0000000000652018 in execute_native_thread_routine () #6 0x0000000000603bb0 in start_thread (arg=0xfffffffff64f) at pthread_create.c:463 #7 0x00000000006f2b7c in thread_start () (gdb)
__atomic_fetch_and & __atomic_fetch_or
void Event::TriggerWithoutLock(short int event) { for (auto & kv : waitings_) { EventTrigger * trigger = kv.first; EventWaiter & waiter = kv.second; switch (event) { case POLLIN: if (*waiter.events & POLLIN) { DebugPrint(dbg_event, "fd = %d, epfd = %d, trigger event = POLLIN. waiter.revents = %s", Fd(), trigger->epollfd, PollEvent2Str(*waiter.revents)); __atomic_fetch_or(waiter.revents, POLLIN, std::memory_order_seq_cst); DebugPrint(dbg_event, "fd = %d, epfd = %d, trigger event = POLLIN. waiter.revents = %s", Fd(), trigger->epollfd, PollEvent2Str(*waiter.revents)); trigger->Trigger(POLLIN); } break; case POLLOUT: if (*waiter.events & POLLOUT) { DebugPrint(dbg_event, "fd = %d, epfd = %d, trigger event = POLLOUT. waiter.revents = %s", Fd(), trigger->epollfd, PollEvent2Str(*waiter.revents)); __atomic_fetch_or(waiter.revents, POLLOUT, std::memory_order_seq_cst); DebugPrint(dbg_event, "fd = %d, epfd = %d, trigger event = POLLOUT. waiter.revents = %s", Fd(), trigger->epollfd, PollEvent2Str(*waiter.revents)); trigger->Trigger(POLLOUT); } break; case POLLERR: DebugPrint(dbg_event, "fd = %d, epfd = %d, trigger event = POLLERR. waiter.revents = %s", Fd(), trigger->epollfd, PollEvent2Str(*waiter.revents)); __atomic_fetch_or(waiter.revents, POLLERR, std::memory_order_seq_cst); DebugPrint(dbg_event, "fd = %d, epfd = %d, trigger event = POLLERR. waiter.revents = %s", Fd(), trigger->epollfd, PollEvent2Str(*waiter.revents)); trigger->Trigger(POLLERR); break; default: break; } } }
int QuicEpollerEntry::Poll(struct epoll_event *events, int maxevents) { std::unique_lock<std::mutex> lock(mtx_); int i = 0; for (auto & kv : fds_) { if (i >= maxevents) break; quic_epoll_event & qev = *(kv.second.second); short int event = qev.events | POLLERR; // DebugPrint(dbg_event, "fd = %d, qev.revents = %s, event = %s", // kv.first, PollEvent2Str(qev.revents), PollEvent2Str(event)); short int revents = __atomic_fetch_and(&qev.revents, ~event, std::memory_order_seq_cst); revents &= event; DebugPrint(dbg_event, "after __atomic_fetch_and fd = %d, qev.revents = %s, revents = %s", kv.first, PollEvent2Str(qev.revents), PollEvent2Str(revents)); if (revents == 0) continue; struct epoll_event & ev = events[i++]; ev.data = qev.data; ev.events = Poll2Epoll(revents); } DebugPrint(dbg_epoll, "QuicEpollerEntry::Poll returns %d", i); return i; }
SetXXX函数
void SetReadable(bool b); void SetWritable(bool b); void SetError(int err, int quicErr = 0);
void QuartcStream::OnDataAvailable() { // Do not deliver data until the entire stream's data is available. if (deliver_on_complete_ && sequencer()->ReadableBytes() + sequencer()->NumBytesConsumed() < sequencer()->close_offset()) { return; } struct iovec iov; while (sequencer()->GetReadableRegion(&iov)) { DCHECK(delegate_); delegate_->OnReceived(this, reinterpret_cast<const char*>(iov.iov_base), iov.iov_len); sequencer()->MarkConsumed(iov.iov_len); } // All the data has been received if the sequencer is closed. // Notify the delegate by calling the callback function one more time with // iov_len = 0. if (sequencer()->IsClosed()) { OnFinRead(); delegate_->OnReceived(this, reinterpret_cast<const char*>(iov.iov_base), 0); } } void QuartcStream::OnClose() { QuicStream::OnClose(); DCHECK(delegate_); delegate_->OnClose(this); } void QuartcStream::OnStreamDataConsumed(size_t bytes_consumed) { QuicStream::OnStreamDataConsumed(bytes_consumed); DCHECK(delegate_); delegate_->OnBufferChanged(this); } void QuartcStream::OnDataBuffered( QuicStreamOffset offset, QuicByteCount data_length, const QuicReferenceCountedPointer<QuicAckListenerInterface>& ack_listener) { DCHECK(delegate_); delegate_->OnBufferChanged(this); } void QuartcStream::OnStreamFrameRetransmitted(QuicStreamOffset offset, QuicByteCount data_length, bool fin_retransmitted) { QuicStream::OnStreamFrameRetransmitted(offset, data_length, fin_retransmitted); DCHECK(delegate_); delegate_->OnBufferChanged(this); } void QuartcStream::OnStreamFrameLost(QuicStreamOffset offset, QuicByteCount data_length, bool fin_lost) { QuicStream::OnStreamFrameLost(offset, data_length, fin_lost); DCHECK(delegate_); delegate_->OnBufferChanged(this); }
posix_quic::Event::SetReadable
(gdb) bt #0 posix_quic::Event::TriggerWithoutLock (event=1, this=0x84bbc0) at /root/posix_quic/src/event.cpp:71 #1 posix_quic::Event::Trigger (event=1, this=0x84bbc0) at /root/posix_quic/src/event.cpp:55 #2 posix_quic::Event::SetReadable (this=this@entry=0x84bbc0, b=b@entry=true) at /root/posix_quic/src/event.cpp:111 #3 0x000000000041e9dc in posix_quic::QuicStreamEntry::OnReceived (this=0x84bbc0, stream=<optimized out>, data=0x853970 "Hello quic!", size=11) at /root/posix_quic/src/stream_entry.cpp:152 #4 0x0000000000465494 in net::QuartcStream::OnDataAvailable() () #5 0x000000000045696c in net::QuicStreamSequencer::OnStreamFrame(net::QuicStreamFrame const&) () #6 0x0000000000450bb8 in net::QuicStream::OnStreamFrame(net::QuicStreamFrame const&) () #7 0x000000000044cb14 in net::QuicSession::OnStreamFrame(net::QuicStreamFrame const&) () #8 0x000000000043210c in net::QuicConnection::OnStreamFrame(net::QuicStreamFrame const&) () #9 0x000000000043fd64 in net::QuicFramer::ProcessFrameData(net::QuicDataReader*, net::QuicPacketHeader const&) [clone .part.162] [clone .constprop.172] () #10 0x00000000004402f8 in net::QuicFramer::ProcessDataPacket(net::QuicDataReader*, net::QuicPacketHeader*, net::QuicEncryptedPacket const&, char*, unsigned long) [clone .part.163] [clone .constprop.166] () #11 0x0000000000440724 in net::QuicFramer::ProcessPacket(net::QuicEncryptedPacket const&) () #12 0x00000000004349f8 in net::QuicConnection::ProcessUdpPacket(net::QuicSocketAddress const&, net::QuicSocketAddress const&, net::QuicReceivedPacket const&) () #13 0x0000000000417298 in posix_quic::QuicSocketEntry::ProcessUdpPacket (this=this@entry=0x839080, self_address=..., peer_address=..., packet=...) at /root/posix_quic/src/socket_entry.cpp:415 #14 0x0000000000407090 in posix_quic::QuicEpollerEntry::Wait (this=this@entry=0x835950, events=0x461830 <net::QuicSocketAddressImpl::QuicSocketAddressImpl(sockaddr_storage const&)+48>, events@entry=0xffffffff9118, maxevents=65535, maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/epoller_entry.cpp:376 #15 0x000000000041586c in posix_quic::QuicEpollWait (epfd=3, epfd@entry=3421952, events=events@entry=0xffffffff9118, maxevents=maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/quic_socket.cpp:494 #16 0x0000000000401dfc in doLoop (ep=3421952, ep@entry=3) at /root/posix_quic/test/client/src/client.cpp:37 #17 0x00000000004009c8 in main () at /root/posix_quic/test/client/src/client.cpp:149 (gdb) c
Event::SetWritable
(gdb) bt #0 0x0000000000462f8c in net::QuartcPacketWriter::SetWritable() () #1 0x0000000000463380 in net::QuartcSession::OnTransportCanWrite() () #2 0x0000000000419f68 in posix_quic::QuicSocketEntry::Connect (this=this@entry=0x839080, addr=addr@entry=0xfffffffff978, addrlen=addrlen@entry=16) at /root/posix_quic/src/socket_entry.cpp:278 #3 0x0000000000410164 in posix_quic::QuicConnect (sock=sock@entry=1, addr=0xfffffffff978, addr@entry=0xfffffffff998, addrlen=addrlen@entry=16) at /root/posix_quic/src/quic_socket.cpp:98 #4 0x0000000000400984 in main () at /root/posix_quic/test/client/src/client.cpp:138 (gdb) c Continuing. [19:20:10.192801]epoller_entry.cpp:176:(AddInner) [C=0] Add udp socket = 7, quicFd = 1 [19:20:10.192889]epoller_entry.cpp:111:(Add) [C=0] fd = 1, events = EPOLLIN|EPOLLOUT [19:20:10.192906]epoller_entry.cpp:257:(Wait) [C=0] QuicEpollerEntry::Wait begin [19:20:10.192977]epoller_entry.cpp:410:(Poll) [C=0] QuicEpollerEntry::Poll returns 0 [19:20:10.192994]epoller_entry.cpp:410:(Poll) [C=0] QuicEpollerEntry::Poll returns 0 [19:20:10.193004]epoller_entry.cpp:381:(Wait) [C=0] QuicEpollerEntry::Wait returns 0 [19:20:10.193014]epoller_entry.cpp:257:(Wait) [C=0] QuicEpollerEntry::Wait begin [19:20:10.193023]epoller_entry.cpp:410:(Poll) [C=0] QuicEpollerEntry::Poll returns 0 [19:20:10.193868]epoller_entry.cpp:316:(Wait) [C=0] syscall -> recvfrom 127.0.0.1:9700. Udp socket = 7, bytes = 1200, errno = 0 [19:20:10.193887]epoller_entry.cpp:367:(Wait) [C=0] -> recvfrom. Udp socket = 7, connectionId = 15368493319691896021 is exists. [19:20:10.194419]epoller_entry.cpp:316:(Wait) [C=0] syscall -> recvfrom Uninitialized address. Udp socket = 7, bytes = -1, errno = 11 [19:20:10.194435]epoller_entry.cpp:410:(Poll) [C=0] QuicEpollerEntry::Poll returns 0 [19:20:10.194445]epoller_entry.cpp:381:(Wait) [C=0] QuicEpollerEntry::Wait returns 0 [19:20:10.194455]epoller_entry.cpp:257:(Wait) [C=0] QuicEpollerEntry::Wait begin [19:20:10.194464]epoller_entry.cpp:410:(Poll) [C=0] QuicEpollerEntry::Poll returns 0 [19:20:10.195023]epoller_entry.cpp:316:(Wait) [C=0] syscall -> recvfrom 127.0.0.1:9700. Udp socket = 7, bytes = 1200, errno = 0 [19:20:10.195038]epoller_entry.cpp:367:(Wait) [C=0] -> recvfrom. Udp socket = 7, connectionId = 15368493319691896021 is exists. Thread 1 "client" hit Breakpoint 1, posix_quic::Event::SetWritable (this=this@entry=0x839080, b=b@entry=true) at /root/posix_quic/src/event.cpp:114 114 { (gdb) bt #0 posix_quic::Event::SetWritable (this=this@entry=0x839080, b=b@entry=true) at /root/posix_quic/src/event.cpp:114 #1 0x00000000004166a8 in posix_quic::QuicSocketEntry::OnCryptoHandshakeComplete (this=0x839080) at /root/posix_quic/src/socket_entry.cpp:457 #2 0x00000000004ca980 in net::QuicCryptoClientHandshaker::DoReceiveSHLO(net::CryptoHandshakeMessage const*, net::QuicCryptoClientConfig::CachedState*) () #3 0x00000000004cb208 in net::QuicCryptoClientHandshaker::DoHandshakeLoop(net::CryptoHandshakeMessage const*) () #4 0x00000000004cb99c in net::QuicCryptoClientHandshaker::OnHandshakeMessage(net::CryptoHandshakeMessage const&) () #5 0x000000000048af5c in net::CryptoFramer::Process(base::BasicStringPiece<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, net::Perspective) () #6 0x000000000048b74c in net::CryptoFramer::ProcessInput(base::BasicStringPiece<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, net::Perspective) () Backtrace stopped: previous frame identical to this frame (corrupt stack?) (gdb) c
posix_quic::Event::SetError
(gdb) bt #0 posix_quic::Event::SetError (this=0x84cee0, err=9, quicErr=0) at /root/posix_quic/src/event.cpp:120 #1 0x000000000044b704 in net::QuicSession::CloseStreamInner(unsigned int, bool) () #2 0x0000000000465514 in net::QuartcStream::OnDataAvailable() () #3 0x000000000045625c in net::QuicStreamSequencer::MaybeCloseStream() () #4 0x0000000000456400 in net::QuicStreamSequencer::OnStreamFrame(net::QuicStreamFrame const&) () #5 0x0000000000450bb8 in net::QuicStream::OnStreamFrame(net::QuicStreamFrame const&) () #6 0x000000000044cb14 in net::QuicSession::OnStreamFrame(net::QuicStreamFrame const&) () #7 0x000000000043210c in net::QuicConnection::OnStreamFrame(net::QuicStreamFrame const&) () #8 0x000000000043fd64 in net::QuicFramer::ProcessFrameData(net::QuicDataReader*, net::QuicPacketHeader const&) [clone .part.162] [clone .constprop.172] () #9 0x00000000004402f8 in net::QuicFramer::ProcessDataPacket(net::QuicDataReader*, net::QuicPacketHeader*, net::QuicEncryptedPacket const&, char*, unsigned long) [clone .part.163] [clone .constprop.166] () #10 0x0000000000440724 in net::QuicFramer::ProcessPacket(net::QuicEncryptedPacket const&) () #11 0x00000000004349f8 in net::QuicConnection::ProcessUdpPacket(net::QuicSocketAddress const&, net::QuicSocketAddress const&, net::QuicReceivedPacket const&) () #12 0x00000000004172a0 in posix_quic::QuicSocketEntry::ProcessUdpPacket (this=this@entry=0x839080, self_address=..., peer_address=..., packet=...) at /root/posix_quic/src/socket_entry.cpp:415 #13 0x0000000000407098 in posix_quic::QuicEpollerEntry::Wait (this=this@entry=0x835950, events=0x461830 <net::QuicSocketAddressImpl::QuicSocketAddressImpl(sockaddr_storage const&)+48>, events@entry=0xffffffff9128, maxevents=65535, maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/epoller_entry.cpp:376 #14 0x0000000000415874 in posix_quic::QuicEpollWait (epfd=3, epfd@entry=3160372, events=events@entry=0xffffffff9128, maxevents=maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/quic_socket.cpp:494 #15 0x0000000000401e04 in doLoop (ep=3160372, ep@entry=3) at /root/posix_quic/test/client/src/client.cpp:37 #16 0x00000000004009cc in main () at /root/posix_quic/test/client/src/client.cpp:149
SetCloseByPeer
void QuartcSession::OnConnectionClosed(QuicErrorCode error, const QuicString& error_details, ConnectionCloseSource source) { QuicSession::OnConnectionClosed(error, error_details, source); DCHECK(session_delegate_); session_delegate_->OnConnectionClosed(error, error_details, source); }
Thread 1 "server" hit Breakpoint 1, 0x00000000004507c0 in net::QuicStream::OnConnectionClosed(net::QuicErrorCode, net::ConnectionCloseSource) () (gdb) bt #0 0x00000000004507c0 in net::QuicStream::OnConnectionClosed(net::QuicErrorCode, net::ConnectionCloseSource) () #1 0x000000000044a2cc in net::QuicSession::OnConnectionClosed(net::QuicErrorCode, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, net::ConnectionCloseSource) () #2 0x0000000000464aa0 in net::QuartcSession::OnConnectionClosed(net::QuicErrorCode, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, net::ConnectionCloseSource) () #3 0x000000000042d8f0 in net::QuicConnection::TearDownLocalConnectionState(net::QuicErrorCode, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, net::ConnectionCloseSource) () Backtrace stopped: previous frame identical to this frame (corrupt stack?)
Thread 1 "server" hit Breakpoint 2, 0x000000000042d8a8 in net::QuicConnection::TearDownLocalConnectionState(net::QuicErrorCode, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, net::ConnectionCloseSource) () (gdb) bt #0 0x000000000042d8a8 in net::QuicConnection::TearDownLocalConnectionState(net::QuicErrorCode, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, net::ConnectionCloseSource) () #1 0x00000000004305b0 in net::QuicConnection::CloseConnection(net::QuicErrorCode, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, net::ConnectionCloseBehavior) [clone .localalias.197] () #2 0x0000000000427140 in posix_quic::QuicConnectionVisitor::CheckForNoAckTimeout (this=0x191397d8) at /root/posix_quic/src/connection_visitor.cpp:46 #3 posix_quic::NoAckAlarmDelegate::OnAlarm (this=<optimized out>) at /root/posix_quic/src/connection_visitor.cpp:12 #4 0x00000000004621bc in net::(anonymous namespace)::QuartcAlarm::Run() () #5 0x0000000000421aec in posix_quic::QuicTaskRunner::RunOnce (this=this@entry=0x19123ce0) at /root/posix_quic/src/task_runner.cpp:80 #6 0x0000000000406550 in posix_quic::QuicEpollerEntry::Wait (this=this@entry=0x19123950, events=0x810000 <tunable_list+448>, events@entry=0xfffffab8c668, maxevents=0, maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/epoller_entry.cpp:272 #7 0x000000000041577c in posix_quic::QuicEpollWait (epfd=3, epfd@entry=0, events=events@entry=0xfffffab8c668, maxevents=maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/quic_socket.cpp:494 #8 0x0000000000401f74 in doLoop (ep=0, ep@entry=3, listenSock=0, listenSock@entry=1) at /root/posix_quic/test/server/src/server.cpp:48 #9 0x00000000004009ac in main () at /root/posix_quic/test/server/src/server.cpp:153 (gdb) quit
Event::EventTrigger::Wait 没有调用
void Event::EventTrigger::Wait(int timeout) { std::unique_lock<std::mutex> lock(cvMtx); if (triggered) { triggered = false; return ; } if (timeout > 0) { cv.wait_for(lock, std::chrono::milliseconds(timeout)); } else if (timeout == 0) { return ; } else { cv.wait(lock); } }
总结
QuicEpollerEntry::QuicEpollerEntry() { SetFd(epoll_create(10240)); trigger_.epollfd = Fd(); trigger_.epollEntry = this; } QuicEpollerEntry::AddInner Event::EventWaiter waiter = { &qev->events, &qev->revents }; if (!entry->StartWait(waiter, &trigger_)) { errno = EBADF; return -1; } fds_[fd] = std::make_pair(EntryWeakPtr(entry), qev);
1、 QuicEpollerEntry::Wait会调poll遍历fds_[fd]： <entry,event>
2、Event::TriggerWithoutLock会更改qev
void Event::TriggerWithoutLock(short int event) { for (auto & kv : waitings_) { EventTrigger * trigger = kv.first; EventWaiter & waiter = kv.second; switch (event) { case POLLIN: __atomic_fetch_or(waiter.revents, POLLIN, std::memory_order_seq_cst); trigger->Trigger(POLLIN); }
struct EventWaiter {
short int* events;
short int* revents;
};
EventWaiter初始化
EntryCategory::Socket EntryCategory::Stream都会调用QuicEpollCtl(ep, EPOLL_CTL_ADD
int QuicEpollerEntry::AddInner(int fd, struct epoll_event * event) std::shared_ptr<quic_epoll_event> qev(new quic_epoll_event); qev->events = Epoll2Poll(event->events); qev->data = event->data; qev->revents = 0; Event::EventWaiter waiter = { &qev->events, &qev->revents }; if (!entry->StartWait(waiter, &trigger_)) { errno = EBADF; return -1; }
EventWaiter 事件触发
void Event::TriggerWithoutLock(short int event) { for (auto & kv : waitings_) { EventTrigger * trigger = kv.first; EventWaiter & waiter = kv.second; switch (event) { case POLLIN: __atomic_fetch_or(waiter.revents, POLLIN, std::memory_order_seq_cst); trigger->Trigger(POLLIN); }
EventWaiter poll轮询
int QuicEpollerEntry::Poll(struct epoll_event *events, int maxevents) { std::unique_lock<std::mutex> lock(mtx_); int i = 0; for (auto & kv : fds_) { if (i >= maxevents) break; quic_epoll_event & qev = *(kv.second.second); short int event = qev.events | POLLERR; // DebugPrint(dbg_event, "fd = %d, qev.revents = %s, event = %s", // kv.first, PollEvent2Str(qev.revents), PollEvent2Str(event)); short int revents = __atomic_fetch_and(&qev.revents, ~event, std::memory_order_seq_cst); revents &= event; DebugPrint(dbg_event, "after __atomic_fetch_and fd = %d, qev.revents = %s, revents = %s", kv.first, PollEvent2Str(qev.revents), PollEvent2Str(revents)); if (revents == 0) continue; struct epoll_event & ev = events[i++]; ev.data = qev.data; ev.events = Poll2Epoll(revents); } DebugPrint(dbg_epoll, "QuicEpollerEntry::Poll returns %d", i); return i; }